hydra-crawler 1.4.6 → 2.0.2

package/dist/parsers/spelling.parser.js

@@ -1,19 +1,27 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SpellingParser = void 0;
-const dictionary_parser_1 = require("./dictionary.parser");
-class SpellingParser extends dictionary_parser_1.DictionaryParser {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { DictionaryParser } from './dictionary.parser';
+export class SpellingParser extends DictionaryParser {
     constructor(url, outcome, config) {
         super(outcome, config, 'spelling');
         this.url = url;
     }
-    async parseMatches(database, matches, _nonMatches) {
-        if (!this.url)
-            return;
-        if (matches.length > 0)
-            await database.setData(this.url, 'spelling', matches);
-        else
-            await database.unsetData(this.url, 'spelling');
+    parseMatches(database, matches, _nonMatches) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.url)
+                return;
+            if (matches.length > 0)
+                yield database.setData(this.url, 'spelling', matches);
+            else
+                yield database.unsetData(this.url, 'spelling');
+        });
     }
 }
-exports.SpellingParser = SpellingParser;
+//# sourceMappingURL=spelling.parser.js.map

package/dist/parsers/spelling.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"spelling.parser.js","sourceRoot":"","sources":["../../src/parsers/spelling.parser.ts"],"names":[],"mappings":";;;;;;;;;AAOA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,qBAAqB,CAAC;AAE1E,MAAM,OAAO,cAAe,SAAQ,gBAAmC;IACtE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,YAAY,CAAC,QAAyB,EAAE,OAAiB,EAAE,WAAqB;;YAC/F,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;;gBACzE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACrD,CAAC;KAAA;CACD"}

package/dist/parsers/string.parser.d.ts

@@ -0,0 +1,8 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { DataParser, IDataConfig } from './data.parser';
+export declare abstract class StringParser<T extends IDataConfig> extends DataParser<T> {
+    protected stringData: string | undefined;
+    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
+}

package/dist/parsers/string.parser.js

@@ -1,19 +1,16 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.StringParser = void 0;
-const utf_decoder_1 = require("../helpers/utf-decoder");
-const data_parser_1 = require("./data.parser");
-class StringParser extends data_parser_1.DataParser {
+import { UtfDecoder } from '../helpers/utf-decoder';
+import { DataParser } from './data.parser';
+export class StringParser extends DataParser {
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         if (!this.data)
             return;
         try {
-            this.stringData = utf_decoder_1.UtfDecoder.fromBuffer(this.data);
+            this.stringData = UtfDecoder.fromBuffer(this.data);
         }
         catch (ex) {
             // ignore
         }
     }
 }
-exports.StringParser = StringParser;
+//# sourceMappingURL=string.parser.js.map

package/dist/parsers/string.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"string.parser.js","sourceRoot":"","sources":["../../src/parsers/string.parser.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAKpD,OAAO,EAAE,UAAU,EAAe,MAAM,eAAe,CAAC;AAExD,MAAM,OAAgB,YAAoC,SAAQ,UAAa;IAG9E,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,IAAI;YACH,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACnD;QAAC,OAAO,EAAE,EAAE;YACZ,SAAS;SACT;IACF,CAAC;CACD"}

package/dist/parsers/text.parser.d.ts

@@ -0,0 +1,8 @@
+import { DatabaseService } from '../services/database.service';
+import { HtmlParser } from './html.parser';
+import { IDataConfig } from './data.parser';
+export declare abstract class TextParser<T extends IDataConfig> extends HtmlParser<T> {
+    static attemptExtract(html: string): string;
+    protected abstract parseText(database: DatabaseService, text: string): Promise<void>;
+    parse(database: DatabaseService): Promise<void>;
+}

package/dist/parsers/text.parser.js

@@ -1,11 +1,18 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TextParser = void 0;
-const extractor = require("unfluff");
-const html_parser_1 = require("./html.parser");
-class TextParser extends html_parser_1.HtmlParser {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import extractor from 'unfluff';
+import { HtmlParser } from './html.parser';
+export class TextParser extends HtmlParser {
     static attemptExtract(html) {
         try {
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
             const extracted = extractor(html);
             if (!extracted)
                 throw new Error('Unable to run unfluff');
@@ -19,17 +26,16 @@ class TextParser extends html_parser_1.HtmlParser {
             return '';
         }
     }
-    constructor(outcome, config, configKey) {
-        super(outcome, config, configKey);
-    }
-    async parse(database) {
-        if (!this.dom) {
-            await this.parseText(database, '');
-            return;
-        }
-        const html = this.dom.html();
-        const extracted = TextParser.attemptExtract(html);
-        await this.parseText(database, extracted);
+    parse(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.dom) {
+                yield this.parseText(database, '');
+                return;
+            }
+            const html = this.dom.html();
+            const extracted = TextParser.attemptExtract(html);
+            yield this.parseText(database, extracted);
+        });
     }
 }
-exports.TextParser = TextParser;
+//# sourceMappingURL=text.parser.js.map

package/dist/parsers/text.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.parser.js","sourceRoot":"","sources":["../../src/parsers/text.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,SAAS,MAAM,SAAS,CAAC;AAIhC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAG3C,MAAM,OAAgB,UAAkC,SAAQ,UAAa;IACrE,MAAM,CAAC,cAAc,CAAC,IAAY;QACxC,IAAI;YACH,sGAAsG;YACtG,MAAM,SAAS,GAAqB,SAAS,CAAC,IAAI,CAAC,CAAC;YACpD,IAAI,CAAC,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAEzD,OAAO,SAAS,CAAC,IAAI;iBAClB,KAAK,CAAC,KAAK,CAAC;iBACZ,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;iBACxC,IAAI,CAAC,IAAI,CAAC,CAAC;SACd;QAAC,OAAO,EAAE,EAAE;YACZ,OAAO,EAAE,CAAC;SACV;IACF,CAAC;IAIY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;gBACd,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACnC,OAAO;aACP;YAED,MAAM,IAAI,GAAW,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,SAAS,GAAW,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAE1D,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;KAAA;CACD"}

package/dist/parsers/words.parser.d.ts

@@ -0,0 +1,11 @@
+import { DatabaseService } from '../services/database.service';
+import { TextParser } from './text.parser';
+import { IDataConfig } from './data.parser';
+export interface IWordsConfig extends IDataConfig {
+    allowHyphenatedWords?: boolean;
+}
+export declare function isIWordsConfig(test: unknown): test is IWordsConfig;
+export declare abstract class WordsParser<T extends IWordsConfig> extends TextParser<T> {
+    protected abstract parseWords(database: DatabaseService, words: string[]): Promise<void>;
+    protected parseText(database: DatabaseService, text: string): Promise<void>;
+}

package/dist/parsers/words.parser.js

@@ -1,36 +1,40 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WordsParser = exports.isIWordsConfig = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const text_parser_1 = require("./text.parser");
-const data_parser_1 = require("./data.parser");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsTypeHasPropertyBooleanOrUndefined } from 'tscommons-es-core';
+import { TextParser } from './text.parser';
+import { isIDataConfig } from './data.parser';
 const WORD_WITH_HYPHEN = '(?<![a-z0-9])[a-z0-9](?:[-\']?[a-z0-9])*(?![a-z0-9])';
 const WORD_WITHOUT_HYPHEN = '(?<![a-z0-9])[a-z0-9](?:[\']?[a-z0-9])*(?![a-z0-9])';
-function isIWordsConfig(test) {
-    if (!data_parser_1.isIDataConfig(test))
+export function isIWordsConfig(test) {
+    if (!isIDataConfig(test))
         return false;
-    if (!tscommons_core_1.CommonsType.hasPropertyBooleanOrUndefined(test, 'allowHyphenatedWords'))
+    if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'allowHyphenatedWords'))
         return false;
     return true;
 }
-exports.isIWordsConfig = isIWordsConfig;
-class WordsParser extends text_parser_1.TextParser {
-    constructor(outcome, config, configKey) {
-        super(outcome, config, configKey);
-    }
-    async parseText(database, text) {
-        const wordsConfig = this.getConfig(isIWordsConfig);
-        if (!wordsConfig)
-            return;
-        const pattern = wordsConfig.allowHyphenatedWords ? new RegExp(WORD_WITH_HYPHEN, 'ig') : new RegExp(WORD_WITHOUT_HYPHEN, 'ig');
-        const words = [];
-        while (true) {
-            const result = pattern.exec(text);
-            if (result === null)
-                break;
-            words.push(result[0]);
-        }
-        await this.parseWords(database, words);
+export class WordsParser extends TextParser {
+    parseText(database, text) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const wordsConfig = this.getConfig(isIWordsConfig);
+            if (!wordsConfig)
+                return;
+            const pattern = wordsConfig.allowHyphenatedWords ? new RegExp(WORD_WITH_HYPHEN, 'ig') : new RegExp(WORD_WITHOUT_HYPHEN, 'ig');
+            const words = [];
+            while (true) {
+                const result = pattern.exec(text);
+                if (result === null)
+                    break;
+                words.push(result[0]);
+            }
+            yield this.parseWords(database, words);
+        });
     }
 }
-exports.WordsParser = WordsParser;
+//# sourceMappingURL=words.parser.js.map

package/dist/parsers/words.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"words.parser.js","sourceRoot":"","sources":["../../src/parsers/words.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,wCAAwC,EAAE,MAAM,mBAAmB,CAAC;AAI7E,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAE3D,MAAM,gBAAgB,GAAW,sDAAsD,CAAC;AACxF,MAAM,mBAAmB,GAAW,qDAAqD,CAAC;AAK1F,MAAM,UAAU,cAAc,CAAC,IAAa;IAC3C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,sBAAsB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,WAAoC,SAAQ,UAAa;IAG9D,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,MAAM,WAAW,GAA2B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YAC3E,IAAI,CAAC,WAAW;gBAAE,OAAO;YAEzB,MAAM,OAAO,GAAW,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;YAEtI,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,OAAO,IAAI,EAAE;gBACZ,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxD,IAAI,MAAM,KAAK,IAAI;oBAAE,MAAM;gBAE3B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACtB;YAED,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;KAAA;CACD"}

package/dist/queries/complex-english.query.d.ts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery';
+export declare const query: TQuery;

package/dist/queries/complex-english.query.js

@@ -1,42 +1,49 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const cheerio = require("cheerio");
-const tscommons_core_1 = require("tscommons-core");
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_url_1 = require("nodecommons-url");
-const complex_english_parser_1 = require("../parsers/complex-english.parser");
-const elist_1 = require("../enums/elist");
-const query = async (args, databaseService, lists, _expiry, parsersConfig) => {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as cheerio from 'cheerio';
+import { commonsArrayUnique } from 'tscommons-es-core';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult } from 'nodecommons-es-cli';
+import { commonsHttpReadUrlAsBuffer } from 'nodecommons-es-http';
+import { ComplexEnglishParser } from '../parsers/complex-english.parser';
+import { EList } from '../enums/elist';
+export const query = (args, databaseService, lists, _expiry, parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
     const domain = args.getString('domain');
     if (!domain) {
-        nodecommons_cli_1.CommonsOutput.error('No domain specified');
+        commonsOutputError('No domain specified');
         return;
     }
-    const parser = new complex_english_parser_1.ComplexEnglishParser(undefined, undefined, parsersConfig);
+    const parser = new ComplexEnglishParser(undefined, undefined, parsersConfig);
     const regexs = parser.getRegExs();
     const keyDictionary = parser.getDictionary();
     if (!keyDictionary)
         throw new Error('No dictionary available');
-    nodecommons_cli_1.CommonsOutput.doing(`Searching for complex english detections for domain ${domain}`);
+    commonsOutputDoing(`Searching for complex english detections for domain ${domain}`);
     const result = databaseService.getUrls().find({
         domain: domain,
-        status: hydra_crawler_ts_assets_1.EStatus.DONE,
+        status: EStatus.DONE,
         statusCode: 200,
         complexEnglish: { $exists: true }
-    });
+    }, {});
     const matches = [];
     let tally = 0;
     while (true) {
         tally++;
         if ((tally % 100) === 0)
-            nodecommons_cli_1.CommonsOutput.progress(`${tally}`);
-        const row = await result.next();
+            commonsOutputProgress(`${tally}`);
+        const row = yield result.next();
         if (row === null)
             break;
         const typecast = row;
         if (typecast.headers !== undefined && typecast.headers['content-type'] !== undefined) {
-            if (!parser.supports(typecast.headers['content-type'], lists.match(elist_1.EList.ALLOW, row.url)))
+            if (!parser.supports(typecast.headers['content-type'], lists.match(EList.ALLOW, row.url)))
                 continue;
         }
         matches.push({
@@ -44,44 +51,35 @@ const query = async (args, databaseService, lists, _expiry, parsersConfig) => {
             complexEnglish: typecast.complexEnglish
         });
     }
-    nodecommons_cli_1.CommonsOutput.result(tally);
+    commonsOutputResult(tally);
     for (const match of matches) {
         console.log('----------------------------------------------------');
         console.log(match.url);
-        const data = await nodecommons_url_1.CommonsUrl.readUrlAsBuffer(match.url);
+        const data = yield commonsHttpReadUrlAsBuffer(match.url);
         if (!data) {
-            nodecommons_cli_1.CommonsOutput.error('Unable to read URL. Skipping');
+            commonsOutputError('Unable to read URL. Skipping');
             continue;
         }
-        // @ts-ignore
         const dom = cheerio.load(data);
         if (!dom) {
-            nodecommons_cli_1.CommonsOutput.error('Unable to parse HTML. Skipping');
+            commonsOutputError('Unable to parse HTML. Skipping');
             continue;
         }
-        // @ts-ignore
-        // tslint:disable:no-invalid-this
         const nodes = dom('*')
             .contents()
-            .filter(function () {
-            // @ts-ignore
-            return this.nodeType === 3;
-        });
-        // tslint:enable:no-invalid-this
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+            .filter((_index, element) => element.nodeType === 3);
         const deconstruct = [];
-        // tslint:disable:no-invalid-this
-        dom(nodes).each(function () {
-            // @ts-ignore
-            deconstruct.push(dom(this).text());
+        dom(nodes).each((_index, element) => {
+            deconstruct.push(dom(element).text());
         });
-        // tslint:enable:no-invalid-this
         const lines = deconstruct
             .join('\n')
             .replace(/[\t\r\n]+/g, '\n')
             .split('\n')
             .map((s) => s.trim())
             .filter((s) => s !== '');
-        const unique = tscommons_core_1.CommonsArray.unique(lines);
+        const unique = commonsArrayUnique(lines);
         for (const line of unique) {
             let changed = line;
             for (const complex of match.complexEnglish) {
@@ -102,5 +100,6 @@ const query = async (args, databaseService, lists, _expiry, parsersConfig) => {
             }
         }
     }
-};
-exports.default = query;
+});
+// export default query;
+//# sourceMappingURL=complex-english.query.js.map

package/dist/queries/complex-english.query.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"complex-english.query.js","sourceRoot":"","sources":["../../src/queries/complex-english.query.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAc,MAAM,mBAAmB,CAAC;AAEnE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACrI,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAMzE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAUvC,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAAwC,EACzB,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,EAAE;QACZ,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO;KACP;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAmC,MAAM,CAAC,aAAa,EAAE,CAAC;IAC7E,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,kBAAkB,CAAC,uDAAuD,MAAM,EAAE,CAAC,CAAC;IAEpF,MAAM,MAAM,GAAiB,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACzD;QACE,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO,CAAC,IAAI;QACpB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAClC,EACD,EAAE,CACH,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE;QACZ,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;YAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,MAAM,QAAQ,GAAkB,GAA+B,CAAC;QAEhE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,SAAS,EAAE;YACrF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAClB,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAChC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAClC;gBAAE,SAAS;SACZ;QAED,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,cAAc,EAAE,QAAQ,CAAC,cAAc;SACxC,CAAC,CAAC;KACH;IAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAqB,MAAM,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE;YACV,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;YACnD,SAAS;SACT;QAED,MAAM,GAAG,GAA2B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,EAAE;YACT,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;YACrD,SAAS;SACT;QAED,MAAM,KAAK,GAAoB,GAAG,CAAC,GAAG,CAAC;aACpC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAElG,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAQ,EAAE;YAClE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAa,WAAW;aAChC,IAAI,CAAC,IAAI,CAAC;aACV,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAa,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;YAC1B,IAAI,OAAO,GAAW,IAAI,CAAC;YAE3B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,cAAc,EAAE;gBAC3C,MAAM,KAAK,GAAqB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAEhC,MAAM,WAAW,GAAa,aAAa,CAAC,OAAO,CAAC,CAAC;gBAErD,MAAM,KAAK,GAAa,CAAE,KAAK,OAAO,EAAE,CAAE,CAAC;gBAC3C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAErE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aAC7D;YAED,IAAI,OAAO,KAAK,IAAI,EAAE;gBACrB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aAChB;SACD;KACD;AACF,CAAC,CAAA,CAAC;AAEF,wBAAwB"}

package/dist/queries/flash-content.query.d.ts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery';
+export declare const query: TQuery;

package/dist/queries/flash-content.query.js

@@ -1,42 +1,50 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_2 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_3 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_file_1 = require("nodecommons-file");
-const query = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { ECommonsCsvColumnType } from 'tscommons-es-format';
+import { isIUrl } from 'hydra-crawler-ts-assets';
+import { isTLink } from 'hydra-crawler-ts-assets';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsCsv } from 'nodecommons-es-file';
+export const query = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
     const filename = args.getString('filename');
-    nodecommons_cli_1.CommonsOutput.doing('Searching for FLV and SWF URLs');
+    commonsOutputDoing('Searching for FLV and SWF URLs');
     const results = databaseService.getUrls().find({
         url: /\.(flv|swf)$/i,
         status: { $in: [
-                hydra_crawler_ts_assets_3.EStatus.ACTIVE,
-                hydra_crawler_ts_assets_3.EStatus.DENY,
-                hydra_crawler_ts_assets_3.EStatus.DISALLOWED,
-                hydra_crawler_ts_assets_3.EStatus.DONE,
-                hydra_crawler_ts_assets_3.EStatus.FAILED,
-                hydra_crawler_ts_assets_3.EStatus.QUEUED
+                EStatus.ACTIVE,
+                EStatus.DENY,
+                EStatus.DISALLOWED,
+                EStatus.DONE,
+                EStatus.FAILED,
+                EStatus.QUEUED
             ] }
-    });
-    const urls = await databaseService.listQueryResults(results, hydra_crawler_ts_assets_1.isIUrl);
-    nodecommons_cli_1.CommonsOutput.result(urls.length);
-    nodecommons_cli_1.CommonsOutput.doing(`Searching for outgoing links to URLs`);
+    }, {});
+    const urls = yield databaseService.listQueryResults(results, isIUrl);
+    commonsOutputResult(urls.length);
+    commonsOutputDoing('Searching for outgoing links to URLs');
     let tally = 0;
     const urlLinks = new Map();
     for (const url of urls) {
         const results3 = databaseService.getLinks().find({
             outgoing: url.url
-        });
-        const links = await databaseService.listQueryResults(results3, hydra_crawler_ts_assets_2.isTLink);
+        }, {});
+        const links = yield databaseService.listQueryResults(results3, isTLink);
         if (links.length === 0)
             continue;
         tally += links.length;
-        nodecommons_cli_1.CommonsOutput.progress(tally);
+        commonsOutputProgress(tally);
         urlLinks.set(url, links);
     }
-    nodecommons_cli_1.CommonsOutput.result(tally);
-    nodecommons_cli_1.CommonsOutput.doing(`Building CSV array`);
+    commonsOutputResult(tally);
+    commonsOutputDoing('Building CSV array');
     const rows = [];
     for (const url of urls) {
         for (const link of (urlLinks.get(url) || [])) {
@@ -58,17 +66,18 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
             return 1;
         return 0;
     });
-    nodecommons_cli_1.CommonsOutput.success();
-    const csv = new nodecommons_file_1.CommonsCsv([
+    commonsOutputSuccess();
+    const csv = new CommonsCsv([
         {
             name: 'src',
-            type: nodecommons_file_1.ECsvColumnType.STRING
+            type: ECommonsCsvColumnType.STRING
         },
         {
             name: 'dest',
-            type: nodecommons_file_1.ECsvColumnType.STRING
+            type: ECommonsCsvColumnType.STRING
         }
     ]);
     csv.save(rows, filename, true);
-};
-exports.default = query;
+});
+// export default query;
+//# sourceMappingURL=flash-content.query.js.map

package/dist/queries/flash-content.query.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flash-content.query.js","sourceRoot":"","sources":["../../src/queries/flash-content.query.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACvI,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAgBjD,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEpD,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAAiB,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC1D;QACE,GAAG,EAAE,eAAe;QACpB,MAAM,EAAE,EAAE,GAAG,EAAE;gBACb,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;gBAClB,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,MAAM;aACf,EAAE;KACJ,EACD,EAAE,CACH,CAAC;IACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,OAAO,EACP,MAAM,CACP,CAAC;IACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEjC,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,QAAQ,GAAuB,IAAI,GAAG,EAAiB,CAAC;IAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,QAAQ,GAAkB,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAC7D;YACE,QAAQ,EAAE,GAAG,CAAC,GAAG;SAClB,EACD,EAAE,CACH,CAAC;QACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;QAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;QACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAE7B,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;KACzB;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAW,EAAE,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE;YAC7C,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,IAAI,EAAE,GAAG,CAAC,GAAG;aACd,CAAC,CAAC;SACH;KACD;IACD,IAAI;SACD,IAAI,CAAC,CAAC,CAAO,EAAE,CAAO,EAAU,EAAE;QAClC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACV,CAAC,CAAC,CAAC;IAEL,oBAAoB,EAAE,CAAC;IAEvB,MAAM,GAAG,GAAe,IAAI,UAAU,CAAC;QACrC;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CACN,IAAI,EACJ,QAAQ,EACR,IAAI,CACL,CAAC;AACH,CAAC,CAAA,CAAC;AAEF,wBAAwB"}

package/dist/queries/linking-to-domains.query.d.ts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery';
+export declare const query: TQuery;

package/dist/queries/linking-to-domains.query.js

@@ -1,10 +1,17 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_2 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_3 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_4 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { isIUrl } from 'hydra-crawler-ts-assets';
+import { isTLink } from 'hydra-crawler-ts-assets';
+import { isTDomain } from 'hydra-crawler-ts-assets';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
 const domainSort = (a, b) => {
     if (a.startsWith('http://'))
         a = a.substr(7);
@@ -24,31 +31,31 @@ const domainSort = (a, b) => {
         return 1;
     return 0;
 };
-const query = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+export const query = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
     const regexStr = args.getString('regex');
     if (!regexStr) {
-        nodecommons_cli_1.CommonsOutput.error('No domain regex specified');
+        commonsOutputError('No domain regex specified');
         return;
     }
     const regex = new RegExp(regexStr);
-    nodecommons_cli_1.CommonsOutput.doing('Searching for matching domains');
+    commonsOutputDoing('Searching for matching domains');
     const results = databaseService.getDomains().find({
         domain: regex
-    });
-    const domains = await databaseService.listQueryResults(results, hydra_crawler_ts_assets_3.isTDomain);
-    nodecommons_cli_1.CommonsOutput.result(domains.length);
+    }, {});
+    const domains = yield databaseService.listQueryResults(results, isTDomain);
+    commonsOutputResult(domains.length);
     const domainUrls = new Map();
     for (const domain of domains) {
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for URLs for ${domain.domain}`);
+        commonsOutputDoing(`Searching for URLs for ${domain.domain}`);
         const results2 = databaseService.getUrls().find({
             domain: domain.domain,
-            status: { $ne: hydra_crawler_ts_assets_4.EStatus.ARCHIVED }
-        });
-        const urls = await databaseService.listQueryResults(results2, hydra_crawler_ts_assets_1.isIUrl);
-        nodecommons_cli_1.CommonsOutput.result(urls.length);
+            status: { $ne: EStatus.ARCHIVED }
+        }, {});
+        const urls = yield databaseService.listQueryResults(results2, isIUrl);
+        commonsOutputResult(urls.length);
         domainUrls.set(domain, urls);
     }
-    nodecommons_cli_1.CommonsOutput.doing(`Searching for outgoing links to URLs`);
+    commonsOutputDoing('Searching for outgoing links to URLs');
     let tally = 0;
     const domainUrlLinks = new Map();
     for (const domain of domains) {
@@ -59,20 +66,20 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
         for (const url of urls) {
             const results3 = databaseService.getLinks().find({
                 outgoing: url.url
-            });
-            const links = await databaseService.listQueryResults(results3, hydra_crawler_ts_assets_2.isTLink);
+            }, {});
+            const links = yield databaseService.listQueryResults(results3, isTLink);
             if (links.length === 0)
                 continue;
             tally += links.length;
-            nodecommons_cli_1.CommonsOutput.progress(tally);
+            commonsOutputProgress(tally);
             map.set(url, links);
         }
         if (map.size === 0)
             continue;
         domainUrlLinks.set(domain, map);
     }
-    nodecommons_cli_1.CommonsOutput.result(tally);
-    nodecommons_cli_1.CommonsOutput.doing(`Constructing JSON object`);
+    commonsOutputResult(tally);
+    commonsOutputDoing('Constructing JSON object');
     const outcome = {};
     for (const domain of domainUrlLinks.keys()) {
         outcome[domain.domain] = {};
@@ -81,7 +88,7 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
                 .map((link) => link.url);
         }
     }
-    nodecommons_cli_1.CommonsOutput.success();
+    commonsOutputSuccess();
     if (args.hasAttribute('domains-only')) {
         Object.keys(outcome)
             .sort(domainSort)
@@ -119,5 +126,6 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
     else if (args.hasAttribute('json-dump')) {
         console.log(JSON.stringify(outcome));
     }
-};
-exports.default = query;
+});
+// export default query;
+//# sourceMappingURL=linking-to-domains.query.js.map