hydra-crawler 2.8.4 → 3.0.2

package/dist/parsers/paragraphs.parser.mjs

@@ -0,0 +1,38 @@
+import { commonsStringSplitWords, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-esm-core';
+import { HtmlParser } from './html.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIParagraphsConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyNumberOrUndefined(test, 'paragraphWordsThreshold'))
+        return false;
+    return true;
+}
+export class ParagraphsParser extends HtmlParser {
+    static parseParagraphs(dom, paragraphsConfig) {
+        const nodes = dom('p,dd')
+            .contents()
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+            .filter((_index, element) => element.nodeType === 3);
+        const paragraphs = [];
+        dom(nodes).each((_index, element) => {
+            const paragraph = dom(element).text();
+            if (paragraphsConfig.paragraphWordsThreshold && commonsStringSplitWords(paragraph).length >= paragraphsConfig.paragraphWordsThreshold) {
+                paragraphs.push(paragraph);
+            }
+        });
+        return paragraphs
+            .map((p) => p.trim())
+            .filter((p) => p !== '');
+    }
+    async parse(database) {
+        if (!this.dom)
+            return; // invalid parse
+        const paragraphsConfig = this.getConfig(isIParagraphsConfig);
+        if (!paragraphsConfig)
+            return;
+        const paragraphs = ParagraphsParser.parseParagraphs(this.dom, paragraphsConfig);
+        await this.parseParagraphs(database, paragraphs);
+    }
+}
+//# sourceMappingURL=paragraphs.parser.mjs.map

package/dist/parsers/paragraphs.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"paragraphs.parser.mjs","sourceRoot":"","sources":["../../src/parsers/paragraphs.parser.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,uCAAuC,EAAE,MAAM,oBAAoB,CAAC;AAItG,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAK/D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,yBAAyB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,UAAa;IACjF,MAAM,CAAC,eAAe,CAC3B,GAAuB,EACvB,gBAAmC;QAEpC,MAAM,KAAK,GAAyB,GAAG,CAAC,MAAM,CAAC;aAC5C,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAY,EAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAE7E,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CACb,CAAC,MAAc,EAAE,OAAY,EAAQ,EAAE;YACtC,MAAM,SAAS,GAAW,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9C,IAAI,gBAAgB,CAAC,uBAAuB,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,gBAAgB,CAAC,uBAAuB,EAAE,CAAC;gBACvI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;QACF,CAAC,CACF,CAAC;QAEF,OAAO,UAAU;aACd,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC;IAIe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO,CAAC,gBAAgB;QAEvC,MAAM,gBAAgB,GAAgC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAC1F,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAE9B,MAAM,UAAU,GAAa,gBAAgB,CAAC,eAAe,CAC3D,IAAI,CAAC,GAAG,EACR,gBAAgB,CACjB,CAAC;QAEF,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;CACD"}

package/dist/parsers/{parser.d.ts → parser.d.mts}

@@ -1,14 +1,14 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParser } from '../interfaces/iparser';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IParserConfig } from '../interfaces/iparser-config';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParser } from '../interfaces/iparser.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
 export declare abstract class Parser<T extends IParserConfig = IParserConfig> implements IParser {
     protected outcome?: IRequestOutcome | undefined;
     private configKey?;
     private enabled;
     private parserConfig;
-    constructor(outcome?: IRequestOutcome | undefined, config?: TKeyObject<IParserConfig>, configKey?: string | undefined);
+    constructor(outcome?: IRequestOutcome | undefined, config?: TPropertyObject<IParserConfig>, configKey?: string | undefined);
     protected getConfig<P>(// this allows subclasses to get their config without having to have all the T attributes of their own children
     checker: (test: unknown) => test is P): P | undefined;
     isEnabled(): boolean;

package/dist/parsers/parser.mjs

@@ -0,0 +1,45 @@
+import { commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogError } from 'nodecommons-esm-log';
+import { isIParserConfig } from '../interfaces/iparser-config.mjs';
+export class Parser {
+    outcome;
+    configKey;
+    enabled = false;
+    parserConfig;
+    // these all have to be optional because of the Ctor approach used elsewhere
+    // similarly, we don't check types of the config (T) other than the enabled in TParserConfig
+    constructor(outcome, config, configKey) {
+        this.outcome = outcome;
+        this.configKey = configKey;
+        if (!config || !this.configKey)
+            return;
+        if (!commonsTypeHasPropertyT(config, this.configKey, isIParserConfig)) {
+            commonsLogError(`Invalid config for Parser (${this.configKey})`);
+        }
+        this.parserConfig = config[this.configKey];
+        this.enabled = this.parserConfig.enabled;
+    }
+    getConfig(// this allows subclasses to get their config without having to have all the T attributes of their own children
+    checker) {
+        if (!this.parserConfig)
+            return undefined;
+        if (!checker(this.parserConfig))
+            throw new Error(`Invalid parser config for ${this.configKey}`);
+        return this.parserConfig;
+    }
+    isEnabled() {
+        return this.enabled;
+    }
+    async init(_database) {
+        // subclasses can implement
+    }
+    async parse(_database) {
+        // subclasses can implement
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async links() {
+        // subclasses can implement
+        return [];
+    }
+}
+//# sourceMappingURL=parser.mjs.map

package/dist/parsers/parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.mjs","sourceRoot":"","sources":["../../src/parsers/parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAmB,MAAM,oBAAoB,CAAC;AAE9E,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAMtD,OAAO,EAAiB,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAElF,MAAM,OAAgB,MAAM;IAOf;IAEF;IARF,OAAO,GAAY,KAAK,CAAC;IACzB,YAAY,CAA0B;IAE9C,4EAA4E;IAC5E,4FAA4F;IAC5F,YACY,OAAyB,EACnC,MAAuC,EAC/B,SAAkB;QAFhB,YAAO,GAAP,OAAO,CAAkB;QAE3B,cAAS,GAAT,SAAS,CAAS;QAE3B,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAEvC,IAAI,CAAC,uBAAuB,CAAgB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,CAAC;YACtF,eAAe,CAAC,8BAA8B,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAM,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IAC1C,CAAC;IAES,SAAS,CAAK,+GAA+G;IACrI,OAAqC;QAEtC,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,SAAS,CAAC;QAEzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,SAAU,EAAE,CAAC,CAAC;QAEjG,OAAO,IAAI,CAAC,YAAiB,CAAC;IAC/B,CAAC;IAEM,SAAS;QACf,OAAO,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAIM,KAAK,CAAC,IAAI,CAAC,SAA0B;QAC3C,2BAA2B;IAC5B,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,SAA0B;QAC5C,2BAA2B;IAC5B,CAAC;IAED,4DAA4D;IACrD,KAAK,CAAC,KAAK;QACjB,2BAA2B;QAE3B,OAAO,EAAE,CAAC;IACX,CAAC;CACD"}

package/dist/parsers/php-error.parser.d.mts

@@ -0,0 +1,12 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { StringParser } from './string.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare class PhpErrorParser extends StringParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    supports(_contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/php-error.parser.mjs

@@ -0,0 +1,42 @@
+import { deriveEPhpErrorType } from 'hydra-crawler-esm-assets';
+import { StringParser } from './string.parser.mjs';
+const PHP_ERROR = '<br />\\s{1,2}<b>([A-Za-z ]+?)</b>:  ([^<]+?) in <b>([^<]+?)</b> on line <b>([0-9]+?)</b><br />';
+export class PhpErrorParser extends StringParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'phpError');
+        this.url = url;
+    }
+    async parse(database) {
+        if (!this.stringData || !this.url)
+            return;
+        const pattern = new RegExp(PHP_ERROR, 'g');
+        const errors = [];
+        while (true) {
+            const result = pattern.exec(this.stringData);
+            if (result === null)
+                break;
+            const type = deriveEPhpErrorType(result[1]);
+            if (type === undefined)
+                continue;
+            const message = result[2];
+            const file = result[3];
+            const line = parseInt(result[4], 10);
+            const error = {
+                type: type,
+                message: message,
+                file: file,
+                line: line
+            };
+            errors.push(error);
+        }
+        if (errors.length > 0)
+            await database.setData(this.url, 'phpErrors', errors);
+        else
+            await database.unsetData(this.url, 'phpErrors');
+    }
+    supports(_contentType, isAllow) {
+        return isAllow; // everything allowlisted
+    }
+}
+//# sourceMappingURL=php-error.parser.mjs.map

package/dist/parsers/php-error.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"php-error.parser.mjs","sourceRoot":"","sources":["../../src/parsers/php-error.parser.mts"],"names":[],"mappings":"AAGA,OAAO,EAAiB,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAO9E,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGnD,MAAM,SAAS,GAAW,iGAAiG,CAAC;AAE5H,MAAM,OAAO,cAAe,SAAQ,YAAyB;IAElD;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAE1C,MAAM,OAAO,GAAW,IAAI,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACnE,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM;YAE3B,MAAM,IAAI,GAA4B,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACrE,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YAEjC,MAAM,OAAO,GAAW,MAAM,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,IAAI,GAAW,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,IAAI,GAAW,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAE7C,MAAM,KAAK,GAAc;gBACvB,IAAI,EAAE,IAAI;gBACV,OAAO,EAAE,OAAO;gBAChB,IAAI,EAAE,IAAI;gBACV,IAAI,EAAE,IAAI;aACX,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;;YACxE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACtD,CAAC;IAEM,QAAQ,CAAC,YAAoB,EAAE,OAAgB;QACrD,OAAO,OAAO,CAAC,CAAC,yBAAyB;IAC1C,CAAC;CACD"}

package/dist/parsers/{phrase.parser.d.ts → phrase.parser.d.mts}

@@ -1,6 +1,6 @@
-import { DatabaseService } from '../services/database.service';
-import { RegexParser } from './regex.parser';
-import { IDictionaryConfig } from './dictionary.parser';
+import { DatabaseService } from '../services/database.service.mjs';
+import { RegexParser } from './regex.parser.mjs';
+import { IDictionaryConfig } from './dictionary.parser.mjs';
 export declare abstract class PhraseParser<T extends IDictionaryConfig> extends RegexParser<T> {
     setPhrases(phrases: string[], caseInsensitive: boolean): void;
     protected abstract parsePhrases(database: DatabaseService, phrases: string[]): Promise<void>;

package/dist/parsers/phrase.parser.mjs

@@ -0,0 +1,15 @@
+import { RegexParser } from './regex.parser.mjs';
+export class PhraseParser extends RegexParser {
+    setPhrases(phrases, caseInsensitive) {
+        const map = new Map();
+        for (const phrase of phrases) {
+            const regex = new RegExp(`(^|[^0-9A-Za-z])${phrase}($|[^0-9A-Za-z])`, `g${caseInsensitive ? 'i' : ''}`);
+            map.set(phrase, regex);
+        }
+        this.setRegExs(map);
+    }
+    async parseRegExs(database, matches) {
+        await this.parsePhrases(database, Array.from(matches.keys()));
+    }
+}
+//# sourceMappingURL=phrase.parser.mjs.map

package/dist/parsers/phrase.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"phrase.parser.mjs","sourceRoot":"","sources":["../../src/parsers/phrase.parser.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,OAAgB,YAA0C,SAAQ,WAAc;IAC9E,UAAU,CACf,OAAiB,EACjB,eAAwB;QAEzB,MAAM,GAAG,GAAwB,IAAI,GAAG,EAAkB,CAAC;QAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,mBAAmB,MAAM,kBAAkB,EAAE,IAAI,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChH,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAIS,KAAK,CAAC,WAAW,CAAC,QAAyB,EAAE,OAAuC;QAC7F,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;CACD"}

package/dist/parsers/{regex.parser.d.ts → regex.parser.d.mts}

@@ -1,6 +1,6 @@
-import { DatabaseService } from '../services/database.service';
-import { TextParser } from './text.parser';
-import { IDataConfig } from './data.parser';
+import { DatabaseService } from '../services/database.service.mjs';
+import { TextParser } from './text.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export declare abstract class RegexParser<T extends IDataConfig> extends TextParser<T> {
     private regexs;
     protected setRegExs(regexs: Map<string, RegExp>): void;

package/dist/parsers/regex.parser.mjs

@@ -0,0 +1,29 @@
+import { TextParser } from './text.parser.mjs';
+export class RegexParser extends TextParser {
+    regexs = new Map();
+    setRegExs(regexs) {
+        for (const key of regexs.keys()) {
+            this.regexs.set(key, regexs.get(key));
+        }
+    }
+    getRegExs() {
+        return this.regexs;
+    }
+    async parseText(database, text) {
+        const matches = new Map();
+        for (const key of this.regexs.keys()) {
+            const regex = this.regexs.get(key);
+            const results = [];
+            while (true) {
+                const result = regex.exec(text);
+                if (!result || result === null)
+                    break;
+                results.push(result);
+            }
+            if (results.length > 0)
+                matches.set(key, results);
+        }
+        await this.parseRegExs(database, matches);
+    }
+}
+//# sourceMappingURL=regex.parser.mjs.map

package/dist/parsers/regex.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"regex.parser.mjs","sourceRoot":"","sources":["../../src/parsers/regex.parser.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,MAAM,OAAgB,WAAmC,SAAQ,UAAa;IACrE,MAAM,GAAwB,IAAI,GAAG,EAAkB,CAAC;IAEtD,SAAS,CAAC,MAA2B;QAC9C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;IAEM,SAAS;QACf,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;IAIS,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,MAAM,OAAO,GAAmC,IAAI,GAAG,EAA6B,CAAC;QACrF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAW,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YAE5C,MAAM,OAAO,GAAsB,EAAE,CAAC;YACtC,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,MAAM,GAAyB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,IAAI;oBAAE,MAAM;gBAEtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;CACD"}

package/dist/parsers/server.parser.d.mts

@@ -0,0 +1,11 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { Parser } from './parser.mjs';
+export declare class ServerParser extends Parser<IParserConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    supports(_contentType: string): boolean;
+    parse(database: DatabaseService): Promise<void>;
+}

package/dist/parsers/server.parser.mjs

@@ -0,0 +1,57 @@
+import { EServer } from '../enums/eserver.mjs';
+import { EXPoweredBy } from '../enums/ex-powered-by.mjs';
+import { Parser } from './parser.mjs';
+export class ServerParser extends Parser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'server');
+        this.url = url;
+    }
+    supports(_contentType) {
+        return true; // everything
+    }
+    async parse(database) {
+        if (!this.outcome || !this.url)
+            return;
+        const server = this.outcome.headers['server'] || undefined;
+        const xPoweredBy = this.outcome.headers['x-powered-by'] || undefined;
+        if (server === undefined && xPoweredBy === undefined) {
+            await database.unsetData(this.url, 'server');
+            return;
+        }
+        const data = {};
+        if (server) {
+            if (/^Apache($|\/)/.test(server))
+                data.server = EServer.APACHE;
+            if (/^Microsoft-IIS($|\/)/.test(server))
+                data.server = EServer.IIS;
+            if (/^nginx($|\/)/.test(server))
+                data.server = EServer.NGINX;
+            if (/^cloudflare($|\/)/.test(server))
+                data.server = EServer.CLOUDFLARE;
+            if (/^AmazonS3($|\/)/.test(server))
+                data.server = EServer.AMAZONS3;
+            if (/^Jetty($|\/)/.test(server))
+                data.server = EServer.JETTY;
+        }
+        if (xPoweredBy) {
+            if (/^PHP($|\/)/.test(xPoweredBy))
+                data.xPoweredBy = EXPoweredBy.PHP;
+            if (/^ASP.NET($|\/)/.test(xPoweredBy))
+                data.xPoweredBy = EXPoweredBy.ASPNET;
+            if (/^Express($|\/)/.test(xPoweredBy))
+                data.xPoweredBy = EXPoweredBy.EXPRESS;
+            if (/^Zope($|\/)/.test(xPoweredBy))
+                data.xPoweredBy = EXPoweredBy.ZOPE;
+        }
+        if (data.server === undefined && data.xPoweredBy === undefined) {
+            await database.unsetData(this.url, 'server');
+            return;
+        }
+        try {
+            await database.setData(this.url, 'server', data);
+        }
+        catch (_ex) { /* do nothing */ }
+    }
+}
+//# sourceMappingURL=server.parser.mjs.map

package/dist/parsers/server.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.parser.mjs","sourceRoot":"","sources":["../../src/parsers/server.parser.mts"],"names":[],"mappings":"AAQA,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAEzD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,MAAM,OAAO,YAAa,SAAQ,MAAqB;IAE5C;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAJxB,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEM,QAAQ,CAAC,YAAoB;QACnC,OAAO,IAAI,CAAC,CAAC,aAAa;IAC3B,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEvC,MAAM,MAAM,GAAsB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAY,IAAI,SAAS,CAAC;QACzF,MAAM,UAAU,GAAsB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAY,IAAI,SAAS,CAAC;QAEnG,IAAI,MAAM,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YACtD,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAC7C,OAAO;QACR,CAAC;QAED,MAAM,IAAI,GAAY,EAAE,CAAC;QAEzB,IAAI,MAAM,EAAE,CAAC;YACZ,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC/D,IAAI,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;YACnE,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;YAC7D,IAAI,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;YACvE,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;YACnE,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC9D,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YAChB,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC;YACrE,IAAI,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;YAC5E,IAAI,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC;YAC7E,IAAI,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC;QACxE,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAC7C,OAAO;QACR,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACnC,CAAC;CACD"}

package/dist/parsers/{spelling.parser.d.ts → spelling.parser.d.mts}

@@ -1,10 +1,10 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { DictionaryParser, IDictionaryConfig } from './dictionary.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { DictionaryParser, IDictionaryConfig } from './dictionary.parser.mjs';
 export declare class SpellingParser extends DictionaryParser<IDictionaryConfig> {
     private url?;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseMatches(database: DatabaseService, matches: string[], _nonMatches: string[]): Promise<void>;
 }

package/dist/parsers/spelling.parser.mjs

@@ -0,0 +1,17 @@
+import { DictionaryParser } from './dictionary.parser.mjs';
+export class SpellingParser extends DictionaryParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'spelling');
+        this.url = url;
+    }
+    async parseMatches(database, matches, _nonMatches) {
+        if (!this.url)
+            return;
+        if (matches.length > 0)
+            await database.setData(this.url, 'spelling', matches);
+        else
+            await database.unsetData(this.url, 'spelling');
+    }
+}
+//# sourceMappingURL=spelling.parser.mjs.map

package/dist/parsers/spelling.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"spelling.parser.mjs","sourceRoot":"","sources":["../../src/parsers/spelling.parser.mts"],"names":[],"mappings":"AAOA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,yBAAyB,CAAC;AAE9E,MAAM,OAAO,cAAe,SAAQ,gBAAmC;IAE5D;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAES,KAAK,CAAC,YAAY,CAAC,QAAyB,EAAE,OAAiB,EAAE,WAAqB;QAC/F,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;;YACzE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACrD,CAAC;CACD"}

package/dist/parsers/string.parser.d.mts

@@ -0,0 +1,8 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { DataParser, IDataConfig } from './data.parser.mjs';
+export declare abstract class StringParser<T extends IDataConfig> extends DataParser<T> {
+    protected stringData: string | undefined;
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
+}

package/dist/parsers/{string.parser.js → string.parser.mjs}

@@ -1,6 +1,7 @@
-import { UtfDecoder } from '../helpers/utf-decoder';
-import { DataParser } from './data.parser';
+import { UtfDecoder } from '../helpers/utf-decoder.mjs';
+import { DataParser } from './data.parser.mjs';
 export class StringParser extends DataParser {
+    stringData;
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         if (!this.data)
@@ -8,9 +9,9 @@ export class StringParser extends DataParser {
         try {
             this.stringData = UtfDecoder.fromBuffer(this.data);
         }
-        catch (ex) {
+        catch (_ex) {
             // ignore
         }
     }
 }
-//# sourceMappingURL=string.parser.js.map
+//# sourceMappingURL=string.parser.mjs.map

package/dist/parsers/string.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"string.parser.mjs","sourceRoot":"","sources":["../../src/parsers/string.parser.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAKxD,OAAO,EAAE,UAAU,EAAe,MAAM,mBAAmB,CAAC;AAE5D,MAAM,OAAgB,YAAoC,SAAQ,UAAa;IACpE,UAAU,CAAmB;IAEvC,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,IAAI,CAAC;YACJ,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,SAAS;QACV,CAAC;IACF,CAAC;CACD"}

package/dist/parsers/{text.parser.d.ts → text.parser.d.mts}

@@ -1,6 +1,6 @@
-import { DatabaseService } from '../services/database.service';
-import { HtmlParser } from './html.parser';
-import { IDataConfig } from './data.parser';
+import { DatabaseService } from '../services/database.service.mjs';
+import { HtmlParser } from './html.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export declare abstract class TextParser<T extends IDataConfig> extends HtmlParser<T> {
     static attemptExtract(html: string): string;
     protected abstract parseText(database: DatabaseService, text: string): Promise<void>;

package/dist/parsers/text.parser.mjs

@@ -0,0 +1,30 @@
+import extractor from 'unfluff';
+import { HtmlParser } from './html.parser.mjs';
+export class TextParser extends HtmlParser {
+    static attemptExtract(html) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+            const extracted = extractor(html);
+            if (!extracted)
+                throw new Error('Unable to run unfluff');
+            return extracted.text
+                .split(/\n+/)
+                .map((s) => s.trim())
+                .filter((s) => s !== '')
+                .join('\n');
+        }
+        catch (_ex) {
+            return '';
+        }
+    }
+    async parse(database) {
+        if (!this.dom) {
+            await this.parseText(database, '');
+            return;
+        }
+        const html = this.dom.html();
+        const extracted = TextParser.attemptExtract(html);
+        await this.parseText(database, extracted);
+    }
+}
+//# sourceMappingURL=text.parser.mjs.map

package/dist/parsers/text.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.parser.mjs","sourceRoot":"","sources":["../../src/parsers/text.parser.mts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,SAAS,CAAC;AAIhC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,MAAM,OAAgB,UAAkC,SAAQ,UAAa;IACrE,MAAM,CAAC,cAAc,CAAC,IAAY;QACxC,IAAI,CAAC;YACJ,sGAAsG;YACtG,MAAM,SAAS,GAAqB,SAAS,CAAC,IAAI,CAAC,CAAC;YACpD,IAAI,CAAC,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAEzD,OAAO,SAAS,CAAC,IAAI;iBAClB,KAAK,CAAC,KAAK,CAAC;iBACZ,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;iBACxC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,EAAE,CAAC;QACX,CAAC;IACF,CAAC;IAIe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACnC,OAAO;QACR,CAAC;QAED,MAAM,IAAI,GAAW,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACrC,MAAM,SAAS,GAAW,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAE1D,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC3C,CAAC;CACD"}

package/dist/parsers/{words.parser.d.ts → words.parser.d.mts}

@@ -1,6 +1,6 @@
-import { DatabaseService } from '../services/database.service';
-import { TextParser } from './text.parser';
-import { IDataConfig } from './data.parser';
+import { DatabaseService } from '../services/database.service.mjs';
+import { TextParser } from './text.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IWordsConfig extends IDataConfig {
     allowHyphenatedWords?: boolean;
 }

package/dist/parsers/words.parser.mjs

@@ -0,0 +1,29 @@
+import { commonsTypeHasPropertyBooleanOrUndefined } from 'tscommons-esm-core';
+import { TextParser } from './text.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+const WORD_WITH_HYPHEN = '(?<![a-z0-9])[a-z0-9](?:[-\']?[a-z0-9])*(?![a-z0-9])';
+const WORD_WITHOUT_HYPHEN = '(?<![a-z0-9])[a-z0-9](?:[\']?[a-z0-9])*(?![a-z0-9])';
+export function isIWordsConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'allowHyphenatedWords'))
+        return false;
+    return true;
+}
+export class WordsParser extends TextParser {
+    async parseText(database, text) {
+        const wordsConfig = this.getConfig(isIWordsConfig);
+        if (!wordsConfig)
+            return;
+        const pattern = wordsConfig.allowHyphenatedWords ? new RegExp(WORD_WITH_HYPHEN, 'ig') : new RegExp(WORD_WITHOUT_HYPHEN, 'ig');
+        const words = [];
+        while (true) {
+            const result = pattern.exec(text);
+            if (result === null)
+                break;
+            words.push(result[0]);
+        }
+        await this.parseWords(database, words);
+    }
+}
+//# sourceMappingURL=words.parser.mjs.map

package/dist/parsers/words.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"words.parser.mjs","sourceRoot":"","sources":["../../src/parsers/words.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,MAAM,oBAAoB,CAAC;AAI9E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAE/D,MAAM,gBAAgB,GAAW,sDAAsD,CAAC;AACxF,MAAM,mBAAmB,GAAW,qDAAqD,CAAC;AAK1F,MAAM,UAAU,cAAc,CAAC,IAAa;IAC3C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,sBAAsB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,WAAoC,SAAQ,UAAa;IAGpE,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,MAAM,WAAW,GAA2B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QAC3E,IAAI,CAAC,WAAW;YAAE,OAAO;QAEzB,MAAM,OAAO,GAAW,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;QAEtI,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxD,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM;YAE3B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;CACD"}

package/dist/queries/complex-english.query.d.mts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery.mjs';
+export declare const QUERY: TQuery;

package/dist/queries/{complex-english.query.js → complex-english.query.mjs}

@@ -1,23 +1,14 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 import * as cheerio from 'cheerio';
-import { commonsArrayUnique } from 'tscommons-es-core';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult } from 'nodecommons-es-cli';
-import { commonsHttpReadUrlAsBuffer } from 'nodecommons-es-http';
-import { ComplexEnglishParser } from '../parsers/complex-english.parser';
-import { EList } from '../enums/elist';
-export const QUERY = (args, databaseService, lists, _expiry, parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
-    const domain = args.getString('domain');
+import { commonsArrayUnique } from 'tscommons-esm-core';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
+import { commonsHttpReadUrlAsBuffer } from 'nodecommons-esm-http';
+import { ComplexEnglishParser } from '../parsers/complex-english.parser.mjs';
+import { EList } from '../enums/elist.mjs';
+export const QUERY = async (args, databaseService, lists, _expiry, parsersConfig) => {
+    const domain = args.getFirstValue('domain');
     if (!domain) {
-        commonsOutputError('No domain specified');
+        commonsLogError('No domain specified');
         return;
     }
     const parser = new ComplexEnglishParser(undefined, undefined, parsersConfig);
@@ -25,7 +16,7 @@ export const QUERY = (args, databaseService, lists, _expiry, parsersConfig) => _
     const keyDictionary = parser.getDictionary();
     if (!keyDictionary)
         throw new Error('No dictionary available');
-    commonsOutputDoing(`Searching for complex english detections for domain ${domain}`);
+    const log = commonsLogDoing('complex-english-parser', `Searching for complex english detections for domain ${domain}`);
     const result = databaseService.getUrls().find({
         domain: domain,
         status: EStatus.DONE,
@@ -37,32 +28,39 @@ export const QUERY = (args, databaseService, lists, _expiry, parsersConfig) => _
     while (true) {
         tally++;
         if ((tally % 100) === 0)
-            commonsOutputProgress(`${tally}`);
-        const row = yield result.next();
+            log.progress(`${tally}`);
+        const row = await result.next();
         if (row === null)
             break;
         const typecast = row;
         if (typecast.headers !== undefined && typecast.headers['content-type'] !== undefined) {
-            if (!parser.supports(typecast.headers['content-type'], lists.match(EList.ALLOW, row.url)))
+            try {
+                if (!parser.supports(typecast.headers['content-type'], lists.match(EList.ALLOW, row.url)))
+                    continue;
+            }
+            catch (_e) {
+                // probably an invalid URL
+                // so ignore
                 continue;
+            }
         }
         matches.push({
             url: row.url,
             complexEnglish: typecast.complexEnglish
         });
     }
-    commonsOutputResult(tally);
+    log.result(tally);
     for (const match of matches) {
         console.log('----------------------------------------------------');
         console.log(match.url);
-        const data = yield commonsHttpReadUrlAsBuffer(match.url);
+        const data = await commonsHttpReadUrlAsBuffer(match.url);
         if (!data) {
-            commonsOutputError('Unable to read URL. Skipping');
+            commonsLogError('Unable to read URL. Skipping');
             continue;
         }
         const dom = cheerio.load(data);
         if (!dom) {
-            commonsOutputError('Unable to parse HTML. Skipping');
+            commonsLogError('Unable to parse HTML. Skipping');
             continue;
         }
         const nodes = dom('*')
@@ -100,5 +98,5 @@ export const QUERY = (args, databaseService, lists, _expiry, parsersConfig) => _
             }
         }
     }
-});
-//# sourceMappingURL=complex-english.query.js.map
+};
+//# sourceMappingURL=complex-english.query.mjs.map

package/dist/queries/complex-english.query.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"complex-english.query.mjs","sourceRoot":"","sources":["../../src/queries/complex-english.query.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAmB,MAAM,oBAAoB,CAAC;AAEzE,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAInD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,OAAO,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAM7E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAU3C,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAA6C,EAC9B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAwC,MAAM,CAAC,aAAa,EAAE,CAAC;IAClF,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,MAAM,GAAG,GAAuB,eAAe,CAAC,wBAAwB,EAAE,uDAAuD,MAAM,EAAE,CAAC,CAAC;IAE3I,MAAM,MAAM,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACrE;QACE,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO,CAAC,IAAI;QACpB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAClC,EACD,EAAE,CACH,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE,CAAC;QACb,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;YAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAElD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,MAAM,QAAQ,GAAkB,GAA+B,CAAC;QAEhE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,SAAS,EAAE,CAAC;YACtF,IAAI,CAAC;gBACJ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAClB,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAChC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAClC;oBAAE,SAAS;YACb,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,0BAA0B;gBAC1B,YAAY;gBACZ,SAAS;YACV,CAAC;QACF,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,cAAc,EAAE,QAAQ,CAAC,cAAc;SACxC,CAAC,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAqB,MAAM,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,eAAe,CAAC,8BAA8B,CAAC,CAAC;YAChD,SAAS;QACV,CAAC;QAED,MAAM,GAAG,GAAiC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,eAAe,CAAC,gCAAgC,CAAC,CAAC;YAClD,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAyB,GAAG,CAAC,GAAG,CAAC;aACzC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAgB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAE1F,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwC,EAAQ,EAAE;YAClF,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAa,WAAW;aAChC,IAAI,CAAC,IAAI,CAAC;aACV,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAa,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,OAAO,GAAW,IAAI,CAAC;YAE3B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAqB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAEhC,MAAM,WAAW,GAAa,aAAa,CAAC,OAAO,CAAC,CAAC;gBAErD,MAAM,KAAK,GAAa,CAAE,KAAK,OAAO,EAAE,CAAE,CAAC;gBAC3C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAErE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9D,CAAC;YAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC,CAAC"}

package/dist/queries/flash-content.query.d.mts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery.mjs';
+export declare const QUERY: TQuery;