hydra-crawler 2.8.3 → 3.0.1

package/dist/interfaces/{irequest-outcome.d.ts → irequest-outcome.d.mts}

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
 import { IncomingHttpHeaders } from 'http';
 export interface IRequestOutcome {
     latency: number;

package/dist/interfaces/irequest-outcome.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=irequest-outcome.mjs.map

package/dist/interfaces/irequest-outcome.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"irequest-outcome.mjs","sourceRoot":"","sources":["../../src/interfaces/irequest-outcome.mts"],"names":[],"mappings":""}

package/dist/interfaces/iserver.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=iserver.mjs.map

package/dist/interfaces/iserver.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"iserver.mjs","sourceRoot":"","sources":["../../src/interfaces/iserver.mts"],"names":[],"mappings":""}

package/dist/parsers/accessibility-metrics.parser.d.mts

@@ -0,0 +1,11 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { TextParser } from './text.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare class AccessibilityMetricsParser extends TextParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    protected parseText(database: DatabaseService, text: string): Promise<void>;
+}

package/dist/parsers/accessibility-metrics.parser.mjs

@@ -0,0 +1,30 @@
+import { commonsStringAutomatedReadabilityIndex, commonsStringFleschKincaidReadingEase, commonsStringSplitSentences, commonsStringSplitWords } from 'tscommons-esm-core';
+import { commonsLogDebug } from 'nodecommons-esm-log';
+import { TextParser } from './text.parser.mjs';
+export class AccessibilityMetricsParser extends TextParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'accessibilityMetrics');
+        this.url = url;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'accessibilityMetrics');
+            return;
+        }
+        const paragraphs = text.split('\n');
+        const all = paragraphs.join('. ');
+        const metrics = {
+            paragraphs: paragraphs.length,
+            sentences: commonsStringSplitSentences(all).length,
+            words: commonsStringSplitWords(all).length,
+            fkre: commonsStringFleschKincaidReadingEase(all),
+            ari: commonsStringAutomatedReadabilityIndex(all)
+        };
+        commonsLogDebug(`Accessibility metrics are: ${JSON.stringify(metrics)}`);
+        await database.setData(this.url, 'accessibilityMetrics', metrics);
+    }
+}
+//# sourceMappingURL=accessibility-metrics.parser.mjs.map

package/dist/parsers/accessibility-metrics.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"accessibility-metrics.parser.mjs","sourceRoot":"","sources":["../../src/parsers/accessibility-metrics.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sCAAsC,EAAE,qCAAqC,EAAE,2BAA2B,EAAE,uBAAuB,EAAmB,MAAM,oBAAoB,CAAC;AAE1L,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAOtD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAW/C,MAAM,OAAO,0BAA2B,SAAQ,UAAuB;IAE5D;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,sBAAsB,CACvB,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;IAStB,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEtB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,CAAC,CAAC;YAC3D,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAW,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE1C,MAAM,OAAO,GAA0B;YACrC,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,SAAS,EAAE,2BAA2B,CAAC,GAAG,CAAC,CAAC,MAAM;YAClD,KAAK,EAAE,uBAAuB,CAAC,GAAG,CAAC,CAAC,MAAM;YAC1C,IAAI,EAAE,qCAAqC,CAAC,GAAG,CAAC;YAChD,GAAG,EAAE,sCAAsC,CAAC,GAAG,CAAC;SACjD,CAAC;QAEF,eAAe,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACzE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;CACD"}

package/dist/parsers/asp-error.parser.d.mts

@@ -0,0 +1,12 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { StringParser } from './string.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare class AspErrorParser extends StringParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    supports(_contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/asp-error.parser.mjs

@@ -0,0 +1,38 @@
+import { StringParser } from './string.parser.mjs';
+const ASP_ERROR = 'Server Error in \'[^\']+\' Application[.]';
+const ASP_ERROR_PARSE = 'Server Error in \'([^\']+)\' Application[.].*<h2>\\s*<i>(.+?)</i>\\s*</h2>.+?<code><pre>(.+?)</pre></code>';
+export class AspErrorParser extends StringParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'aspError');
+        this.url = url;
+    }
+    async parse(database) {
+        if (!this.url || !this.stringData)
+            return;
+        const singlified = this.stringData.replace(/[\t\n\r]/g, ' ');
+        const pattern = new RegExp(ASP_ERROR, 'g');
+        const result = pattern.exec(singlified);
+        if (result === null) {
+            await database.unsetData(this.url, 'aspErrors');
+        }
+        else {
+            const detail = new RegExp(ASP_ERROR_PARSE, 'g');
+            const result2 = detail.exec(singlified);
+            if (result2 === null) {
+                await database.setData(this.url, 'aspErrors', true);
+            }
+            else {
+                await database.setData(this.url, 'aspErrors', {
+                    application: result2[1],
+                    message: result2[2],
+                    stack: result2[3]
+                });
+            }
+        }
+    }
+    supports(_contentType, isAllow) {
+        return isAllow; // everything allowlisted
+    }
+}
+//# sourceMappingURL=asp-error.parser.mjs.map

package/dist/parsers/asp-error.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"asp-error.parser.mjs","sourceRoot":"","sources":["../../src/parsers/asp-error.parser.mts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGnD,MAAM,SAAS,GAAW,2CAA2C,CAAC;AACtE,MAAM,eAAe,GAAW,4GAA4G,CAAC;AAE7I,MAAM,OAAO,cAAe,SAAQ,YAAyB;IAElD;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAE1C,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAErE,MAAM,OAAO,GAAW,IAAI,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9D,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACrB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,GAAW,IAAI,MAAM,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;YACxD,MAAM,OAAO,GAAyB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACtB,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE;oBAC5C,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;oBACvB,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;oBACnB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;iBAClB,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAEM,QAAQ,CAAC,YAAoB,EAAE,OAAgB;QACrD,OAAO,OAAO,CAAC,CAAC,yBAAyB;IAC1C,CAAC;CACD"}

package/dist/parsers/{bad-words.parser.d.ts → bad-words.parser.d.mts}

@@ -1,10 +1,10 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { DictionaryParser, IDictionaryConfig } from './dictionary.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { DictionaryParser, IDictionaryConfig } from './dictionary.parser.mjs';
 export declare class BadWordsParser extends DictionaryParser<IDictionaryConfig> {
     private url?;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseMatches(database: DatabaseService, matches: string[], _nonMatches: string[]): Promise<void>;
 }

package/dist/parsers/bad-words.parser.mjs

@@ -0,0 +1,17 @@
+import { DictionaryParser } from './dictionary.parser.mjs';
+export class BadWordsParser extends DictionaryParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'badWords');
+        this.url = url;
+    }
+    async parseMatches(database, matches, _nonMatches) {
+        if (!this.url)
+            return;
+        if (matches.length > 0)
+            await database.setData(this.url, 'badWords', matches);
+        else
+            await database.unsetData(this.url, 'badWords');
+    }
+}
+//# sourceMappingURL=bad-words.parser.mjs.map

package/dist/parsers/bad-words.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"bad-words.parser.mjs","sourceRoot":"","sources":["../../src/parsers/bad-words.parser.mts"],"names":[],"mappings":"AAOA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,yBAAyB,CAAC;AAE9E,MAAM,OAAO,cAAe,SAAQ,gBAAmC;IAE5D;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAES,KAAK,CAAC,YAAY,CAAC,QAAyB,EAAE,OAAiB,EAAE,WAAqB;QAC/F,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;;YACzE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACrD,CAAC;CACD"}

package/dist/parsers/complex-english.parser.d.mts

@@ -0,0 +1,15 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IDictionaryConfig } from './dictionary.parser.mjs';
+import { PhraseParser } from './phrase.parser.mjs';
+export declare class ComplexEnglishParser extends PhraseParser<IDictionaryConfig> {
+    private url?;
+    static loadKeyObject(config: IDictionaryConfig): TPropertyObject<string[]>;
+    static loadDictionary(config: IDictionaryConfig): string[];
+    private dictionary;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    getDictionary(): TPropertyObject<string[]> | undefined;
+    protected parsePhrases(database: DatabaseService, matches: string[]): Promise<void>;
+}

package/dist/parsers/complex-english.parser.mjs

@@ -0,0 +1,52 @@
+import { commonsTypeHasPropertyT, commonsTypeIsStringArrayPropertyObjectT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-esm-file';
+import { isIDictionaryConfig } from './dictionary.parser.mjs';
+import { PhraseParser } from './phrase.parser.mjs';
+let CACHE;
+export class ComplexEnglishParser extends PhraseParser {
+    url;
+    static loadKeyObject(config) {
+        const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
+        const json = commonsFileReadJsonFile(filename);
+        if (json === undefined)
+            throw new Error('Unable to read dictionary file');
+        if (!commonsTypeIsStringArrayPropertyObjectT(json))
+            throw new Error('Dictionary file is not a JSON array');
+        return json;
+    }
+    static loadDictionary(config) {
+        return Object.keys(ComplexEnglishParser.loadKeyObject(config));
+    }
+    dictionary;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'complexEnglish');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'complexEnglish', isIDictionaryConfig)) {
+            commonsLogError('Invalid config for ComplexEnglishParser');
+        }
+        const dictionaryConfig = config['complexEnglish'];
+        if (!CACHE)
+            CACHE = ComplexEnglishParser.loadKeyObject(dictionaryConfig);
+        this.dictionary = CACHE;
+        const keys = Object.keys(this.dictionary);
+        this.setPhrases(keys, dictionaryConfig.caseSensitive || false);
+    }
+    getDictionary() {
+        return this.dictionary;
+    }
+    async parsePhrases(database, matches) {
+        if (!this.url)
+            return;
+        if (matches.length > 0) {
+            commonsLogDebug(`Found ${matches.length} potentially complex phrases`);
+            await database.setData(this.url, 'complexEnglish', matches);
+        }
+        else {
+            await database.unsetData(this.url, 'complexEnglish');
+        }
+    }
+}
+//# sourceMappingURL=complex-english.parser.mjs.map

package/dist/parsers/complex-english.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"complex-english.parser.mjs","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,uCAAuC,EAAmB,MAAM,oBAAoB,CAAC;AAEvH,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAOrG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,IAAI,KAA0C,CAAC;AAE/C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;IAmB9D;IAlBH,MAAM,CAAC,aAAa,CAAC,MAAyB;QACpD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,IAAI,GAAY,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAE1E,IAAI,CAAC,uCAAuC,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAE3G,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,MAAyB;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,UAAU,CAAsC;IAExD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAJhC,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,EAAE,CAAC;YAChG,eAAe,CAAC,yCAAyC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,gBAAgB,CAAsB,CAAC;QAE1F,IAAI,CAAC,KAAK;YAAE,KAAK,GAAG,oBAAoB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB,MAAM,IAAI,GAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC;IAChE,CAAC;IAEM,aAAa;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,YAAY,CAAC,QAAyB,EAAE,OAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,eAAe,CAAC,SAAS,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAC;YACvE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;QACtD,CAAC;IACF,CAAC;CACD"}

package/dist/parsers/data.parser.d.mts

@@ -0,0 +1,13 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { Parser } from './parser.mjs';
+export interface IDataConfig extends IParserConfig {
+    maxDataBytes?: number;
+    allowExceeded?: boolean;
+}
+export declare function isIDataConfig(test: unknown): test is IDataConfig;
+export declare abstract class DataParser<T extends IDataConfig> extends Parser<T> {
+    protected data: Buffer | undefined;
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
+}

package/dist/parsers/{data.parser.js → data.parser.mjs}

@@ -1,7 +1,7 @@
-import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-es-core';
-import { commonsOutputError } from 'nodecommons-es-cli';
-import { isIParserConfig } from '../interfaces/iparser-config';
-import { Parser } from './parser';
+import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-esm-core';
+import { commonsLogError } from 'nodecommons-esm-log';
+import { isIParserConfig } from '../interfaces/iparser-config.mjs';
+import { Parser } from './parser.mjs';
 export function isIDataConfig(test) {
     if (!isIParserConfig(test))
         return false;
@@ -13,6 +13,7 @@ export function isIDataConfig(test) {
 }
 const MAX_DATA_BYTES_HARD_LIMIT = 1024 * 1024 * 1024; // 1Gb
 export class DataParser extends Parser {
+    data;
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         if (!outcome || !outcome.data)
@@ -20,17 +21,17 @@ export class DataParser extends Parser {
         const dataConfig = this.getConfig(isIDataConfig);
         const maxByteSize = (dataConfig && dataConfig.maxDataBytes) ? dataConfig.maxDataBytes : MAX_DATA_BYTES_HARD_LIMIT;
         if (outcome.data.byteLength >= maxByteSize) {
-            commonsOutputError(`Data to be parsed exceeded parser limit of ${maxByteSize}. Skipping parse.`);
+            commonsLogError(`Data to be parsed exceeded parser limit of ${maxByteSize}. Skipping parse.`);
             this.data = undefined;
             return;
         }
         if ((!dataConfig || !dataConfig.allowExceeded) && outcome.exceeded) {
             // by default don't parse data which has exceeded Hydra's upper download limit, so will be corrupt. But there is a parser option to force it to do so if desired for some reason
-            commonsOutputError('Data to be parsed exceeded Hydra\'s upper download limit and allowExceed it set. Skipping parse.');
+            commonsLogError('Data to be parsed exceeded Hydra\'s upper download limit and allowExceed it set. Skipping parse.');
             this.data = undefined;
             return;
         }
         this.data = outcome.data;
     }
 }
-//# sourceMappingURL=data.parser.js.map
+//# sourceMappingURL=data.parser.mjs.map

package/dist/parsers/data.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.parser.mjs","sourceRoot":"","sources":["../../src/parsers/data.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,uCAAuC,EAAmB,MAAM,oBAAoB,CAAC;AAExI,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGtD,OAAO,EAAiB,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAElF,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAOtC,MAAM,UAAU,aAAa,CAAC,IAAa;IAC1C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEzC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IACjF,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,yBAAyB,GAAW,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AAEpE,MAAM,OAAgB,UAAkC,SAAQ,MAAS;IAC9D,IAAI,CAAmB;IAEjC,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAEtC,MAAM,UAAU,GAA0B,IAAI,CAAC,SAAS,CAAc,aAAa,CAAC,CAAC;QACrF,MAAM,WAAW,GAAW,CAAC,UAAU,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,yBAAyB,CAAC;QAC1H,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,WAAW,EAAE,CAAC;YAC5C,eAAe,CAAC,8CAA8C,WAAW,mBAAmB,CAAC,CAAC;YAC9F,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;QACR,CAAC;QAED,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACpE,gLAAgL;YAEhL,eAAe,CAAC,kGAAkG,CAAC,CAAC;YACpH,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;QACR,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,CAAC;CACD"}

package/dist/parsers/{dictionary.parser.d.ts → dictionary.parser.d.mts}

@@ -1,8 +1,8 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { WordsParser, IWordsConfig } from './words.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { WordsParser, IWordsConfig } from './words.parser.mjs';
 export interface IDictionaryConfig extends IWordsConfig {
     dictionary: string;
     caseSensitive?: boolean;
@@ -12,7 +12,7 @@ export declare abstract class DictionaryParser<T extends IDictionaryConfig> exte
     protected caseSensitive: boolean;
     private dictionary;
     private cacheId;
-    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
     protected getDictionary(): string[];
     protected abstract parseMatches(database: DatabaseService, matches: string[], nonMatches: string[]): Promise<void>;
     protected parseWords(database: DatabaseService, words: string[]): Promise<void>;

package/dist/parsers/dictionary.parser.mjs

@@ -0,0 +1,63 @@
+import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT, commonsTypeIsStringArray } from 'tscommons-esm-core';
+import { commonsLogError } from 'nodecommons-esm-log';
+import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-esm-file';
+import { WordsParser, isIWordsConfig } from './words.parser.mjs';
+export function isIDictionaryConfig(test) {
+    if (!isIWordsConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'dictionary'))
+        return false;
+    if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'caseSensitive'))
+        return false;
+    return true;
+}
+const CACHE = new Map();
+function loadDictionary(config) {
+    const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
+    const json = commonsFileReadJsonFile(filename);
+    if (json === undefined)
+        throw new Error('Unable to read dictionary file');
+    if (!commonsTypeIsStringArray(json))
+        throw new Error('Dictionary file is not a JSON array');
+    return json;
+}
+export class DictionaryParser extends WordsParser {
+    caseSensitive = false;
+    dictionary = [];
+    cacheId;
+    constructor(outcome, config, configKey) {
+        super(outcome, config, configKey);
+        if (!config || !configKey)
+            return;
+        this.cacheId = configKey;
+        if (!commonsTypeHasPropertyT(config, configKey, isIDictionaryConfig)) {
+            commonsLogError(`Invalid config for DictionaryParser (${configKey})`);
+        }
+        const dictionaryConfig = config[configKey];
+        this.caseSensitive = dictionaryConfig.caseSensitive || false;
+        if (!CACHE.has(this.cacheId)) {
+            const loaded = loadDictionary(dictionaryConfig);
+            CACHE.set(this.cacheId, loaded);
+        }
+    }
+    getDictionary() {
+        if (!this.cacheId)
+            return [];
+        return CACHE.get(this.cacheId) || [];
+    }
+    async parseWords(database, words) {
+        this.dictionary = this.getDictionary()
+            .map((word) => this.caseSensitive ? word : word.toLowerCase());
+        const matches = [];
+        const nonMatches = [];
+        for (const word of words) {
+            const compare = this.caseSensitive ? word : word.toLowerCase();
+            if (this.dictionary.includes(compare))
+                matches.push(word);
+            else
+                nonMatches.push(word);
+        }
+        await this.parseMatches(database, matches, nonMatches);
+    }
+}
+//# sourceMappingURL=dictionary.parser.mjs.map

package/dist/parsers/dictionary.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionary.parser.mjs","sourceRoot":"","sources":["../../src/parsers/dictionary.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,4BAA4B,EAAE,uBAAuB,EAAE,wBAAwB,EAAmB,MAAM,oBAAoB,CAAC;AAEhL,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAOrG,OAAO,EAAE,WAAW,EAAgB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAM/E,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAExC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,YAAY,CAAC;QAAE,OAAO,KAAK,CAAC;IACpE,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,KAAK,GAA0B,IAAI,GAAG,EAAoB,CAAC;AAEjE,SAAS,cAAc,CAAC,MAAyB;IAChD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAY,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE1E,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,WAAc;IAC/E,aAAa,GAAY,KAAK,CAAC;IACjC,UAAU,GAAa,EAAE,CAAC;IAC1B,OAAO,CAAmB;IAElC,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS;YAAE,OAAO;QAElC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAEzB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,SAAS,EAAE,mBAAmB,CAAC,EAAE,CAAC;YACzF,eAAe,CAAC,wCAAwC,SAAS,GAAG,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,SAAS,CAAsB,CAAC;QAEnF,IAAI,CAAC,aAAa,GAAG,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC;QAE7D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAa,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC1D,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAES,aAAa;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAE7B,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;IAIS,KAAK,CAAC,UAAU,CAAC,QAAyB,EAAE,KAAe;QACpE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE;aACnC,GAAG,CAAC,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAEjF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAW,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACvE,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBACrD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC;CACD"}

package/dist/parsers/html.parser.d.mts

@@ -0,0 +1,13 @@
+import * as cheerio from 'cheerio';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { StringParser } from './string.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare abstract class HtmlParser<T extends IDataConfig> extends StringParser<T> {
+    static collapseWhitespace(raw: string): string;
+    static loadDom(html: string): cheerio.CheerioAPI | undefined;
+    protected dom: cheerio.CheerioAPI | undefined;
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
+    supports(contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/{html.parser.js → html.parser.mjs}

@@ -1,5 +1,5 @@
 import * as cheerio from 'cheerio';
-import { StringParser } from './string.parser';
+import { StringParser } from './string.parser.mjs';
 export class HtmlParser extends StringParser {
     static collapseWhitespace(raw) {
         return raw.replace(/\s+/g, ' ');
@@ -8,11 +8,12 @@ export class HtmlParser extends StringParser {
         try {
             return cheerio.load(html);
         }
-        catch (ex) {
+        catch (_ex) {
             // ignore
             return undefined;
         }
     }
+    dom;
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         if (this.stringData && this.stringData.length) {
@@ -26,4 +27,4 @@ export class HtmlParser extends StringParser {
         return /^text\/html([^a-z0-9].*)?$/.test(contentType);
     }
 }
-//# sourceMappingURL=html.parser.js.map
+//# sourceMappingURL=html.parser.mjs.map

package/dist/parsers/html.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.parser.mjs","sourceRoot":"","sources":["../../src/parsers/html.parser.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGnD,MAAM,OAAgB,UAAkC,SAAQ,YAAe;IACvE,MAAM,CAAC,kBAAkB,CAAC,GAAW;QAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,IAAY;QACjC,IAAI,CAAC;YACJ,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,SAAS;YACT,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;IAES,GAAG,CAA+B;IAE5C,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAW,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;IACF,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,4BAA4B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;CACD"}

package/dist/parsers/hyperlinks.parser.d.mts

@@ -0,0 +1,20 @@
+import * as cheerio from 'cheerio';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { HtmlParser } from './html.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export interface IHyperlinksConfig extends IDataConfig {
+    sources: string[];
+}
+export declare function isIHyperlinksConfig(test: unknown): test is IHyperlinksConfig;
+export declare class HyperlinksParser extends HtmlParser<IHyperlinksConfig> {
+    private url?;
+    static parseTagLinks(dom: cheerio.CheerioAPI, tag: string, attribute: string): string[];
+    static getBaseHref(dom: cheerio.CheerioAPI, url: string): string | undefined;
+    private hyperlinksConfig;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    links(): Promise<string[]>;
+}

package/dist/parsers/hyperlinks.parser.mjs

@@ -0,0 +1,104 @@
+import { URL } from 'url';
+import { commonsArrayUnique, commonsTypeHasPropertyStringArray } from 'tscommons-esm-core';
+import { HtmlParser } from './html.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIHyperlinksConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'sources'))
+        return false;
+    return true;
+}
+export class HyperlinksParser extends HtmlParser {
+    url;
+    static parseTagLinks(dom, tag, attribute) {
+        const links = [];
+        dom(`${tag}[${attribute}]`).each((_index, element) => {
+            const link = dom(element).attr(attribute);
+            if (link !== undefined)
+                links.push(link);
+            return true;
+        });
+        return commonsArrayUnique(links);
+    }
+    static getBaseHref(dom, url) {
+        try {
+            const base = dom('base[href]');
+            if (base.length === 0)
+                return undefined;
+            const href = dom(base[0]).attr('href');
+            if (!href)
+                return undefined;
+            const whatwg = new URL(href, url);
+            return whatwg.toString();
+        }
+        catch (_ex) { /* do nothing */ }
+        return undefined;
+    }
+    hyperlinksConfig;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'hyperlinks');
+        this.url = url;
+        this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
+    }
+    async parse(database) {
+        if (!this.dom || !this.url || !this.hyperlinksConfig)
+            return; // invalid parse
+        let title;
+        try {
+            title = this.dom('title').text();
+        }
+        catch (_ex) { /* do nothing */ }
+        if (title !== undefined && title !== null && title !== '') {
+            await database.setData(this.url, 'title', title);
+        }
+        const links = {};
+        for (const source of this.hyperlinksConfig.sources) {
+            const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
+            if (pattern)
+                links[pattern[1]] = HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2]);
+        }
+        await database.setData(this.url, 'links', links);
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async links() {
+        if (!this.dom || !this.url || !this.hyperlinksConfig)
+            return []; // invalid parse
+        let links = [];
+        for (const source of this.hyperlinksConfig.sources) {
+            const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
+            if (pattern) {
+                if (pattern[1] === 'form')
+                    continue;
+                links = [...links, ...HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2])];
+            }
+        }
+        const unique = [...new Set(links)];
+        let baseHref = HyperlinksParser.getBaseHref(this.dom, this.url);
+        if (baseHref === undefined)
+            baseHref = this.url;
+        const results = [];
+        for (let l of unique) {
+            const anchor = l.split('#');
+            l = anchor[0];
+            if (l === '')
+                continue;
+            try {
+                const u = new URL(l, baseHref);
+                if (u.toString().indexOf('#') !== -1) {
+                    process.exit(1);
+                }
+                if (!u.protocol.match(/^http(s?):$/))
+                    continue;
+                if (u.toString() === this.url || u.toString() === baseHref) {
+                    // skip self-links, or presumed self-links
+                    continue;
+                }
+                results.push(u.toString());
+            }
+            catch (_ex) { /* do nothing */ }
+        }
+        return results;
+    }
+}
+//# sourceMappingURL=hyperlinks.parser.mjs.map

package/dist/parsers/hyperlinks.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"hyperlinks.parser.mjs","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAI1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAmB,MAAM,oBAAoB,CAAC;AAO5G,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAK/D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;IAmCxD;IAjCH,MAAM,CAAC,aAAa,CACzB,GAAuB,EACvB,GAAW,EACX,SAAiB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAY,EAAW,EAAE;YAC1E,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5D,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,GAAuB,EAAE,GAAW;QAC7D,IAAI,CAAC;YACJ,MAAM,IAAI,GAAyB,GAAG,CAAC,YAAY,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAExC,MAAM,IAAI,GAAqB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAE5B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAClC,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,gBAAgB,CAA8B;IAEtD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAJ5B,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAC7D,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,CAAC,gBAAgB;QAE9E,IAAI,KAAuB,CAAC;QAC5B,IAAI,CAAC;YACJ,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAElC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,KAAK,GAAkC,EAAE,CAAC;QAChD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YACpD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACvF,IAAI,OAAO;gBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACnG,CAAC;QAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED,4DAA4D;IAC5C,KAAK,CAAC,KAAK;QAC1B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,EAAE,CAAC,CAAC,gBAAgB;QAEjF,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YACpD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACvF,IAAI,OAAO,EAAE,CAAC;gBACb,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;oBAAE,SAAS;gBACpC,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAa,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAE7C,IAAI,QAAQ,GAAqB,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAClF,IAAI,QAAQ,KAAK,SAAS;YAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;QAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;YACtB,MAAM,MAAM,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACtC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,KAAK,EAAE;gBAAE,SAAS;YAEvB,IAAI,CAAC;gBACJ,MAAM,CAAC,GAAQ,IAAI,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjB,CAAC;gBACD,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;oBAAE,SAAS;gBAC/C,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,QAAQ,EAAE,CAAC;oBAC5D,0CAA0C;oBAC1C,SAAS;gBACV,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,OAAO,CAAC;IAChB,CAAC;CACD"}

package/dist/parsers/image-tags.parser.d.mts

@@ -0,0 +1,19 @@
+import * as cheerio from 'cheerio';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { HtmlParser } from './html.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+type TImageTag = {
+    src: string;
+    alt?: string;
+    title?: string;
+};
+export declare class ImageTagsParser extends HtmlParser<IDataConfig> {
+    private url?;
+    static parseImageTags(dom: cheerio.CheerioAPI): TImageTag[];
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+}
+export {};