hydra-crawler 2.8.4 → 3.0.2

package/dist/enums/{ex-powered-by.js → ex-powered-by.mjs}

@@ -5,4 +5,4 @@ export var EXPoweredBy;
     EXPoweredBy["EXPRESS"] = "express";
     EXPoweredBy["ZOPE"] = "zope";
 })(EXPoweredBy || (EXPoweredBy = {}));
-//# sourceMappingURL=ex-powered-by.js.map
+//# sourceMappingURL=ex-powered-by.mjs.map

package/dist/enums/ex-powered-by.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ex-powered-by.mjs","sourceRoot":"","sources":["../../src/enums/ex-powered-by.mts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,WAKX;AALD,WAAY,WAAW;IACrB,0BAAW,CAAA;IACX,iCAAkB,CAAA;IAClB,kCAAmB,CAAA;IACnB,4BAAa,CAAA;AACf,CAAC,EALW,WAAW,KAAX,WAAW,QAKtB"}

package/dist/helpers/{matcher.d.ts → matcher.d.mts}

@@ -1,4 +1,4 @@
-import { IMatch } from '../interfaces/imatch';
+import { IMatch } from '../interfaces/imatch.mjs';
 export declare abstract class Matcher {
     static matches(url: URL, test: IMatch): boolean;
     static isAnyMatch(url: string, list: IMatch[]): boolean;

package/dist/helpers/{matcher.js → matcher.mjs}

@@ -16,4 +16,4 @@ export class Matcher {
         return false;
     }
 }
-//# sourceMappingURL=matcher.js.map
+//# sourceMappingURL=matcher.mjs.map

package/dist/helpers/matcher.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"matcher.mjs","sourceRoot":"","sources":["../../src/helpers/matcher.mts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,OAAO;IACrB,MAAM,CAAC,OAAO,CAAC,GAAQ,EAAE,IAAY;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACrC,MAAM,KAAK,GAAW,IAAI,MAAM,CAAE,IAAgC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAE,GAA0C,CAAC,GAAG,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;QACjF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,UAAU,CAAC,GAAW,EAAE,IAAc;QACnD,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,WAAW,IAAI,IAAI,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC;gBAAE,OAAO,IAAI,CAAC;QACvD,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}

package/dist/helpers/{random.d.ts → random.d.mts}

@@ -1,4 +1,4 @@
-import { TCrawlConfig } from '../types/tcrawl-config';
+import { TCrawlConfig } from '../types/tcrawl-config.mjs';
 export declare abstract class Random {
     static randomise(value: number, crawlConfig: TCrawlConfig): number;
 }

package/dist/helpers/{random.js → random.mjs}

@@ -5,4 +5,4 @@ export class Random {
         return value * (1 + ratio);
     }
 }
-//# sourceMappingURL=random.js.map
+//# sourceMappingURL=random.mjs.map

package/dist/helpers/random.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"random.mjs","sourceRoot":"","sources":["../../src/helpers/random.mts"],"names":[],"mappings":"AAEA,MAAM,OAAgB,MAAM;IACpB,MAAM,CAAC,SAAS,CAAC,KAAa,EAAE,WAAyB;QAC/D,MAAM,KAAK,GAAW,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,KAAK,GAAW,WAAW,CAAC,aAAa,GAAG,KAAK,CAAC;QAExD,OAAO,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;IAC5B,CAAC;CACD"}

package/dist/helpers/{utf-decoder.d.ts → utf-decoder.d.mts}

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 export declare abstract class UtfDecoder {
     static fromBuffer(incoming: Buffer): string | undefined;
 }

package/dist/helpers/{utf-decoder.js → utf-decoder.mjs}

@@ -18,16 +18,16 @@ export class UtfDecoder {
                     return incoming.toString();
             }
         }
-        catch (ex) {
+        catch (_ex) {
             // might be a chardet failure, so check that first
             try {
                 return incoming.toString();
             }
-            catch (ex2) {
+            catch (_ex2) {
                 // ignore
             }
         }
         return undefined;
     }
 }
-//# sourceMappingURL=utf-decoder.js.map
+//# sourceMappingURL=utf-decoder.mjs.map

package/dist/helpers/utf-decoder.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"utf-decoder.mjs","sourceRoot":"","sources":["../../src/helpers/utf-decoder.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,OAAgB,UAAU;IACxB,MAAM,CAAC,UAAU,CAAC,QAAgB;QACxC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAE7C,IAAI,CAAC;YACJ,MAAM,MAAM,GAAW,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAE5B,MAAM,KAAK,GAAW,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACjD,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;YACnC,QAAQ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,KAAK,UAAU;oBACd,OAAO,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACrC,KAAK,UAAU;oBACd,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAC3D;oBACC,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC7B,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,kDAAkD;YAClD,IAAI,CAAC;gBACJ,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC;YAAC,OAAO,IAAI,EAAE,CAAC;gBACf,SAAS;YACV,CAAC;QACF,CAAC;QAED,OAAO,SAAS,CAAC;IAClB,CAAC;CACD"}

package/dist/interfaces/{iexpiry.d.ts → iexpiry.d.mts}

@@ -1,4 +1,4 @@
-import { IMatch } from './imatch';
+import { IMatch } from './imatch.mjs';
 export interface IExpiry {
     match: IMatch;
     expiry: number;

package/dist/interfaces/{iexpiry.js → iexpiry.mjs}

@@ -1,5 +1,5 @@
-import { commonsTypeHasPropertyNumber, commonsTypeHasPropertyObject, commonsTypeHasPropertyString } from 'tscommons-es-core';
-import { isIMatch } from './imatch';
+import { commonsTypeHasPropertyNumber, commonsTypeHasPropertyObject, commonsTypeHasPropertyString } from 'tscommons-esm-core';
+import { isIMatch } from './imatch.mjs';
 export function isIExpiry(test) {
     // Can't use hasPropertyT as we need to pass a true parameter
     if (!commonsTypeHasPropertyObject(test, 'match'))
@@ -62,4 +62,4 @@ export function toIExpiry(data) {
         throw new Error('Object is not an instance of IExpiry');
     return data;
 }
-//# sourceMappingURL=iexpiry.js.map
+//# sourceMappingURL=iexpiry.mjs.map

package/dist/interfaces/iexpiry.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"iexpiry.mjs","sourceRoot":"","sources":["../../src/interfaces/iexpiry.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,4BAA4B,EAAE,4BAA4B,EAAe,MAAM,oBAAoB,CAAC;AAE3I,OAAO,EAAU,QAAQ,EAAE,MAAM,cAAc,CAAC;AAOhD,MAAM,UAAU,SAAS,CAAC,IAAa;IACtC,6DAA6D;IAC7D,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,IAAK,SAQJ;AARD,WAAK,SAAS;IACZ,8BAAiB,CAAA;IACjB,8BAAiB,CAAA;IACjB,0BAAa,CAAA;IACb,wBAAW,CAAA;IACX,0BAAa,CAAA;IACb,4BAAe,CAAA;IACf,0BAAa,CAAA;AACf,CAAC,EARI,SAAS,KAAT,SAAS,QAQb;AAED,MAAM,eAAe,GAAW,CAAC,CAAC;AAClC,MAAM,eAAe,GAAW,eAAe,GAAG,EAAE,CAAC;AACrD,MAAM,aAAa,GAAW,eAAe,GAAG,EAAE,CAAC;AACnD,MAAM,YAAY,GAAW,aAAa,GAAG,EAAE,CAAC;AAChD,MAAM,aAAa,GAAW,YAAY,GAAG,CAAC,CAAC;AAC/C,MAAM,aAAa,GAAW,YAAY,GAAG,MAAM,CAAC;AACpD,MAAM,cAAc,GAAW,aAAa,GAAG,EAAE,CAAC;AAElD,MAAM,cAAc,GAAmC;IACrD,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,eAAe;IACnC,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,eAAe;IACnC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;IAC/B,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,YAAY;IAC7B,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;IAC/B,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,cAAc;IACjC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;CAChC,CAAC;AAEF,SAAS,eAAe,CAAC,MAAc;IACtC,IAAI,KAAK,GAAW,CAAC,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QAChD,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,4BAA4B,IAAI,mBAAmB,EAAE,GAAG,CAAC,CAAC;YAC3F,MAAM,MAAM,GAAyB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM;YAE3B,IAAI,KAAK,GAAW,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,KAAK,IAAI,cAAc,CAAC,IAAiB,CAAC,CAAC;YAC3C,KAAK,IAAI,KAAK,CAAC;YAEf,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;IACF,CAAC;IAED,IAAI,EAAE,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAEhF,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAa;IACtC,IAAI,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAW,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAE9E,OAAO,IAAI,CAAC;AACb,CAAC"}

package/dist/interfaces/{imatch.js → imatch.mjs}

@@ -1,4 +1,4 @@
-import { commonsTypeHasPropertyStringOrUndefined } from 'tscommons-es-core';
+import { commonsTypeHasPropertyStringOrUndefined } from 'tscommons-esm-core';
 export function isIMatch(test, allowBlank = false) {
     if (!commonsTypeHasPropertyStringOrUndefined(test, 'hostname'))
         return false;
@@ -10,4 +10,4 @@ export function isIMatch(test, allowBlank = false) {
         return false;
     return true;
 }
-//# sourceMappingURL=imatch.js.map
+//# sourceMappingURL=imatch.mjs.map

package/dist/interfaces/imatch.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"imatch.mjs","sourceRoot":"","sources":["../../src/interfaces/imatch.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uCAAuC,EAAE,MAAM,oBAAoB,CAAC;AAQ7E,MAAM,UAAU,QAAQ,CAAC,IAAa,EAAE,aAAsB,KAAK;IAClE,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3E,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAElF,OAAO,IAAI,CAAC;AACb,CAAC"}

package/dist/interfaces/{iparser-config.js → iparser-config.mjs}

@@ -1,7 +1,7 @@
-import { commonsTypeHasPropertyBoolean } from 'tscommons-es-core';
+import { commonsTypeHasPropertyBoolean } from 'tscommons-esm-core';
 export function isIParserConfig(test) {
     if (!commonsTypeHasPropertyBoolean(test, 'enabled'))
         return false;
     return true;
 }
-//# sourceMappingURL=iparser-config.js.map
+//# sourceMappingURL=iparser-config.mjs.map

package/dist/interfaces/iparser-config.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"iparser-config.mjs","sourceRoot":"","sources":["../../src/interfaces/iparser-config.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,oBAAoB,CAAC;AAQnE,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,6BAA6B,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAElE,OAAO,IAAI,CAAC;AACb,CAAC"}

package/dist/interfaces/{iparser.d.ts → iparser.d.mts}

@@ -1,4 +1,4 @@
-import { DatabaseService } from '../services/database.service';
+import { DatabaseService } from '../services/database.service.mjs';
 export interface IParser {
     isEnabled(): boolean;
     supports(contentType: string, isAllow: boolean): boolean;

package/dist/interfaces/iparser.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=iparser.mjs.map

package/dist/interfaces/iparser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"iparser.mjs","sourceRoot":"","sources":["../../src/interfaces/iparser.mts"],"names":[],"mappings":""}

package/dist/interfaces/{irequest-outcome.d.ts → irequest-outcome.d.mts}

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
 import { IncomingHttpHeaders } from 'http';
 export interface IRequestOutcome {
     latency: number;

package/dist/interfaces/irequest-outcome.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=irequest-outcome.mjs.map

package/dist/interfaces/irequest-outcome.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"irequest-outcome.mjs","sourceRoot":"","sources":["../../src/interfaces/irequest-outcome.mts"],"names":[],"mappings":""}

package/dist/interfaces/iserver.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=iserver.mjs.map

package/dist/interfaces/iserver.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"iserver.mjs","sourceRoot":"","sources":["../../src/interfaces/iserver.mts"],"names":[],"mappings":""}

package/dist/parsers/accessibility-metrics.parser.d.mts

@@ -0,0 +1,11 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { TextParser } from './text.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare class AccessibilityMetricsParser extends TextParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    protected parseText(database: DatabaseService, text: string): Promise<void>;
+}

package/dist/parsers/accessibility-metrics.parser.mjs

@@ -0,0 +1,30 @@
+import { commonsStringAutomatedReadabilityIndex, commonsStringFleschKincaidReadingEase, commonsStringSplitSentences, commonsStringSplitWords } from 'tscommons-esm-core';
+import { commonsLogDebug } from 'nodecommons-esm-log';
+import { TextParser } from './text.parser.mjs';
+export class AccessibilityMetricsParser extends TextParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'accessibilityMetrics');
+        this.url = url;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'accessibilityMetrics');
+            return;
+        }
+        const paragraphs = text.split('\n');
+        const all = paragraphs.join('. ');
+        const metrics = {
+            paragraphs: paragraphs.length,
+            sentences: commonsStringSplitSentences(all).length,
+            words: commonsStringSplitWords(all).length,
+            fkre: commonsStringFleschKincaidReadingEase(all),
+            ari: commonsStringAutomatedReadabilityIndex(all)
+        };
+        commonsLogDebug(`Accessibility metrics are: ${JSON.stringify(metrics)}`);
+        await database.setData(this.url, 'accessibilityMetrics', metrics);
+    }
+}
+//# sourceMappingURL=accessibility-metrics.parser.mjs.map

package/dist/parsers/accessibility-metrics.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"accessibility-metrics.parser.mjs","sourceRoot":"","sources":["../../src/parsers/accessibility-metrics.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sCAAsC,EAAE,qCAAqC,EAAE,2BAA2B,EAAE,uBAAuB,EAAmB,MAAM,oBAAoB,CAAC;AAE1L,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAOtD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAW/C,MAAM,OAAO,0BAA2B,SAAQ,UAAuB;IAE5D;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,sBAAsB,CACvB,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;IAStB,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEtB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,CAAC,CAAC;YAC3D,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAW,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE1C,MAAM,OAAO,GAA0B;YACrC,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,SAAS,EAAE,2BAA2B,CAAC,GAAG,CAAC,CAAC,MAAM;YAClD,KAAK,EAAE,uBAAuB,CAAC,GAAG,CAAC,CAAC,MAAM;YAC1C,IAAI,EAAE,qCAAqC,CAAC,GAAG,CAAC;YAChD,GAAG,EAAE,sCAAsC,CAAC,GAAG,CAAC;SACjD,CAAC;QAEF,eAAe,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACzE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;CACD"}

package/dist/parsers/asp-error.parser.d.mts

@@ -0,0 +1,12 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { StringParser } from './string.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare class AspErrorParser extends StringParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    supports(_contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/asp-error.parser.mjs

@@ -0,0 +1,38 @@
+import { StringParser } from './string.parser.mjs';
+const ASP_ERROR = 'Server Error in \'[^\']+\' Application[.]';
+const ASP_ERROR_PARSE = 'Server Error in \'([^\']+)\' Application[.].*<h2>\\s*<i>(.+?)</i>\\s*</h2>.+?<code><pre>(.+?)</pre></code>';
+export class AspErrorParser extends StringParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'aspError');
+        this.url = url;
+    }
+    async parse(database) {
+        if (!this.url || !this.stringData)
+            return;
+        const singlified = this.stringData.replace(/[\t\n\r]/g, ' ');
+        const pattern = new RegExp(ASP_ERROR, 'g');
+        const result = pattern.exec(singlified);
+        if (result === null) {
+            await database.unsetData(this.url, 'aspErrors');
+        }
+        else {
+            const detail = new RegExp(ASP_ERROR_PARSE, 'g');
+            const result2 = detail.exec(singlified);
+            if (result2 === null) {
+                await database.setData(this.url, 'aspErrors', true);
+            }
+            else {
+                await database.setData(this.url, 'aspErrors', {
+                    application: result2[1],
+                    message: result2[2],
+                    stack: result2[3]
+                });
+            }
+        }
+    }
+    supports(_contentType, isAllow) {
+        return isAllow; // everything allowlisted
+    }
+}
+//# sourceMappingURL=asp-error.parser.mjs.map

package/dist/parsers/asp-error.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"asp-error.parser.mjs","sourceRoot":"","sources":["../../src/parsers/asp-error.parser.mts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGnD,MAAM,SAAS,GAAW,2CAA2C,CAAC;AACtE,MAAM,eAAe,GAAW,4GAA4G,CAAC;AAE7I,MAAM,OAAO,cAAe,SAAQ,YAAyB;IAElD;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAE1C,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAErE,MAAM,OAAO,GAAW,IAAI,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9D,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACrB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACP,MAAM,MAAM,GAAW,IAAI,MAAM,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;YACxD,MAAM,OAAO,GAAyB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACtB,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE;oBAC5C,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;oBACvB,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;oBACnB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;iBAClB,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAEM,QAAQ,CAAC,YAAoB,EAAE,OAAgB;QACrD,OAAO,OAAO,CAAC,CAAC,yBAAyB;IAC1C,CAAC;CACD"}

package/dist/parsers/{bad-words.parser.d.ts → bad-words.parser.d.mts}

@@ -1,10 +1,10 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { DictionaryParser, IDictionaryConfig } from './dictionary.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { DictionaryParser, IDictionaryConfig } from './dictionary.parser.mjs';
 export declare class BadWordsParser extends DictionaryParser<IDictionaryConfig> {
     private url?;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseMatches(database: DatabaseService, matches: string[], _nonMatches: string[]): Promise<void>;
 }

package/dist/parsers/bad-words.parser.mjs

@@ -0,0 +1,17 @@
+import { DictionaryParser } from './dictionary.parser.mjs';
+export class BadWordsParser extends DictionaryParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'badWords');
+        this.url = url;
+    }
+    async parseMatches(database, matches, _nonMatches) {
+        if (!this.url)
+            return;
+        if (matches.length > 0)
+            await database.setData(this.url, 'badWords', matches);
+        else
+            await database.unsetData(this.url, 'badWords');
+    }
+}
+//# sourceMappingURL=bad-words.parser.mjs.map

package/dist/parsers/bad-words.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"bad-words.parser.mjs","sourceRoot":"","sources":["../../src/parsers/bad-words.parser.mts"],"names":[],"mappings":"AAOA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,yBAAyB,CAAC;AAE9E,MAAM,OAAO,cAAe,SAAQ,gBAAmC;IAE5D;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAES,KAAK,CAAC,YAAY,CAAC,QAAyB,EAAE,OAAiB,EAAE,WAAqB;QAC/F,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;;YACzE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACrD,CAAC;CACD"}

package/dist/parsers/complex-english.parser.d.mts

@@ -0,0 +1,15 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IDictionaryConfig } from './dictionary.parser.mjs';
+import { PhraseParser } from './phrase.parser.mjs';
+export declare class ComplexEnglishParser extends PhraseParser<IDictionaryConfig> {
+    private url?;
+    static loadKeyObject(config: IDictionaryConfig): TPropertyObject<string[]>;
+    static loadDictionary(config: IDictionaryConfig): string[];
+    private dictionary;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    getDictionary(): TPropertyObject<string[]> | undefined;
+    protected parsePhrases(database: DatabaseService, matches: string[]): Promise<void>;
+}

package/dist/parsers/complex-english.parser.mjs

@@ -0,0 +1,52 @@
+import { commonsTypeHasPropertyT, commonsTypeIsStringArrayPropertyObjectT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-esm-file';
+import { isIDictionaryConfig } from './dictionary.parser.mjs';
+import { PhraseParser } from './phrase.parser.mjs';
+let CACHE;
+export class ComplexEnglishParser extends PhraseParser {
+    url;
+    static loadKeyObject(config) {
+        const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
+        const json = commonsFileReadJsonFile(filename);
+        if (json === undefined)
+            throw new Error('Unable to read dictionary file');
+        if (!commonsTypeIsStringArrayPropertyObjectT(json))
+            throw new Error('Dictionary file is not a JSON array');
+        return json;
+    }
+    static loadDictionary(config) {
+        return Object.keys(ComplexEnglishParser.loadKeyObject(config));
+    }
+    dictionary;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'complexEnglish');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'complexEnglish', isIDictionaryConfig)) {
+            commonsLogError('Invalid config for ComplexEnglishParser');
+        }
+        const dictionaryConfig = config['complexEnglish'];
+        if (!CACHE)
+            CACHE = ComplexEnglishParser.loadKeyObject(dictionaryConfig);
+        this.dictionary = CACHE;
+        const keys = Object.keys(this.dictionary);
+        this.setPhrases(keys, dictionaryConfig.caseSensitive || false);
+    }
+    getDictionary() {
+        return this.dictionary;
+    }
+    async parsePhrases(database, matches) {
+        if (!this.url)
+            return;
+        if (matches.length > 0) {
+            commonsLogDebug(`Found ${matches.length} potentially complex phrases`);
+            await database.setData(this.url, 'complexEnglish', matches);
+        }
+        else {
+            await database.unsetData(this.url, 'complexEnglish');
+        }
+    }
+}
+//# sourceMappingURL=complex-english.parser.mjs.map

package/dist/parsers/complex-english.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"complex-english.parser.mjs","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,uCAAuC,EAAmB,MAAM,oBAAoB,CAAC;AAEvH,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAOrG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,IAAI,KAA0C,CAAC;AAE/C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;IAmB9D;IAlBH,MAAM,CAAC,aAAa,CAAC,MAAyB;QACpD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,IAAI,GAAY,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAE1E,IAAI,CAAC,uCAAuC,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAE3G,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,MAAyB;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,UAAU,CAAsC;IAExD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAJhC,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,EAAE,CAAC;YAChG,eAAe,CAAC,yCAAyC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,gBAAgB,CAAsB,CAAC;QAE1F,IAAI,CAAC,KAAK;YAAE,KAAK,GAAG,oBAAoB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB,MAAM,IAAI,GAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC;IAChE,CAAC;IAEM,aAAa;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,YAAY,CAAC,QAAyB,EAAE,OAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,eAAe,CAAC,SAAS,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAC;YACvE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;QACtD,CAAC;IACF,CAAC;CACD"}

package/dist/parsers/data.parser.d.mts

@@ -0,0 +1,13 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { Parser } from './parser.mjs';
+export interface IDataConfig extends IParserConfig {
+    maxDataBytes?: number;
+    allowExceeded?: boolean;
+}
+export declare function isIDataConfig(test: unknown): test is IDataConfig;
+export declare abstract class DataParser<T extends IDataConfig> extends Parser<T> {
+    protected data: Buffer | undefined;
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
+}

package/dist/parsers/{data.parser.js → data.parser.mjs}

@@ -1,7 +1,7 @@
-import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-es-core';
-import { commonsOutputError } from 'nodecommons-es-cli';
-import { isIParserConfig } from '../interfaces/iparser-config';
-import { Parser } from './parser';
+import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-esm-core';
+import { commonsLogError } from 'nodecommons-esm-log';
+import { isIParserConfig } from '../interfaces/iparser-config.mjs';
+import { Parser } from './parser.mjs';
 export function isIDataConfig(test) {
     if (!isIParserConfig(test))
         return false;
@@ -13,6 +13,7 @@ export function isIDataConfig(test) {
 }
 const MAX_DATA_BYTES_HARD_LIMIT = 1024 * 1024 * 1024; // 1Gb
 export class DataParser extends Parser {
+    data;
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         if (!outcome || !outcome.data)
@@ -20,17 +21,17 @@ export class DataParser extends Parser {
         const dataConfig = this.getConfig(isIDataConfig);
         const maxByteSize = (dataConfig && dataConfig.maxDataBytes) ? dataConfig.maxDataBytes : MAX_DATA_BYTES_HARD_LIMIT;
         if (outcome.data.byteLength >= maxByteSize) {
-            commonsOutputError(`Data to be parsed exceeded parser limit of ${maxByteSize}. Skipping parse.`);
+            commonsLogError(`Data to be parsed exceeded parser limit of ${maxByteSize}. Skipping parse.`);
             this.data = undefined;
             return;
         }
         if ((!dataConfig || !dataConfig.allowExceeded) && outcome.exceeded) {
             // by default don't parse data which has exceeded Hydra's upper download limit, so will be corrupt. But there is a parser option to force it to do so if desired for some reason
-            commonsOutputError('Data to be parsed exceeded Hydra\'s upper download limit and allowExceed it set. Skipping parse.');
+            commonsLogError('Data to be parsed exceeded Hydra\'s upper download limit and allowExceed it set. Skipping parse.');
             this.data = undefined;
             return;
         }
         this.data = outcome.data;
     }
 }
-//# sourceMappingURL=data.parser.js.map
+//# sourceMappingURL=data.parser.mjs.map

package/dist/parsers/data.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"data.parser.mjs","sourceRoot":"","sources":["../../src/parsers/data.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,uCAAuC,EAAmB,MAAM,oBAAoB,CAAC;AAExI,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGtD,OAAO,EAAiB,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAElF,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAOtC,MAAM,UAAU,aAAa,CAAC,IAAa;IAC1C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEzC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IACjF,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,yBAAyB,GAAW,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AAEpE,MAAM,OAAgB,UAAkC,SAAQ,MAAS;IAC9D,IAAI,CAAmB;IAEjC,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAEtC,MAAM,UAAU,GAA0B,IAAI,CAAC,SAAS,CAAc,aAAa,CAAC,CAAC;QACrF,MAAM,WAAW,GAAW,CAAC,UAAU,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,yBAAyB,CAAC;QAC1H,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,WAAW,EAAE,CAAC;YAC5C,eAAe,CAAC,8CAA8C,WAAW,mBAAmB,CAAC,CAAC;YAC9F,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;QACR,CAAC;QAED,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACpE,gLAAgL;YAEhL,eAAe,CAAC,kGAAkG,CAAC,CAAC;YACpH,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;QACR,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,CAAC;CACD"}

package/dist/parsers/{dictionary.parser.d.ts → dictionary.parser.d.mts}

@@ -1,8 +1,8 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { WordsParser, IWordsConfig } from './words.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { WordsParser, IWordsConfig } from './words.parser.mjs';
 export interface IDictionaryConfig extends IWordsConfig {
     dictionary: string;
     caseSensitive?: boolean;
@@ -12,7 +12,7 @@ export declare abstract class DictionaryParser<T extends IDictionaryConfig> exte
     protected caseSensitive: boolean;
     private dictionary;
     private cacheId;
-    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
     protected getDictionary(): string[];
     protected abstract parseMatches(database: DatabaseService, matches: string[], nonMatches: string[]): Promise<void>;
     protected parseWords(database: DatabaseService, words: string[]): Promise<void>;

package/dist/parsers/dictionary.parser.mjs

@@ -0,0 +1,63 @@
+import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT, commonsTypeIsStringArray } from 'tscommons-esm-core';
+import { commonsLogError } from 'nodecommons-esm-log';
+import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-esm-file';
+import { WordsParser, isIWordsConfig } from './words.parser.mjs';
+export function isIDictionaryConfig(test) {
+    if (!isIWordsConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'dictionary'))
+        return false;
+    if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'caseSensitive'))
+        return false;
+    return true;
+}
+const CACHE = new Map();
+function loadDictionary(config) {
+    const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
+    const json = commonsFileReadJsonFile(filename);
+    if (json === undefined)
+        throw new Error('Unable to read dictionary file');
+    if (!commonsTypeIsStringArray(json))
+        throw new Error('Dictionary file is not a JSON array');
+    return json;
+}
+export class DictionaryParser extends WordsParser {
+    caseSensitive = false;
+    dictionary = [];
+    cacheId;
+    constructor(outcome, config, configKey) {
+        super(outcome, config, configKey);
+        if (!config || !configKey)
+            return;
+        this.cacheId = configKey;
+        if (!commonsTypeHasPropertyT(config, configKey, isIDictionaryConfig)) {
+            commonsLogError(`Invalid config for DictionaryParser (${configKey})`);
+        }
+        const dictionaryConfig = config[configKey];
+        this.caseSensitive = dictionaryConfig.caseSensitive || false;
+        if (!CACHE.has(this.cacheId)) {
+            const loaded = loadDictionary(dictionaryConfig);
+            CACHE.set(this.cacheId, loaded);
+        }
+    }
+    getDictionary() {
+        if (!this.cacheId)
+            return [];
+        return CACHE.get(this.cacheId) || [];
+    }
+    async parseWords(database, words) {
+        this.dictionary = this.getDictionary()
+            .map((word) => this.caseSensitive ? word : word.toLowerCase());
+        const matches = [];
+        const nonMatches = [];
+        for (const word of words) {
+            const compare = this.caseSensitive ? word : word.toLowerCase();
+            if (this.dictionary.includes(compare))
+                matches.push(word);
+            else
+                nonMatches.push(word);
+        }
+        await this.parseMatches(database, matches, nonMatches);
+    }
+}
+//# sourceMappingURL=dictionary.parser.mjs.map

package/dist/parsers/dictionary.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionary.parser.mjs","sourceRoot":"","sources":["../../src/parsers/dictionary.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,4BAA4B,EAAE,uBAAuB,EAAE,wBAAwB,EAAmB,MAAM,oBAAoB,CAAC;AAEhL,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAOrG,OAAO,EAAE,WAAW,EAAgB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAM/E,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAExC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,YAAY,CAAC;QAAE,OAAO,KAAK,CAAC;IACpE,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,KAAK,GAA0B,IAAI,GAAG,EAAoB,CAAC;AAEjE,SAAS,cAAc,CAAC,MAAyB;IAChD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAY,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE1E,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,WAAc;IAC/E,aAAa,GAAY,KAAK,CAAC;IACjC,UAAU,GAAa,EAAE,CAAC;IAC1B,OAAO,CAAmB;IAElC,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS;YAAE,OAAO;QAElC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAEzB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,SAAS,EAAE,mBAAmB,CAAC,EAAE,CAAC;YACzF,eAAe,CAAC,wCAAwC,SAAS,GAAG,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,SAAS,CAAsB,CAAC;QAEnF,IAAI,CAAC,aAAa,GAAG,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC;QAE7D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAa,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC1D,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAES,aAAa;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAE7B,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;IAIS,KAAK,CAAC,UAAU,CAAC,QAAyB,EAAE,KAAe;QACpE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE;aACnC,GAAG,CAAC,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAEjF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAW,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACvE,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBACrD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC;CACD"}

package/dist/parsers/html.parser.d.mts

@@ -0,0 +1,13 @@
+import * as cheerio from 'cheerio';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { StringParser } from './string.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare abstract class HtmlParser<T extends IDataConfig> extends StringParser<T> {
+    static collapseWhitespace(raw: string): string;
+    static loadDom(html: string): cheerio.CheerioAPI | undefined;
+    protected dom: cheerio.CheerioAPI | undefined;
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
+    supports(contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/{html.parser.js → html.parser.mjs}

@@ -1,5 +1,5 @@
 import * as cheerio from 'cheerio';
-import { StringParser } from './string.parser';
+import { StringParser } from './string.parser.mjs';
 export class HtmlParser extends StringParser {
     static collapseWhitespace(raw) {
         return raw.replace(/\s+/g, ' ');
@@ -8,11 +8,12 @@ export class HtmlParser extends StringParser {
         try {
             return cheerio.load(html);
         }
-        catch (ex) {
+        catch (_ex) {
             // ignore
             return undefined;
         }
     }
+    dom;
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         if (this.stringData && this.stringData.length) {
@@ -26,4 +27,4 @@ export class HtmlParser extends StringParser {
         return /^text\/html([^a-z0-9].*)?$/.test(contentType);
     }
 }
-//# sourceMappingURL=html.parser.js.map
+//# sourceMappingURL=html.parser.mjs.map

package/dist/parsers/html.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.parser.mjs","sourceRoot":"","sources":["../../src/parsers/html.parser.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGnD,MAAM,OAAgB,UAAkC,SAAQ,YAAe;IACvE,MAAM,CAAC,kBAAkB,CAAC,GAAW;QAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,IAAY;QACjC,IAAI,CAAC;YACJ,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,SAAS;YACT,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;IAES,GAAG,CAA+B;IAE5C,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAW,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;IACF,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,4BAA4B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;CACD"}