hydra-crawler 1.4.6 → 2.0.2

package/dist/parsers/paragraphs.parser.js

@@ -1,56 +1,49 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParagraphsParser = exports.isIParagraphsConfig = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const tscommons_core_2 = require("tscommons-core");
-const html_parser_1 = require("./html.parser");
-const data_parser_1 = require("./data.parser");
-function isIParagraphsConfig(test) {
-    if (!data_parser_1.isIDataConfig(test))
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsStringSplitWords, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-es-core';
+import { HtmlParser } from './html.parser';
+import { isIDataConfig } from './data.parser';
+export function isIParagraphsConfig(test) {
+    if (!isIDataConfig(test))
         return false;
-    if (!tscommons_core_1.CommonsType.hasPropertyNumberOrUndefined(test, 'paragraphWordsThreshold'))
+    if (!commonsTypeHasPropertyNumberOrUndefined(test, 'paragraphWordsThreshold'))
         return false;
     return true;
 }
-exports.isIParagraphsConfig = isIParagraphsConfig;
-class ParagraphsParser extends html_parser_1.HtmlParser {
-    static parseParagraphs(
-    // @ts-ignore
-    dom, paragraphsConfig) {
-        // tslint:disable:no-invalid-this
-        // @ts-ignore
+export class ParagraphsParser extends HtmlParser {
+    static parseParagraphs(dom, paragraphsConfig) {
         const nodes = dom('p,dd')
             .contents()
-            .filter(function () {
-            // @ts-ignore
-            return this.nodeType === 3;
-        });
-        // tslint:enable:no-invalid-this
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+            .filter((_index, element) => element.nodeType === 3);
         const paragraphs = [];
-        // tslint:disable:no-invalid-this
-        dom(nodes).each(function () {
-            // @ts-ignore
-            const paragraph = dom(this).text();
-            if (paragraphsConfig.paragraphWordsThreshold && tscommons_core_2.CommonsString.splitWords(paragraph).length >= paragraphsConfig.paragraphWordsThreshold) {
+        dom(nodes).each((_index, element) => {
+            const paragraph = dom(element).text();
+            if (paragraphsConfig.paragraphWordsThreshold && commonsStringSplitWords(paragraph).length >= paragraphsConfig.paragraphWordsThreshold) {
                 paragraphs.push(paragraph);
             }
         });
-        // tslint:enable:no-invalid-this
         return paragraphs
             .map((p) => p.trim())
             .filter((p) => p !== '');
     }
-    constructor(outcome, config, configKey) {
-        super(outcome, config, configKey);
-    }
-    async parse(database) {
-        if (!this.dom)
-            return; // invalid parse
-        const paragraphsConfig = this.getConfig(isIParagraphsConfig);
-        if (!paragraphsConfig)
-            return;
-        const paragraphs = ParagraphsParser.parseParagraphs(this.dom, paragraphsConfig);
-        await this.parseParagraphs(database, paragraphs);
+    parse(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.dom)
+                return; // invalid parse
+            const paragraphsConfig = this.getConfig(isIParagraphsConfig);
+            if (!paragraphsConfig)
+                return;
+            const paragraphs = ParagraphsParser.parseParagraphs(this.dom, paragraphsConfig);
+            yield this.parseParagraphs(database, paragraphs);
+        });
     }
 }
-exports.ParagraphsParser = ParagraphsParser;
+//# sourceMappingURL=paragraphs.parser.js.map

package/dist/parsers/paragraphs.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paragraphs.parser.js","sourceRoot":"","sources":["../../src/parsers/paragraphs.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,uCAAuC,EAAE,MAAM,mBAAmB,CAAC;AAIrG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,yBAAyB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,UAAa;IACjF,MAAM,CAAC,eAAe,CAC3B,GAAiB,EACjB,gBAAmC;QAEpC,MAAM,KAAK,GAAoB,GAAG,CAAC,MAAM,CAAC;aACvC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAElG,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CACb,CAAC,MAAc,EAAE,OAAwB,EAAQ,EAAE;YAClD,MAAM,SAAS,GAAW,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9C,IAAI,gBAAgB,CAAC,uBAAuB,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,gBAAgB,CAAC,uBAAuB,EAAE;gBACtI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aAC3B;QACF,CAAC,CACF,CAAC;QAEF,OAAO,UAAU;aACd,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC;IAIY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gBAAgB;YAEvC,MAAM,gBAAgB,GAAgC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC1F,IAAI,CAAC,gBAAgB;gBAAE,OAAO;YAE9B,MAAM,UAAU,GAAa,gBAAgB,CAAC,eAAe,CAC3D,IAAI,CAAC,GAAG,EACR,gBAAgB,CACjB,CAAC;YAEF,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;KAAA;CACD"}

package/dist/parsers/parser.d.ts

@@ -0,0 +1,19 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IParser } from '../interfaces/iparser';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { IParserConfig } from '../interfaces/iparser-config';
+export declare abstract class Parser<T extends IParserConfig = IParserConfig> implements IParser {
+    protected outcome?: IRequestOutcome | undefined;
+    private configKey?;
+    private enabled;
+    private parserConfig;
+    constructor(outcome?: IRequestOutcome | undefined, config?: TKeyObject<IParserConfig>, configKey?: string | undefined);
+    protected getConfig<P>(// this allows subclasses to get their config without having to have all the T attributes of their own children
+    checker: (test: unknown) => test is P): P | undefined;
+    isEnabled(): boolean;
+    abstract supports(contentType: string, isAllow: boolean): boolean;
+    init(_database: DatabaseService): Promise<void>;
+    parse(_database: DatabaseService): Promise<void>;
+    links(): Promise<string[]>;
+}

package/dist/parsers/parser.js

@@ -1,10 +1,16 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Parser = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const iparser_config_1 = require("../interfaces/iparser-config");
-class Parser {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsTypeHasPropertyT } from 'tscommons-es-core';
+import { commonsOutputError } from 'nodecommons-es-cli';
+import { isIParserConfig } from '../interfaces/iparser-config';
+export class Parser {
     // these all have to be optional because of the Ctor approach used elsewhere
     // similarly, we don't check types of the config (T) other than the enabled in TParserConfig
     constructor(outcome, config, configKey) {
@@ -13,8 +19,8 @@ class Parser {
         this.enabled = false;
         if (!config || !this.configKey)
             return;
-        if (!tscommons_core_1.CommonsType.hasPropertyT(config, this.configKey, iparser_config_1.isIParserConfig)) {
-            nodecommons_cli_1.CommonsOutput.error(`Invalid config for Parser (${this.configKey})`);
+        if (!commonsTypeHasPropertyT(config, this.configKey, isIParserConfig)) {
+            commonsOutputError(`Invalid config for Parser (${this.configKey})`);
         }
         this.parserConfig = config[this.configKey];
         this.enabled = this.parserConfig.enabled;
@@ -30,15 +36,22 @@ class Parser {
     isEnabled() {
         return this.enabled;
     }
-    async init(_database) {
-        // subclasses can implement
+    init(_database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // subclasses can implement
+        });
     }
-    async parse(_database) {
-        // subclasses can implement
+    parse(_database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // subclasses can implement
+        });
     }
-    async links() {
-        // subclasses can implement
-        return [];
+    // eslint-disable-next-line @typescript-eslint/require-await
+    links() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // subclasses can implement
+            return [];
+        });
     }
 }
-exports.Parser = Parser;
+//# sourceMappingURL=parser.js.map

package/dist/parsers/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/parsers/parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAExE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAMxD,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAE9E,MAAM,OAAgB,MAAM;IAI3B,4EAA4E;IAC5E,4FAA4F;IAC5F,YACY,OAAyB,EACnC,MAAkC,EAC1B,SAAkB;QAFhB,YAAO,GAAP,OAAO,CAAkB;QAE3B,cAAS,GAAT,SAAS,CAAS;QARpB,YAAO,GAAY,KAAK,CAAC;QAUhC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAEvC,IAAI,CAAC,uBAAuB,CAAgB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE;YACrF,kBAAkB,CAAC,8BAA8B,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;SACpE;QAED,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAM,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IAC1C,CAAC;IAES,SAAS,CAAK,+GAA+G;IACrI,OAAqC;QAEtC,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,SAAS,CAAC;QAEzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,SAAU,EAAE,CAAC,CAAC;QAEjG,OAAO,IAAI,CAAC,YAAiB,CAAC;IAC/B,CAAC;IAEM,SAAS;QACf,OAAO,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAIY,IAAI,CAAC,SAA0B;;YAC3C,2BAA2B;QAC5B,CAAC;KAAA;IAEY,KAAK,CAAC,SAA0B;;YAC5C,2BAA2B;QAC5B,CAAC;KAAA;IAED,4DAA4D;IAC/C,KAAK;;YACjB,2BAA2B;YAE3B,OAAO,EAAE,CAAC;QACX,CAAC;KAAA;CACD"}

package/dist/parsers/php-error.parser.d.ts

@@ -0,0 +1,12 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { StringParser } from './string.parser';
+import { IDataConfig } from './data.parser';
+export declare class PhpErrorParser extends StringParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    supports(_contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/php-error.parser.js

@@ -1,44 +1,52 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PhpErrorParser = void 0;
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const string_parser_1 = require("./string.parser");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { deriveEPhpErrorType } from 'hydra-crawler-ts-assets';
+import { StringParser } from './string.parser';
 const PHP_ERROR = '<br />\\s{1,2}<b>([A-Za-z ]+?)</b>:  ([^<]+?) in <b>([^<]+?)</b> on line <b>([0-9]+?)</b><br />';
-class PhpErrorParser extends string_parser_1.StringParser {
+export class PhpErrorParser extends StringParser {
     constructor(url, outcome, config) {
         super(outcome, config, 'phpError');
         this.url = url;
     }
-    async parse(database) {
-        if (!this.stringData || !this.url)
-            return;
-        const pattern = new RegExp(PHP_ERROR, 'g');
-        const errors = [];
-        while (true) {
-            const result = pattern.exec(this.stringData);
-            if (result === null)
-                break;
-            const type = hydra_crawler_ts_assets_1.deriveEPhpErrorType(result[1]);
-            if (type === undefined)
-                continue;
-            const message = result[2];
-            const file = result[3];
-            const line = parseInt(result[4], 10);
-            const error = {
-                type: type,
-                message: message,
-                file: file,
-                line: line
-            };
-            errors.push(error);
-        }
-        if (errors.length > 0)
-            await database.setData(this.url, 'phpErrors', errors);
-        else
-            await database.unsetData(this.url, 'phpErrors');
+    parse(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.stringData || !this.url)
+                return;
+            const pattern = new RegExp(PHP_ERROR, 'g');
+            const errors = [];
+            while (true) {
+                const result = pattern.exec(this.stringData);
+                if (result === null)
+                    break;
+                const type = deriveEPhpErrorType(result[1]);
+                if (type === undefined)
+                    continue;
+                const message = result[2];
+                const file = result[3];
+                const line = parseInt(result[4], 10);
+                const error = {
+                    type: type,
+                    message: message,
+                    file: file,
+                    line: line
+                };
+                errors.push(error);
+            }
+            if (errors.length > 0)
+                yield database.setData(this.url, 'phpErrors', errors);
+            else
+                yield database.unsetData(this.url, 'phpErrors');
+        });
     }
     supports(_contentType, isAllow) {
         return isAllow; // everything allowlisted
     }
 }
-exports.PhpErrorParser = PhpErrorParser;
+//# sourceMappingURL=php-error.parser.js.map

package/dist/parsers/php-error.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"php-error.parser.js","sourceRoot":"","sources":["../../src/parsers/php-error.parser.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAiB,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAO7E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,SAAS,GAAW,iGAAiG,CAAC;AAE5H,MAAM,OAAO,cAAe,SAAQ,YAAyB;IAC5D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAE1C,MAAM,OAAO,GAAW,IAAI,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAEnD,MAAM,MAAM,GAAgB,EAAE,CAAC;YAC/B,OAAO,IAAI,EAAE;gBACZ,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACnE,IAAI,MAAM,KAAK,IAAI;oBAAE,MAAM;gBAE3B,MAAM,IAAI,GAA4B,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACrE,IAAI,IAAI,KAAK,SAAS;oBAAE,SAAS;gBAEjC,MAAM,OAAO,GAAW,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClC,MAAM,IAAI,GAAW,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,IAAI,GAAW,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAE7C,MAAM,KAAK,GAAc;oBACvB,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,OAAO;oBAChB,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,IAAI;iBACX,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACnB;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;;gBACxE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACtD,CAAC;KAAA;IAEM,QAAQ,CAAC,YAAoB,EAAE,OAAgB;QACrD,OAAO,OAAO,CAAC,CAAC,yBAAyB;IAC1C,CAAC;CACD"}

package/dist/parsers/phrase.parser.d.ts

@@ -0,0 +1,8 @@
+import { DatabaseService } from '../services/database.service';
+import { RegexParser } from './regex.parser';
+import { IDictionaryConfig } from './dictionary.parser';
+export declare abstract class PhraseParser<T extends IDictionaryConfig> extends RegexParser<T> {
+    setPhrases(phrases: string[], caseInsensitive: boolean): void;
+    protected abstract parsePhrases(database: DatabaseService, phrases: string[]): Promise<void>;
+    protected parseRegExs(database: DatabaseService, matches: Map<string, RegExpExecArray[]>): Promise<void>;
+}

package/dist/parsers/phrase.parser.js

@@ -1,11 +1,14 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PhraseParser = void 0;
-const regex_parser_1 = require("./regex.parser");
-class PhraseParser extends regex_parser_1.RegexParser {
-    constructor(outcome, config, configKey) {
-        super(outcome, config, configKey);
-    }
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { RegexParser } from './regex.parser';
+export class PhraseParser extends RegexParser {
     setPhrases(phrases, caseInsensitive) {
         const map = new Map();
         for (const phrase of phrases) {
@@ -14,8 +17,10 @@ class PhraseParser extends regex_parser_1.RegexParser {
         }
         this.setRegExs(map);
     }
-    async parseRegExs(database, matches) {
-        this.parsePhrases(database, Array.from(matches.keys()));
+    parseRegExs(database, matches) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.parsePhrases(database, Array.from(matches.keys()));
+        });
     }
 }
-exports.PhraseParser = PhraseParser;
+//# sourceMappingURL=phrase.parser.js.map

package/dist/parsers/phrase.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"phrase.parser.js","sourceRoot":"","sources":["../../src/parsers/phrase.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG7C,MAAM,OAAgB,YAA0C,SAAQ,WAAc;IAC9E,UAAU,CACf,OAAiB,EACjB,eAAwB;QAEzB,MAAM,GAAG,GAAwB,IAAI,GAAG,EAAkB,CAAC;QAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC7B,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,mBAAmB,MAAM,kBAAkB,EAAE,IAAI,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChH,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;SACvB;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAIe,WAAW,CAAC,QAAyB,EAAE,OAAuC;;YAC7F,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;KAAA;CACD"}

package/dist/parsers/regex.parser.d.ts

@@ -0,0 +1,10 @@
+import { DatabaseService } from '../services/database.service';
+import { TextParser } from './text.parser';
+import { IDataConfig } from './data.parser';
+export declare abstract class RegexParser<T extends IDataConfig> extends TextParser<T> {
+    private regexs;
+    protected setRegExs(regexs: Map<string, RegExp>): void;
+    getRegExs(): Map<string, RegExp>;
+    protected abstract parseRegExs(database: DatabaseService, matches: Map<string, RegExpExecArray[]>): Promise<void>;
+    protected parseText(database: DatabaseService, text: string): Promise<void>;
+}

package/dist/parsers/regex.parser.js

@@ -1,10 +1,16 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RegexParser = void 0;
-const text_parser_1 = require("./text.parser");
-class RegexParser extends text_parser_1.TextParser {
-    constructor(outcome, config, configKey) {
-        super(outcome, config, configKey);
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { TextParser } from './text.parser';
+export class RegexParser extends TextParser {
+    constructor() {
+        super(...arguments);
         this.regexs = new Map();
     }
     setRegExs(regexs) {
@@ -15,21 +21,23 @@ class RegexParser extends text_parser_1.TextParser {
     getRegExs() {
         return this.regexs;
     }
-    async parseText(database, text) {
-        const matches = new Map();
-        for (const key of this.regexs.keys()) {
-            const regex = this.regexs.get(key);
-            const results = [];
-            while (true) {
-                const result = regex.exec(text);
-                if (!result || result === null)
-                    break;
-                results.push(result);
+    parseText(database, text) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const matches = new Map();
+            for (const key of this.regexs.keys()) {
+                const regex = this.regexs.get(key);
+                const results = [];
+                while (true) {
+                    const result = regex.exec(text);
+                    if (!result || result === null)
+                        break;
+                    results.push(result);
+                }
+                if (results.length > 0)
+                    matches.set(key, results);
             }
-            if (results.length > 0)
-                matches.set(key, results);
-        }
-        await this.parseRegExs(database, matches);
+            yield this.parseRegExs(database, matches);
+        });
     }
 }
-exports.RegexParser = RegexParser;
+//# sourceMappingURL=regex.parser.js.map

package/dist/parsers/regex.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"regex.parser.js","sourceRoot":"","sources":["../../src/parsers/regex.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAG3C,MAAM,OAAgB,WAAmC,SAAQ,UAAa;IAA9E;;QACS,WAAM,GAAwB,IAAI,GAAG,EAAkB,CAAC;IAgCjE,CAAC;IA9BU,SAAS,CAAC,MAA2B;QAC9C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE;YAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,CAAC;SACvC;IACF,CAAC;IAEM,SAAS;QACf,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;IAIe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,MAAM,OAAO,GAAmC,IAAI,GAAG,EAA6B,CAAC;YACrF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE;gBACrC,MAAM,KAAK,GAAW,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;gBAE5C,MAAM,OAAO,GAAsB,EAAE,CAAC;gBACtC,OAAO,IAAI,EAAE;oBACZ,MAAM,MAAM,GAAyB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtD,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,IAAI;wBAAE,MAAM;oBAEtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACrB;gBAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;aAClD;YAED,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;KAAA;CACD"}

package/dist/parsers/server.parser.d.ts

@@ -0,0 +1,11 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { Parser } from './parser';
+export declare class ServerParser extends Parser<IParserConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    supports(_contentType: string): boolean;
+    parse(database: DatabaseService): Promise<void>;
+}

package/dist/parsers/server.parser.js

@@ -1,66 +1,67 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ServerParser = void 0;
-const eserver_1 = require("../enums/eserver");
-const ex_powered_by_1 = require("../enums/ex-powered-by");
-const parser_1 = require("./parser");
-class ServerParser extends parser_1.Parser {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { EServer } from '../enums/eserver';
+import { EXPoweredBy } from '../enums/ex-powered-by';
+import { Parser } from './parser';
+export class ServerParser extends Parser {
     constructor(url, outcome, config) {
         super(outcome, config, 'server');
         this.url = url;
     }
-    async init(database) {
-        const urls = database.getUrls();
-        if (!urls)
-            return;
-        await urls.createIndex({ 'server.server': 1 }, { unique: false });
-        await urls.createIndex({ 'server.xPoweredBy': 1 }, { unique: false });
-    }
     supports(_contentType) {
         return true; // everything
     }
-    async parse(database) {
-        if (!this.outcome || !this.url)
-            return;
-        const server = this.outcome.headers['server'] || undefined;
-        const xPoweredBy = this.outcome.headers['x-powered-by'] || undefined;
-        if (server === undefined && xPoweredBy === undefined) {
-            await database.unsetData(this.url, 'server');
-            return;
-        }
-        const data = {};
-        if (server) {
-            if (/^Apache($|\/)/.test(server))
-                data.server = eserver_1.EServer.APACHE;
-            if (/^Microsoft-IIS($|\/)/.test(server))
-                data.server = eserver_1.EServer.IIS;
-            if (/^nginx($|\/)/.test(server))
-                data.server = eserver_1.EServer.NGINX;
-            if (/^cloudflare($|\/)/.test(server))
-                data.server = eserver_1.EServer.CLOUDFLARE;
-            if (/^AmazonS3($|\/)/.test(server))
-                data.server = eserver_1.EServer.AMAZONS3;
-            if (/^Jetty($|\/)/.test(server))
-                data.server = eserver_1.EServer.JETTY;
-        }
-        if (xPoweredBy) {
-            if (/^PHP($|\/)/.test(xPoweredBy))
-                data.xPoweredBy = ex_powered_by_1.EXPoweredBy.PHP;
-            if (/^ASP.NET($|\/)/.test(xPoweredBy))
-                data.xPoweredBy = ex_powered_by_1.EXPoweredBy.ASPNET;
-            if (/^Express($|\/)/.test(xPoweredBy))
-                data.xPoweredBy = ex_powered_by_1.EXPoweredBy.EXPRESS;
-            if (/^Zope($|\/)/.test(xPoweredBy))
-                data.xPoweredBy = ex_powered_by_1.EXPoweredBy.ZOPE;
-        }
-        if (data.server === undefined && data.xPoweredBy === undefined) {
-            await database.unsetData(this.url, 'server');
-            return;
-        }
-        try {
-            await database.setData(this.url, 'server', data);
-        }
-        catch (ex) { /* do nothing */ }
+    parse(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.outcome || !this.url)
+                return;
+            const server = this.outcome.headers['server'] || undefined;
+            const xPoweredBy = this.outcome.headers['x-powered-by'] || undefined;
+            if (server === undefined && xPoweredBy === undefined) {
+                yield database.unsetData(this.url, 'server');
+                return;
+            }
+            const data = {};
+            if (server) {
+                if (/^Apache($|\/)/.test(server))
+                    data.server = EServer.APACHE;
+                if (/^Microsoft-IIS($|\/)/.test(server))
+                    data.server = EServer.IIS;
+                if (/^nginx($|\/)/.test(server))
+                    data.server = EServer.NGINX;
+                if (/^cloudflare($|\/)/.test(server))
+                    data.server = EServer.CLOUDFLARE;
+                if (/^AmazonS3($|\/)/.test(server))
+                    data.server = EServer.AMAZONS3;
+                if (/^Jetty($|\/)/.test(server))
+                    data.server = EServer.JETTY;
+            }
+            if (xPoweredBy) {
+                if (/^PHP($|\/)/.test(xPoweredBy))
+                    data.xPoweredBy = EXPoweredBy.PHP;
+                if (/^ASP.NET($|\/)/.test(xPoweredBy))
+                    data.xPoweredBy = EXPoweredBy.ASPNET;
+                if (/^Express($|\/)/.test(xPoweredBy))
+                    data.xPoweredBy = EXPoweredBy.EXPRESS;
+                if (/^Zope($|\/)/.test(xPoweredBy))
+                    data.xPoweredBy = EXPoweredBy.ZOPE;
+            }
+            if (data.server === undefined && data.xPoweredBy === undefined) {
+                yield database.unsetData(this.url, 'server');
+                return;
+            }
+            try {
+                yield database.setData(this.url, 'server', data);
+            }
+            catch (ex) { /* do nothing */ }
+        });
     }
 }
-exports.ServerParser = ServerParser;
+//# sourceMappingURL=server.parser.js.map

package/dist/parsers/server.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.parser.js","sourceRoot":"","sources":["../../src/parsers/server.parser.ts"],"names":[],"mappings":";;;;;;;;;AAQA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,MAAM,OAAO,YAAa,SAAQ,MAAqB;IACtD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAJxB,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEM,QAAQ,CAAC,YAAoB;QACnC,OAAO,IAAI,CAAC,CAAC,aAAa;IAC3B,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEvC,MAAM,MAAM,GAAsB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAY,IAAI,SAAS,CAAC;YACzF,MAAM,UAAU,GAAsB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAY,IAAI,SAAS,CAAC;YAEnG,IAAI,MAAM,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE;gBACrD,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAC7C,OAAO;aACP;YAED,MAAM,IAAI,GAAY,EAAE,CAAC;YAEzB,IAAI,MAAM,EAAE;gBACX,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;gBAC/D,IAAI,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;gBACnE,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;gBAC7D,IAAI,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;gBACvE,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;gBACnE,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;aAC7D;YAED,IAAI,UAAU,EAAE;gBACf,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC;oBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC;gBACrE,IAAI,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC;oBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;gBAC5E,IAAI,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC;oBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC;gBAC7E,IAAI,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC;oBAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC;aACvE;YAED,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE;gBAC/D,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAC7C,OAAO;aACP;YAED,IAAI;gBACH,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;aACjD;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QAClC,CAAC;KAAA;CACD"}

package/dist/parsers/spelling.parser.d.ts

@@ -0,0 +1,10 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { DictionaryParser, IDictionaryConfig } from './dictionary.parser';
+export declare class SpellingParser extends DictionaryParser<IDictionaryConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    protected parseMatches(database: DatabaseService, matches: string[], _nonMatches: string[]): Promise<void>;
+}