hydra-crawler 1.4.6 → 2.0.0

package/dist/parsers/dictionary.parser.d.ts

@@ -0,0 +1,19 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { WordsParser, IWordsConfig } from './words.parser';
+export interface IDictionaryConfig extends IWordsConfig {
+    dictionary: string;
+    caseSensitive?: boolean;
+}
+export declare function isIDictionaryConfig(test: unknown): test is IDictionaryConfig;
+export declare abstract class DictionaryParser<T extends IDictionaryConfig> extends WordsParser<T> {
+    protected caseSensitive: boolean;
+    private dictionary;
+    private cacheId;
+    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
+    protected getDictionary(): string[];
+    protected abstract parseMatches(database: DatabaseService, matches: string[], nonMatches: string[]): Promise<void>;
+    protected parseWords(database: DatabaseService, words: string[]): Promise<void>;
+}

package/dist/parsers/dictionary.parser.js

@@ -1,22 +1,36 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DictionaryParser = exports.isIDictionaryConfig = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_file_1 = require("nodecommons-file");
-const words_parser_1 = require("./words.parser");
-function isIDictionaryConfig(test) {
-    if (!words_parser_1.isIWordsConfig(test))
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT, commonsTypeIsStringArray } from 'tscommons-es-core';
+import { commonsOutputError } from 'nodecommons-es-cli';
+import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-es-file';
+import { WordsParser, isIWordsConfig } from './words.parser';
+export function isIDictionaryConfig(test) {
+    if (!isIWordsConfig(test))
         return false;
-    if (!tscommons_core_1.CommonsType.hasPropertyString(test, 'dictionary'))
+    if (!commonsTypeHasPropertyString(test, 'dictionary'))
         return false;
-    if (!tscommons_core_1.CommonsType.hasPropertyBooleanOrUndefined(test, 'caseSensitive'))
+    if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'caseSensitive'))
         return false;
     return true;
 }
-exports.isIDictionaryConfig = isIDictionaryConfig;
 const CACHE = new Map();
-class DictionaryParser extends words_parser_1.WordsParser {
+function loadDictionary(config) {
+    const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
+    const json = commonsFileReadJsonFile(filename);
+    if (json === undefined)
+        throw new Error('Unable to read dictionary file');
+    if (!commonsTypeIsStringArray(json))
+        throw new Error('Dictionary file is not a JSON array');
+    return json;
+}
+export class DictionaryParser extends WordsParser {
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         this.caseSensitive = false;
@@ -24,42 +38,36 @@ class DictionaryParser extends words_parser_1.WordsParser {
         if (!config || !configKey)
             return;
         this.cacheId = configKey;
-        if (!tscommons_core_1.CommonsType.hasPropertyT(config, configKey, isIDictionaryConfig)) {
-            nodecommons_cli_1.CommonsOutput.error(`Invalid config for DictionaryParser (${configKey})`);
+        if (!commonsTypeHasPropertyT(config, configKey, isIDictionaryConfig)) {
+            commonsOutputError(`Invalid config for DictionaryParser (${configKey})`);
         }
         const dictionaryConfig = config[configKey];
         this.caseSensitive = dictionaryConfig.caseSensitive || false;
         if (!CACHE.has(this.cacheId)) {
-            CACHE.set(this.cacheId, DictionaryParser.loadDictionary(dictionaryConfig));
+            const loaded = loadDictionary(dictionaryConfig);
+            CACHE.set(this.cacheId, loaded);
         }
     }
-    static loadDictionary(config) {
-        const filename = `${__dirname}/../../config/${config.dictionary}`;
-        const json = nodecommons_file_1.CommonsFile.readJsonFile(filename);
-        if (json === undefined)
-            throw new Error('Unable to read dictionary file');
-        if (!tscommons_core_1.CommonsType.isStringArray(json))
-            throw new Error('Dictionary file is not a JSON array');
-        return json;
-    }
     getDictionary() {
         if (!this.cacheId)
             return [];
         return CACHE.get(this.cacheId) || [];
     }
-    async parseWords(database, words) {
-        this.dictionary = this.getDictionary()
-            .map((word) => this.caseSensitive ? word : word.toLowerCase());
-        const matches = [];
-        const nonMatches = [];
-        for (const word of words) {
-            const compare = this.caseSensitive ? word : word.toLowerCase();
-            if (this.dictionary.includes(compare))
-                matches.push(word);
-            else
-                nonMatches.push(word);
-        }
-        await this.parseMatches(database, matches, nonMatches);
+    parseWords(database, words) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.dictionary = this.getDictionary()
+                .map((word) => this.caseSensitive ? word : word.toLowerCase());
+            const matches = [];
+            const nonMatches = [];
+            for (const word of words) {
+                const compare = this.caseSensitive ? word : word.toLowerCase();
+                if (this.dictionary.includes(compare))
+                    matches.push(word);
+                else
+                    nonMatches.push(word);
+            }
+            yield this.parseMatches(database, matches, nonMatches);
+        });
     }
 }
-exports.DictionaryParser = DictionaryParser;
+//# sourceMappingURL=dictionary.parser.js.map

package/dist/parsers/dictionary.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionary.parser.js","sourceRoot":"","sources":["../../src/parsers/dictionary.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,wCAAwC,EAAE,4BAA4B,EAAE,uBAAuB,EAAE,wBAAwB,EAAc,MAAM,mBAAmB,CAAC;AAE1K,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAE,WAAW,EAAgB,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAM3E,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAExC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,YAAY,CAAC;QAAE,OAAO,KAAK,CAAC;IACpE,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,KAAK,GAA0B,IAAI,GAAG,EAAoB,CAAC;AAEjE,SAAS,cAAc,CAAC,MAAyB;IAChD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAsB,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE1E,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,WAAc;IAKzF,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QATzB,kBAAa,GAAY,KAAK,CAAC;QACjC,eAAU,GAAa,EAAE,CAAC;QAUjC,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS;YAAE,OAAO;QAElC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAEzB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,SAAS,EAAE,mBAAmB,CAAC,EAAE;YACxF,kBAAkB,CAAC,wCAAwC,SAAS,GAAG,CAAC,CAAC;SACzE;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,SAAS,CAAsB,CAAC;QAEnF,IAAI,CAAC,aAAa,GAAG,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC;QAE7D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC7B,MAAM,MAAM,GAAa,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC1D,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;SAChC;IACF,CAAC;IAES,aAAa;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAE7B,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;IAIe,UAAU,CAAC,QAAyB,EAAE,KAAe;;YACpE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE;iBACnC,GAAG,CAAC,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAEjF,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAa,EAAE,CAAC;YAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACzB,MAAM,OAAO,GAAW,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACvE,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;oBACrD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC3B;YAED,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;CACD"}

package/dist/parsers/html.parser.d.ts

@@ -0,0 +1,13 @@
+/// <reference types="cheerio" />
+import { TKeyObject } from 'tscommons-es-core';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { StringParser } from './string.parser';
+import { IDataConfig } from './data.parser';
+export declare abstract class HtmlParser<T extends IDataConfig> extends StringParser<T> {
+    static collapseWhitespace(raw: string): string;
+    static loadDom(html: string): cheerio.Root | undefined;
+    protected dom: cheerio.Root | undefined;
+    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
+    supports(contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/html.parser.js

@@ -1,9 +1,6 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HtmlParser = void 0;
-const cheerio = require("cheerio");
-const string_parser_1 = require("./string.parser");
-class HtmlParser extends string_parser_1.StringParser {
+import * as cheerio from 'cheerio';
+import { StringParser } from './string.parser';
+export class HtmlParser extends StringParser {
     constructor(outcome, config, configKey) {
         super(outcome, config, configKey);
         if (this.stringData && this.stringData.length) {
@@ -14,7 +11,6 @@ class HtmlParser extends string_parser_1.StringParser {
     static collapseWhitespace(raw) {
         return raw.replace(/\s+/g, ' ');
     }
-    // @ts-ignore
     static loadDom(html) {
         try {
             return cheerio.load(html);
@@ -30,4 +26,4 @@ class HtmlParser extends string_parser_1.StringParser {
         return /^text\/html([^a-z0-9].*)?$/.test(contentType);
     }
 }
-exports.HtmlParser = HtmlParser;
+//# sourceMappingURL=html.parser.js.map

package/dist/parsers/html.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.parser.js","sourceRoot":"","sources":["../../src/parsers/html.parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,OAAgB,UAAkC,SAAQ,YAAe;IAgB9E,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAC9C,MAAM,SAAS,GAAW,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;SACzC;IACF,CAAC;IA3BM,MAAM,CAAC,kBAAkB,CAAC,GAAW;QAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,IAAY;QACjC,IAAI;YACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC1B;QAAC,OAAO,EAAE,EAAE;YACZ,SAAS;YACT,OAAO,SAAS,CAAC;SACjB;IACF,CAAC;IAkBM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,4BAA4B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;CACD"}

package/dist/parsers/hyperlinks.parser.d.ts

@@ -0,0 +1,20 @@
+/// <reference types="cheerio" />
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { HtmlParser } from './html.parser';
+import { IDataConfig } from './data.parser';
+export interface IHyperlinksConfig extends IDataConfig {
+    sources: string[];
+}
+export declare function isIHyperlinksConfig(test: unknown): test is IHyperlinksConfig;
+export declare class HyperlinksParser extends HtmlParser<IHyperlinksConfig> {
+    private url?;
+    static parseTagLinks(dom: cheerio.Root, tag: string, attribute: string): string[];
+    static getBaseHref(dom: cheerio.Root, url: string): string | undefined;
+    private hyperlinksConfig;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    links(): Promise<string[]>;
+}

package/dist/parsers/hyperlinks.parser.js

@@ -1,110 +1,115 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HyperlinksParser = exports.isIHyperlinksConfig = void 0;
-const url_1 = require("url");
-const tscommons_core_1 = require("tscommons-core");
-const html_parser_1 = require("./html.parser");
-const data_parser_1 = require("./data.parser");
-function isIHyperlinksConfig(test) {
-    if (!data_parser_1.isIDataConfig(test))
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { URL } from 'url';
+import { commonsArrayUnique, commonsTypeHasPropertyStringArray } from 'tscommons-es-core';
+import { HtmlParser } from './html.parser';
+import { isIDataConfig } from './data.parser';
+export function isIHyperlinksConfig(test) {
+    if (!isIDataConfig(test))
         return false;
-    if (!tscommons_core_1.CommonsType.hasPropertyStringArray(test, 'sources'))
+    if (!commonsTypeHasPropertyStringArray(test, 'sources'))
         return false;
     return true;
 }
-exports.isIHyperlinksConfig = isIHyperlinksConfig;
-class HyperlinksParser extends html_parser_1.HtmlParser {
+export class HyperlinksParser extends HtmlParser {
     constructor(url, outcome, config) {
         super(outcome, config, 'hyperlinks');
         this.url = url;
         this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
     }
-    static parseTagLinks(
-    // @ts-ignore
-    dom, tag, attribute) {
+    static parseTagLinks(dom, tag, attribute) {
         const links = [];
-        // @ts-ignore
-        dom(`${tag}[${attribute}]`).each(function (_index, _element) {
-            // @ts-ignore
-            const link = dom(this).attr(attribute);
-            links.push(link);
+        dom(`${tag}[${attribute}]`).each((_index, element) => {
+            const link = dom(element).attr(attribute);
+            if (link !== undefined)
+                links.push(link);
             return true;
         });
-        return [...new Set(links)];
+        return commonsArrayUnique(links);
     }
-    // @ts-ignore
     static getBaseHref(dom, url) {
         try {
-            // @ts-ignore
             const base = dom('base[href]');
             if (base.length === 0)
                 return undefined;
             const href = dom(base[0]).attr('href');
             if (!href)
                 return undefined;
-            const whatwg = new url_1.URL(href, url);
+            const whatwg = new URL(href, url);
             return whatwg.toString();
         }
         catch (ex) { /* do nothing */ }
         return undefined;
     }
-    async parse(database) {
-        if (!this.dom || !this.url || !this.hyperlinksConfig)
-            return; // invalid parse
-        let title;
-        try {
-            title = this.dom('title').text();
-        }
-        catch (ex) { /* do nothing */ }
-        if (title !== undefined && title !== null && title !== '') {
-            await database.setData(this.url, 'title', title);
-        }
-        const links = {};
-        for (const source of this.hyperlinksConfig.sources) {
-            const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
-            if (pattern)
-                links[pattern[1]] = HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2]);
-        }
-        await database.setData(this.url, 'links', links);
-    }
-    async links() {
-        if (!this.dom || !this.url || !this.hyperlinksConfig)
-            return []; // invalid parse
-        let links = [];
-        for (const source of this.hyperlinksConfig.sources) {
-            const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
-            if (pattern) {
-                if (pattern[1] === 'form')
-                    continue;
-                links = [...links, ...HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2])];
-            }
-        }
-        const unique = [...new Set(links)];
-        let baseHref = HyperlinksParser.getBaseHref(this.dom, this.url);
-        if (baseHref === undefined)
-            baseHref = this.url;
-        const results = [];
-        for (let l of unique) {
-            const anchor = l.split('#');
-            l = anchor[0];
-            if (l === '')
-                continue;
+    parse(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.dom || !this.url || !this.hyperlinksConfig)
+                return; // invalid parse
+            let title;
             try {
-                const u = new url_1.URL(l, baseHref);
-                if (u.toString().indexOf('#') !== -1) {
-                    process.exit(1);
+                title = this.dom('title').text();
+            }
+            catch (ex) { /* do nothing */ }
+            if (title !== undefined && title !== null && title !== '') {
+                yield database.setData(this.url, 'title', title);
+            }
+            const links = {};
+            for (const source of this.hyperlinksConfig.sources) {
+                const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
+                if (pattern)
+                    links[pattern[1]] = HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2]);
+            }
+            yield database.setData(this.url, 'links', links);
+        });
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    links() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.dom || !this.url || !this.hyperlinksConfig)
+                return []; // invalid parse
+            let links = [];
+            for (const source of this.hyperlinksConfig.sources) {
+                const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
+                if (pattern) {
+                    if (pattern[1] === 'form')
+                        continue;
+                    links = [...links, ...HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2])];
                 }
-                if (!u.protocol.match(/^http(s?):$/))
-                    continue;
-                if (u.toString() === this.url || u.toString() === baseHref) {
-                    // skip self-links, or presumed self-links
+            }
+            const unique = [...new Set(links)];
+            let baseHref = HyperlinksParser.getBaseHref(this.dom, this.url);
+            if (baseHref === undefined)
+                baseHref = this.url;
+            const results = [];
+            for (let l of unique) {
+                const anchor = l.split('#');
+                l = anchor[0];
+                if (l === '')
                     continue;
+                try {
+                    const u = new URL(l, baseHref);
+                    if (u.toString().indexOf('#') !== -1) {
+                        process.exit(1);
+                    }
+                    if (!u.protocol.match(/^http(s?):$/))
+                        continue;
+                    if (u.toString() === this.url || u.toString() === baseHref) {
+                        // skip self-links, or presumed self-links
+                        continue;
+                    }
+                    results.push(u.toString());
                 }
-                results.push(u.toString());
+                catch (ex) { /* do nothing */ }
             }
-            catch (ex) { /* do nothing */ }
-        }
-        return results;
+            return results;
+        });
     }
 }
-exports.HyperlinksParser = HyperlinksParser;
+//# sourceMappingURL=hyperlinks.parser.js.map

package/dist/parsers/hyperlinks.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hyperlinks.parser.js","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAOtG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;IAkClE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAJ5B,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAC7D,CAAC;IAxCM,MAAM,CAAC,aAAa,CACzB,GAAiB,EACjB,GAAW,EACX,SAAiB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACtF,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5D,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,GAAiB,EAAE,GAAW;QACvD,IAAI;YACH,MAAM,IAAI,GAAoB,GAAG,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAExC,MAAM,IAAI,GAAqB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAE5B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;SACzB;QAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QACjC,OAAO,SAAS,CAAC;IAClB,CAAC;IAcY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,CAAC,gBAAgB;YAE9E,IAAI,KAAuB,CAAC;YAC5B,IAAI;gBACH,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACjC;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;YAEjC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE;gBAC1D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;aACjD;YAED,MAAM,KAAK,GAAkC,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO;oBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;aAClG;YAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;KAAA;IAED,4DAA4D;IAC/C,KAAK;;YACjB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,EAAE,CAAC,CAAC,gBAAgB;YAEjF,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO,EAAE;oBACZ,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;wBAAE,SAAS;oBACpC,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxF;aACD;YAED,MAAM,MAAM,GAAa,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAE7C,IAAI,QAAQ,GAAqB,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAClF,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,IAAI,MAAM,EAAE;gBACrB,MAAM,MAAM,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,KAAK,EAAE;oBAAE,SAAS;gBAEvB,IAAI;oBACH,MAAM,CAAC,GAAQ,IAAI,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;wBACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;qBAChB;oBACD,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;wBAAE,SAAS;oBAC/C,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,QAAQ,EAAE;wBAC3D,0CAA0C;wBAC1C,SAAS;qBACT;oBAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;iBAC3B;gBAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;aACjC;YAED,OAAO,OAAO,CAAC;QAChB,CAAC;KAAA;CACD"}

package/dist/parsers/image-tags.parser.d.ts

@@ -0,0 +1,20 @@
+/// <reference types="cheerio" />
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { HtmlParser } from './html.parser';
+import { IDataConfig } from './data.parser';
+declare type TImageTag = {
+    src: string;
+    alt?: string;
+    title?: string;
+};
+export declare class ImageTagsParser extends HtmlParser<IDataConfig> {
+    private url?;
+    static parseImageTags(dom: cheerio.Root): TImageTag[];
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    init(database: DatabaseService): Promise<void>;
+    parse(database: DatabaseService): Promise<void>;
+}
+export {};

package/dist/parsers/image-tags.parser.js

@@ -1,27 +1,27 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ImageTagsParser = void 0;
-const nodecommons_cli_1 = require("nodecommons-cli");
-const html_parser_1 = require("./html.parser");
-class ImageTagsParser extends html_parser_1.HtmlParser {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsOutputDebug } from 'nodecommons-es-cli';
+import { HtmlParser } from './html.parser';
+export class ImageTagsParser extends HtmlParser {
     constructor(url, outcome, config) {
         super(outcome, config, 'imageTags');
         this.url = url;
     }
-    static parseImageTags(
-    // @ts-ignore
-    dom) {
+    static parseImageTags(dom) {
         const images = [];
-        // @ts-ignore
-        dom('img').each(function (_index, _element) {
-            // @ts-ignore
-            const src = dom(this).attr('src');
+        dom('img').each((_index, element) => {
+            const src = dom(element).attr('src');
             if (src === null || src === undefined)
                 return true;
-            // @ts-ignore
-            const alt = dom(this).attr('alt');
-            // @ts-ignore
-            const title = dom(this).attr('title');
+            const alt = dom(element).attr('alt');
+            const title = dom(element).attr('title');
             const image = {
                 src: src
             };
@@ -34,23 +34,27 @@ class ImageTagsParser extends html_parser_1.HtmlParser {
         });
         return images;
     }
-    async init(database) {
-        const urls = database.getUrls();
-        if (!urls)
-            return;
-        await urls.createIndex({ 'images.alt': 1 }, { unique: false });
+    init(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const urls = database.getUrls();
+            if (!urls)
+                return;
+            yield urls.createIndex({ 'images.alt': 1 }, { unique: false });
+        });
     }
-    async parse(database) {
-        if (!this.dom || !this.url)
-            return; // invalid parse
-        const images = ImageTagsParser.parseImageTags(this.dom);
-        if (images.length > 0) {
-            nodecommons_cli_1.CommonsOutput.debug(`Detected images for ${this.url}`);
-            await database.setData(this.url, 'images', images);
-        }
-        else {
-            await database.unsetData(this.url, 'images');
-        }
+    parse(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.dom || !this.url)
+                return; // invalid parse
+            const images = ImageTagsParser.parseImageTags(this.dom);
+            if (images.length > 0) {
+                commonsOutputDebug(`Detected images for ${this.url}`);
+                yield database.setData(this.url, 'images', images);
+            }
+            else {
+                yield database.unsetData(this.url, 'images');
+            }
+        });
     }
 }
-exports.ImageTagsParser = ImageTagsParser;
+//# sourceMappingURL=image-tags.parser.js.map

package/dist/parsers/image-tags.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-tags.parser.js","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.ts"],"names":[],"mappings":";;;;;;;;;AAMA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAS3C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;IA2B3D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAJ3B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAhCM,MAAM,CAAC,cAAc,CAC1B,GAAiB;QAElB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACrE,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEnD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEhE,MAAM,KAAK,GAAc;gBACvB,GAAG,EAAE,GAAG;aACT,CAAC;YACF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAE/D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAUY,IAAI,CAAC,QAAyB;;YAC1C,MAAM,IAAI,GAA+B,QAAQ,CAAC,OAAO,EAAE,CAAC;YAC5D,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,CAAC;KAAA;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gBAAgB;YAEpD,MAAM,MAAM,GAAgB,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACtB,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACtD,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;aACnD;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;aAC7C;QACF,CAAC;KAAA;CACD"}

package/dist/parsers/jpeg.parser.d.ts

@@ -0,0 +1,11 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { DatabaseService } from '../services/database.service';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { DataParser, IDataConfig } from './data.parser';
+export declare class JpegParser extends DataParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    supports(contentType: string, isAllow: boolean): boolean;
+    parse(database: DatabaseService): Promise<void>;
+}

package/dist/parsers/jpeg.parser.js

@@ -1,9 +1,15 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.JpegParser = void 0;
-const jpeg = require("jpeg-js");
-const data_parser_1 = require("./data.parser");
-class JpegParser extends data_parser_1.DataParser {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as jpeg from 'jpeg-js';
+import { DataParser } from './data.parser';
+export class JpegParser extends DataParser {
     constructor(url, outcome, config) {
         super(outcome, config, 'jpeg');
         this.url = url;
@@ -13,19 +19,21 @@ class JpegParser extends data_parser_1.DataParser {
             return false;
         return contentType.match(/^image\/(jpeg|jpg)$/) ? true : false;
     }
-    async parse(database) {
-        if (!this.outcome || !this.url)
-            return;
-        if (!this.outcome.data)
-            return;
-        if (this.outcome.exceeded)
-            return; // don't try and parse vast JPEGs than haven't been downloaded properly, regardless of what the IDataConfig.allowExceeded is set to
-        try {
-            const rawImageData = jpeg.decode(this.outcome.data);
-            await database.setData(this.url, 'width', rawImageData.width);
-            await database.setData(this.url, 'height', rawImageData.height);
-        }
-        catch (ex) { /* do nothing */ }
+    parse(database) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.outcome || !this.url)
+                return;
+            if (!this.outcome.data)
+                return;
+            if (this.outcome.exceeded)
+                return; // don't try and parse vast JPEGs than haven't been downloaded properly, regardless of what the IDataConfig.allowExceeded is set to
+            try {
+                const rawImageData = jpeg.decode(this.outcome.data);
+                yield database.setData(this.url, 'width', rawImageData.width);
+                yield database.setData(this.url, 'height', rawImageData.height);
+            }
+            catch (ex) { /* do nothing */ }
+        });
     }
 }
-exports.JpegParser = JpegParser;
+//# sourceMappingURL=jpeg.parser.js.map

package/dist/parsers/jpeg.parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jpeg.parser.js","sourceRoot":"","sources":["../../src/parsers/jpeg.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAShC,OAAO,EAAE,UAAU,EAAe,MAAM,eAAe,CAAC;AAExD,MAAM,OAAO,UAAW,SAAQ,UAAuB;IACtD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAJtB,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAChE,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEvC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI;gBAAE,OAAO;YAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,OAAO,CAAC,mIAAmI;YAEtK,IAAI;gBACH,MAAM,YAAY,GAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAEpE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;gBAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;aAChE;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QAClC,CAAC;KAAA;CACD"}

package/dist/parsers/paragraphs.parser.d.ts

@@ -0,0 +1,13 @@
+/// <reference types="cheerio" />
+import { DatabaseService } from '../services/database.service';
+import { HtmlParser } from './html.parser';
+import { IDataConfig } from './data.parser';
+export interface IParagraphsConfig extends IDataConfig {
+    paragraphWordsThreshold?: number;
+}
+export declare function isIParagraphsConfig(test: unknown): test is IParagraphsConfig;
+export declare abstract class ParagraphsParser<T extends IParagraphsConfig> extends HtmlParser<T> {
+    static parseParagraphs(dom: cheerio.Root, paragraphsConfig: IParagraphsConfig): string[];
+    protected abstract parseParagraphs(database: DatabaseService, paragraphs: string[]): Promise<void>;
+    parse(database: DatabaseService): Promise<void>;
+}