hydra-crawler 2.8.3 → 3.0.1

package/dist/parsers/image-tags.parser.mjs

@@ -0,0 +1,42 @@
+import { commonsLogDebug } from 'nodecommons-esm-log';
+import { HtmlParser } from './html.parser.mjs';
+export class ImageTagsParser extends HtmlParser {
+    url;
+    static parseImageTags(dom) {
+        const images = [];
+        dom('img').each((_index, element) => {
+            const src = dom(element).attr('src');
+            if (src === null || src === undefined)
+                return true;
+            const alt = dom(element).attr('alt');
+            const title = dom(element).attr('title');
+            const image = {
+                src: src
+            };
+            if (alt !== undefined && alt !== null)
+                image.alt = alt;
+            if (title !== undefined && title !== null)
+                image.title = title;
+            images.push(image);
+            return true;
+        });
+        return images;
+    }
+    constructor(url, outcome, config) {
+        super(outcome, config, 'imageTags');
+        this.url = url;
+    }
+    async parse(database) {
+        if (!this.dom || !this.url)
+            return; // invalid parse
+        const images = ImageTagsParser.parseImageTags(this.dom);
+        if (images.length > 0) {
+            commonsLogDebug(`Detected images for ${this.url}`);
+            await database.setData(this.url, 'images', images);
+        }
+        else {
+            await database.unsetData(this.url, 'images');
+        }
+    }
+}
+//# sourceMappingURL=image-tags.parser.mjs.map

package/dist/parsers/image-tags.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-tags.parser.mjs","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.mts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAOtD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;IA4BjD;IA3BH,MAAM,CAAC,cAAc,CAC1B,GAAuB;QAExB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAY,EAAW,EAAE;YACzD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEnD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEhE,MAAM,KAAK,GAAc;gBACvB,GAAG,EAAE,GAAG;aACT,CAAC;YACF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAE/D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAJ3B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO,CAAC,gBAAgB;QAEpD,MAAM,MAAM,GAAgB,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,eAAe,CAAC,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACnD,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC9C,CAAC;IACF,CAAC;CACD"}

package/dist/parsers/{interest.parser.d.ts → interest.parser.d.mts}

@@ -1,9 +1,9 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IOllamaConfig, OllamaParser } from './ollama.parser';
-import { IDataConfig } from './data.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IOllamaConfig, OllamaParser } from './ollama.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IInterestConfig extends IDataConfig {
     model: string;
     permittedDomains: string[];
@@ -13,6 +13,6 @@ export declare class InterestParser extends OllamaParser<IOllamaConfig> {
     private url?;
     private model;
     private permittedDomains;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseText(database: DatabaseService, text: string): Promise<void>;
 }

package/dist/parsers/interest.parser.mjs

@@ -0,0 +1,60 @@
+import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service.mjs';
+import { OllamaParser } from './ollama.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIInterestConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'model'))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
+        return false;
+    return true;
+}
+export class InterestParser extends OllamaParser {
+    url;
+    model;
+    permittedDomains = [];
+    constructor(url, outcome, config) {
+        super(outcome, config, 'interest');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'interest', isIInterestConfig)) {
+            commonsLogError('Invalid config for InterestParser');
+        }
+        this.model = config['interest'].model;
+        this.permittedDomains = config['interest'].permittedDomains;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (!this.ollamaUrl || !this.ollamaKey || !this.model)
+            return;
+        const hostname = new URL(this.url).hostname;
+        if (!this.permittedDomains.includes(hostname))
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'interest');
+            return;
+        }
+        const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
+        await this.debugGetLength();
+        // run this outside of await so that it doesn't block other things
+        void (async () => {
+            if (!this.url)
+                return;
+            const score = await rigService.generateInterestScore(text, this.ollamaKeepAlive);
+            if (score === undefined)
+                return;
+            if (score.score < 2)
+                delete score.reason;
+            commonsLogDebug(`Interest score is: ${score.score.toString(10)}`);
+            if (score.reason)
+                commonsLogDebug(`Reason for score is: ${score.reason}`);
+            await database.setData(this.url, 'interest', score);
+        })();
+    }
+}
+//# sourceMappingURL=interest.parser.mjs.map

package/dist/parsers/interest.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"interest.parser.mjs","sourceRoot":"","sources":["../../src/parsers/interest.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE/I,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGvE,OAAO,EAAE,2BAA2B,EAAuB,MAAM,gDAAgD,CAAC;AAKlH,OAAO,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAM/D,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC9C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,cAAe,SAAQ,YAA2B;IAKpD;IAJF,KAAK,CAAmB;IACxB,gBAAgB,GAAa,EAAE,CAAC;IAExC,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,UAAU,CACX,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAUrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAkB,MAAM,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE,CAAC;YACtF,eAAe,CAAC,mCAAmC,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,gBAAgB,CAAC;IAClF,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO;QAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAC/C,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;QAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,kEAAkE;QAClE,KAAK,CAAC,KAAK,IAAmB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,qBAAqB,CACjF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;YACF,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO;YAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;YAEzC,eAAe,CAAC,sBAAsB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAClE,IAAI,KAAK,CAAC,MAAM;gBAAE,eAAe,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAE1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;CACD"}

package/dist/parsers/jpeg.parser.d.mts

@@ -0,0 +1,11 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { DataParser, IDataConfig } from './data.parser.mjs';
+export declare class JpegParser extends DataParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    supports(contentType: string, isAllow: boolean): boolean;
+    parse(database: DatabaseService): Promise<void>;
+}

package/dist/parsers/jpeg.parser.mjs

@@ -0,0 +1,29 @@
+import * as jpeg from 'jpeg-js';
+import { DataParser } from './data.parser.mjs';
+export class JpegParser extends DataParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'jpeg');
+        this.url = url;
+    }
+    supports(contentType, isAllow) {
+        if (!isAllow)
+            return false;
+        return contentType.match(/^image\/(jpeg|jpg)$/) ? true : false;
+    }
+    async parse(database) {
+        if (!this.outcome || !this.url)
+            return;
+        if (!this.outcome.data)
+            return;
+        if (this.outcome.exceeded)
+            return; // don't try and parse vast JPEGs than haven't been downloaded properly, regardless of what the IDataConfig.allowExceeded is set to
+        try {
+            const rawImageData = jpeg.decode(this.outcome.data);
+            await database.setData(this.url, 'width', rawImageData.width);
+            await database.setData(this.url, 'height', rawImageData.height);
+        }
+        catch (_ex) { /* do nothing */ }
+    }
+}
+//# sourceMappingURL=jpeg.parser.mjs.map

package/dist/parsers/jpeg.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"jpeg.parser.mjs","sourceRoot":"","sources":["../../src/parsers/jpeg.parser.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAShC,OAAO,EAAE,UAAU,EAAe,MAAM,mBAAmB,CAAC;AAE5D,MAAM,OAAO,UAAW,SAAQ,UAAuB;IAE5C;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAJtB,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAChE,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEvC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ;YAAE,OAAO,CAAC,mIAAmI;QAEtK,IAAI,CAAC;YACJ,MAAM,YAAY,GAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAEpE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACnC,CAAC;CACD"}

package/dist/parsers/{llama-guard.parser.d.ts → llama-guard.parser.d.mts}

@@ -1,9 +1,9 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IOllamaConfig, OllamaParser } from './ollama.parser';
-import { IDataConfig } from './data.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IOllamaConfig, OllamaParser } from './ollama.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface ILlamaGuardConfig extends IDataConfig {
     model: string;
     permittedDomains: string[];
@@ -13,6 +13,6 @@ export declare class LlamaGuardParser extends OllamaParser<IOllamaConfig> {
     private url?;
     private model;
     private permittedDomains;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseText(database: DatabaseService, text: string): Promise<void>;
 }

package/dist/parsers/llama-guard.parser.mjs

@@ -0,0 +1,56 @@
+import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { ELlamaGuardClassification, RigLlamaGuardService } from '../services/rig-llama-guard.service.mjs';
+import { OllamaParser } from './ollama.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isILlamaGuardConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'model'))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
+        return false;
+    return true;
+}
+export class LlamaGuardParser extends OllamaParser {
+    url;
+    model;
+    permittedDomains = [];
+    constructor(url, outcome, config) {
+        super(outcome, config, 'llamaguard');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'llamaguard', isILlamaGuardConfig)) {
+            commonsLogError('Invalid config for LlamaGuardParser');
+        }
+        this.model = config['llamaguard'].model;
+        this.permittedDomains = config['llamaguard'].permittedDomains;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (!this.ollamaUrl || !this.ollamaKey || !this.model)
+            return;
+        const hostname = new URL(this.url).hostname;
+        if (!this.permittedDomains.includes(hostname))
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'llamaguard');
+            return;
+        }
+        const rigService = new RigLlamaGuardService(this.ollamaUrl, this.ollamaKey, this.model);
+        await this.debugGetLength();
+        // run this outside of await so that it doesn't block other things
+        void (async () => {
+            if (!this.url)
+                return;
+            const result = await rigService.generate(text, this.ollamaKeepAlive);
+            if (result === undefined)
+                return;
+            commonsLogDebug(`LlamaGuard result is: ${result.classification.toString()}${result.classification === ELlamaGuardClassification.UNSAFE ? ` (${result.categories.join(',')})` : ''}`);
+            await database.setData(this.url, 'llamaguard', result);
+        })();
+    }
+}
+//# sourceMappingURL=llama-guard.parser.mjs.map

package/dist/parsers/llama-guard.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-guard.parser.mjs","sourceRoot":"","sources":["../../src/parsers/llama-guard.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE/I,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGvE,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAqB,MAAM,yCAAyC,CAAC;AAK7H,OAAO,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAM/D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAA2B;IAKtD;IAJF,KAAK,CAAmB;IACxB,gBAAgB,GAAa,EAAE,CAAC;IAExC,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAUrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,YAAY,EAAE,mBAAmB,CAAC,EAAE,CAAC;YAC5F,eAAe,CAAC,qCAAqC,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,KAAK,CAAC;QAC/D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,gBAAgB,CAAC;IACtF,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO;QAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YACjD,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAyB,IAAI,oBAAoB,CAC/D,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;QAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,kEAAkE;QAClE,KAAK,CAAC,KAAK,IAAmB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,MAAM,MAAM,GAAgC,MAAM,UAAU,CAAC,QAAQ,CACnE,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;YACF,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO;YAEjC,eAAe,CAAC,yBAAyB,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,cAAc,KAAK,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAErL,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;CACD"}

package/dist/parsers/{offence.parser.d.ts → offence.parser.d.mts}

@@ -1,9 +1,9 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IOllamaConfig, OllamaParser } from './ollama.parser';
-import { IDataConfig } from './data.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IOllamaConfig, OllamaParser } from './ollama.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IOffenceConfig extends IDataConfig {
     model: string;
     permittedDomains: string[];
@@ -13,6 +13,6 @@ export declare class OffenceParser extends OllamaParser<IOllamaConfig> {
     private url?;
     private model;
     private permittedDomains;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseText(database: DatabaseService, text: string): Promise<void>;
 }

package/dist/parsers/offence.parser.mjs

@@ -0,0 +1,60 @@
+import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service.mjs';
+import { OllamaParser } from './ollama.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIOffenceConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'model'))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
+        return false;
+    return true;
+}
+export class OffenceParser extends OllamaParser {
+    url;
+    model;
+    permittedDomains = [];
+    constructor(url, outcome, config) {
+        super(outcome, config, 'offence');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'offence', isIOffenceConfig)) {
+            commonsLogError('Invalid config for OffenceParser');
+        }
+        this.model = config['offence'].model;
+        this.permittedDomains = config['offence'].permittedDomains;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (!this.ollamaUrl || !this.ollamaKey || !this.model)
+            return;
+        const hostname = new URL(this.url).hostname;
+        if (!this.permittedDomains.includes(hostname))
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'offence');
+            return;
+        }
+        const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
+        await this.debugGetLength();
+        // run this outside of await so that it doesn't block other things
+        void (async () => {
+            if (!this.url)
+                return;
+            const score = await rigService.generateOffenceScore(text, this.ollamaKeepAlive);
+            if (score === undefined)
+                return;
+            if (score.score < 1)
+                delete score.reason;
+            commonsLogDebug(`Offence score is: ${score.score.toString(10)}`);
+            if (score.reason)
+                commonsLogDebug(`Reason for score is: ${score.reason}`);
+            await database.setData(this.url, 'offence', score);
+        })();
+    }
+}
+//# sourceMappingURL=offence.parser.mjs.map

package/dist/parsers/offence.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"offence.parser.mjs","sourceRoot":"","sources":["../../src/parsers/offence.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE/I,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGvE,OAAO,EAAE,2BAA2B,EAAuB,MAAM,gDAAgD,CAAC;AAKlH,OAAO,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAM/D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAKnD;IAJF,KAAK,CAAmB;IACxB,gBAAgB,GAAa,EAAE,CAAC;IAExC,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAUrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACnF,eAAe,CAAC,kCAAkC,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,gBAAgB,CAAC;IAChF,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO;QAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAC9C,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;QAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,kEAAkE;QAClE,KAAK,CAAC,KAAK,IAAmB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,oBAAoB,CAChF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;YACF,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO;YAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;YAEzC,eAAe,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACjE,IAAI,KAAK,CAAC,MAAM;gBAAE,eAAe,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAE1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACpD,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;CACD"}

package/dist/parsers/{ollama.parser.d.ts → ollama.parser.d.mts}

@@ -1,8 +1,8 @@
-import { CommonsFixedDuration, TKeyObject } from 'tscommons-es-core';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { TextParser } from './text.parser';
-import { IDataConfig } from './data.parser';
+import { CommonsFixedDuration, TPropertyObject } from 'tscommons-esm-core';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { TextParser } from './text.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IOllamaConfig extends IDataConfig {
     url: string;
     key: string;
@@ -13,6 +13,6 @@ export declare abstract class OllamaParser<T extends IOllamaConfig> extends Text
     protected readonly ollamaUrl: string | undefined;
     protected readonly ollamaKey: string | undefined;
     protected readonly ollamaKeepAlive: CommonsFixedDuration | undefined;
-    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
     debugGetLength(): Promise<void>;
 }

package/dist/parsers/ollama.parser.mjs

@@ -0,0 +1,43 @@
+import { CommonsFixedDuration, commonsTypeHasPropertyNumberOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { RigQueueLengthService } from '../services/rig-queue-length.service.mjs';
+import { TextParser } from './text.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIOllamaConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'url'))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'key'))
+        return false;
+    if (!commonsTypeHasPropertyNumberOrUndefined(test, 'keepAlive'))
+        return false;
+    return true;
+}
+export class OllamaParser extends TextParser {
+    ollamaUrl;
+    ollamaKey;
+    ollamaKeepAlive;
+    constructor(outcome, config, configKey) {
+        super(outcome, config, configKey);
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'ollama', isIOllamaConfig)) {
+            commonsLogError('Invalid config for OllamaParser');
+        }
+        const ollamaConfig = config['ollama'];
+        this.ollamaUrl = ollamaConfig.url;
+        this.ollamaKey = ollamaConfig.key;
+        if (ollamaConfig.keepAlive) {
+            this.ollamaKeepAlive = CommonsFixedDuration.fromMillis(ollamaConfig.keepAlive);
+        }
+    }
+    async debugGetLength() {
+        if (!this.ollamaUrl || !this.ollamaKey)
+            return;
+        const rigService = new RigQueueLengthService(this.ollamaUrl, this.ollamaKey);
+        const length = await rigService.getLength();
+        commonsLogDebug(`Rig queue length is: ${length.active} active, ${length.queue} queued: ${length.line} total line length.`);
+    }
+}
+//# sourceMappingURL=ollama.parser.mjs.map

package/dist/parsers/ollama.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama.parser.mjs","sourceRoot":"","sources":["../../src/parsers/ollama.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,uCAAuC,EAAE,4BAA4B,EAAE,uBAAuB,EAAmB,MAAM,oBAAoB,CAAC;AAE3K,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,0CAA0C,CAAC;AAOjF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAO/D,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,YAAsC,SAAQ,UAAa;IAC7D,SAAS,CAAmB;IAC5B,SAAS,CAAmB;IAC5B,eAAe,CAAiC;IAEnE,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QAEF,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAgB,MAAM,EAAE,QAAQ,EAAE,eAAe,CAAC,EAAE,CAAC;YAChF,eAAe,CAAC,iCAAiC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,YAAY,GAAkB,MAAM,CAAC,QAAQ,CAAkB,CAAC;QAEtE,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAElC,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;YAC5B,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAChF,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,cAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE/C,MAAM,UAAU,GAA0B,IAAI,qBAAqB,CACjE,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,CACf,CAAC;QAEF,MAAM,MAAM,GAAiB,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1D,eAAe,CAAC,wBAAwB,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,IAAI,qBAAqB,CAAC,CAAC;IAC5H,CAAC;CACD"}

package/dist/parsers/{paragraphs.parser.d.ts → paragraphs.parser.d.mts}

@@ -1,13 +1,13 @@
-/// <reference types="cheerio" />
-import { DatabaseService } from '../services/database.service';
-import { HtmlParser } from './html.parser';
-import { IDataConfig } from './data.parser';
+import * as cheerio from 'cheerio';
+import { DatabaseService } from '../services/database.service.mjs';
+import { HtmlParser } from './html.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IParagraphsConfig extends IDataConfig {
     paragraphWordsThreshold?: number;
 }
 export declare function isIParagraphsConfig(test: unknown): test is IParagraphsConfig;
 export declare abstract class ParagraphsParser<T extends IParagraphsConfig> extends HtmlParser<T> {
-    static parseParagraphs(dom: cheerio.Root, paragraphsConfig: IParagraphsConfig): string[];
+    static parseParagraphs(dom: cheerio.CheerioAPI, paragraphsConfig: IParagraphsConfig): string[];
     protected abstract parseParagraphs(database: DatabaseService, paragraphs: string[]): Promise<void>;
     parse(database: DatabaseService): Promise<void>;
 }

package/dist/parsers/paragraphs.parser.mjs

@@ -0,0 +1,38 @@
+import { commonsStringSplitWords, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-esm-core';
+import { HtmlParser } from './html.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIParagraphsConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyNumberOrUndefined(test, 'paragraphWordsThreshold'))
+        return false;
+    return true;
+}
+export class ParagraphsParser extends HtmlParser {
+    static parseParagraphs(dom, paragraphsConfig) {
+        const nodes = dom('p,dd')
+            .contents()
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+            .filter((_index, element) => element.nodeType === 3);
+        const paragraphs = [];
+        dom(nodes).each((_index, element) => {
+            const paragraph = dom(element).text();
+            if (paragraphsConfig.paragraphWordsThreshold && commonsStringSplitWords(paragraph).length >= paragraphsConfig.paragraphWordsThreshold) {
+                paragraphs.push(paragraph);
+            }
+        });
+        return paragraphs
+            .map((p) => p.trim())
+            .filter((p) => p !== '');
+    }
+    async parse(database) {
+        if (!this.dom)
+            return; // invalid parse
+        const paragraphsConfig = this.getConfig(isIParagraphsConfig);
+        if (!paragraphsConfig)
+            return;
+        const paragraphs = ParagraphsParser.parseParagraphs(this.dom, paragraphsConfig);
+        await this.parseParagraphs(database, paragraphs);
+    }
+}
+//# sourceMappingURL=paragraphs.parser.mjs.map

package/dist/parsers/paragraphs.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"paragraphs.parser.mjs","sourceRoot":"","sources":["../../src/parsers/paragraphs.parser.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,uCAAuC,EAAE,MAAM,oBAAoB,CAAC;AAItG,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAK/D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,yBAAyB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,UAAa;IACjF,MAAM,CAAC,eAAe,CAC3B,GAAuB,EACvB,gBAAmC;QAEpC,MAAM,KAAK,GAAyB,GAAG,CAAC,MAAM,CAAC;aAC5C,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAY,EAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAE7E,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CACb,CAAC,MAAc,EAAE,OAAY,EAAQ,EAAE;YACtC,MAAM,SAAS,GAAW,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9C,IAAI,gBAAgB,CAAC,uBAAuB,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,gBAAgB,CAAC,uBAAuB,EAAE,CAAC;gBACvI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;QACF,CAAC,CACF,CAAC;QAEF,OAAO,UAAU;aACd,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC;IAIe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO,CAAC,gBAAgB;QAEvC,MAAM,gBAAgB,GAAgC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAC1F,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAE9B,MAAM,UAAU,GAAa,gBAAgB,CAAC,eAAe,CAC3D,IAAI,CAAC,GAAG,EACR,gBAAgB,CACjB,CAAC;QAEF,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;CACD"}

package/dist/parsers/{parser.d.ts → parser.d.mts}

@@ -1,14 +1,14 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParser } from '../interfaces/iparser';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IParserConfig } from '../interfaces/iparser-config';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParser } from '../interfaces/iparser.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
 export declare abstract class Parser<T extends IParserConfig = IParserConfig> implements IParser {
     protected outcome?: IRequestOutcome | undefined;
     private configKey?;
     private enabled;
     private parserConfig;
-    constructor(outcome?: IRequestOutcome | undefined, config?: TKeyObject<IParserConfig>, configKey?: string | undefined);
+    constructor(outcome?: IRequestOutcome | undefined, config?: TPropertyObject<IParserConfig>, configKey?: string | undefined);
     protected getConfig<P>(// this allows subclasses to get their config without having to have all the T attributes of their own children
     checker: (test: unknown) => test is P): P | undefined;
     isEnabled(): boolean;

package/dist/parsers/parser.mjs

@@ -0,0 +1,45 @@
+import { commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogError } from 'nodecommons-esm-log';
+import { isIParserConfig } from '../interfaces/iparser-config.mjs';
+export class Parser {
+    outcome;
+    configKey;
+    enabled = false;
+    parserConfig;
+    // these all have to be optional because of the Ctor approach used elsewhere
+    // similarly, we don't check types of the config (T) other than the enabled in TParserConfig
+    constructor(outcome, config, configKey) {
+        this.outcome = outcome;
+        this.configKey = configKey;
+        if (!config || !this.configKey)
+            return;
+        if (!commonsTypeHasPropertyT(config, this.configKey, isIParserConfig)) {
+            commonsLogError(`Invalid config for Parser (${this.configKey})`);
+        }
+        this.parserConfig = config[this.configKey];
+        this.enabled = this.parserConfig.enabled;
+    }
+    getConfig(// this allows subclasses to get their config without having to have all the T attributes of their own children
+    checker) {
+        if (!this.parserConfig)
+            return undefined;
+        if (!checker(this.parserConfig))
+            throw new Error(`Invalid parser config for ${this.configKey}`);
+        return this.parserConfig;
+    }
+    isEnabled() {
+        return this.enabled;
+    }
+    async init(_database) {
+        // subclasses can implement
+    }
+    async parse(_database) {
+        // subclasses can implement
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async links() {
+        // subclasses can implement
+        return [];
+    }
+}
+//# sourceMappingURL=parser.mjs.map

package/dist/parsers/parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.mjs","sourceRoot":"","sources":["../../src/parsers/parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAmB,MAAM,oBAAoB,CAAC;AAE9E,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAMtD,OAAO,EAAiB,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAElF,MAAM,OAAgB,MAAM;IAOf;IAEF;IARF,OAAO,GAAY,KAAK,CAAC;IACzB,YAAY,CAA0B;IAE9C,4EAA4E;IAC5E,4FAA4F;IAC5F,YACY,OAAyB,EACnC,MAAuC,EAC/B,SAAkB;QAFhB,YAAO,GAAP,OAAO,CAAkB;QAE3B,cAAS,GAAT,SAAS,CAAS;QAE3B,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAEvC,IAAI,CAAC,uBAAuB,CAAgB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,CAAC;YACtF,eAAe,CAAC,8BAA8B,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAM,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IAC1C,CAAC;IAES,SAAS,CAAK,+GAA+G;IACrI,OAAqC;QAEtC,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,SAAS,CAAC;QAEzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,SAAU,EAAE,CAAC,CAAC;QAEjG,OAAO,IAAI,CAAC,YAAiB,CAAC;IAC/B,CAAC;IAEM,SAAS;QACf,OAAO,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAIM,KAAK,CAAC,IAAI,CAAC,SAA0B;QAC3C,2BAA2B;IAC5B,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,SAA0B;QAC5C,2BAA2B;IAC5B,CAAC;IAED,4DAA4D;IACrD,KAAK,CAAC,KAAK;QACjB,2BAA2B;QAE3B,OAAO,EAAE,CAAC;IACX,CAAC;CACD"}

package/dist/parsers/php-error.parser.d.mts

@@ -0,0 +1,12 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { StringParser } from './string.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export declare class PhpErrorParser extends StringParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    supports(_contentType: string, isAllow: boolean): boolean;
+}

package/dist/parsers/php-error.parser.mjs

@@ -0,0 +1,42 @@
+import { deriveEPhpErrorType } from 'hydra-crawler-esm-assets';
+import { StringParser } from './string.parser.mjs';
+const PHP_ERROR = '<br />\\s{1,2}<b>([A-Za-z ]+?)</b>:  ([^<]+?) in <b>([^<]+?)</b> on line <b>([0-9]+?)</b><br />';
+export class PhpErrorParser extends StringParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'phpError');
+        this.url = url;
+    }
+    async parse(database) {
+        if (!this.stringData || !this.url)
+            return;
+        const pattern = new RegExp(PHP_ERROR, 'g');
+        const errors = [];
+        while (true) {
+            const result = pattern.exec(this.stringData);
+            if (result === null)
+                break;
+            const type = deriveEPhpErrorType(result[1]);
+            if (type === undefined)
+                continue;
+            const message = result[2];
+            const file = result[3];
+            const line = parseInt(result[4], 10);
+            const error = {
+                type: type,
+                message: message,
+                file: file,
+                line: line
+            };
+            errors.push(error);
+        }
+        if (errors.length > 0)
+            await database.setData(this.url, 'phpErrors', errors);
+        else
+            await database.unsetData(this.url, 'phpErrors');
+    }
+    supports(_contentType, isAllow) {
+        return isAllow; // everything allowlisted
+    }
+}
+//# sourceMappingURL=php-error.parser.mjs.map