hydra-crawler 2.8.4 → 3.0.2

package/dist/parsers/hyperlinks.parser.d.mts

@@ -0,0 +1,20 @@
+import * as cheerio from 'cheerio';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { HtmlParser } from './html.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+export interface IHyperlinksConfig extends IDataConfig {
+    sources: string[];
+}
+export declare function isIHyperlinksConfig(test: unknown): test is IHyperlinksConfig;
+export declare class HyperlinksParser extends HtmlParser<IHyperlinksConfig> {
+    private url?;
+    static parseTagLinks(dom: cheerio.CheerioAPI, tag: string, attribute: string): string[];
+    static getBaseHref(dom: cheerio.CheerioAPI, url: string): string | undefined;
+    private hyperlinksConfig;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+    links(): Promise<string[]>;
+}

package/dist/parsers/hyperlinks.parser.mjs

@@ -0,0 +1,104 @@
+import { URL } from 'url';
+import { commonsArrayUnique, commonsTypeHasPropertyStringArray } from 'tscommons-esm-core';
+import { HtmlParser } from './html.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIHyperlinksConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'sources'))
+        return false;
+    return true;
+}
+export class HyperlinksParser extends HtmlParser {
+    url;
+    static parseTagLinks(dom, tag, attribute) {
+        const links = [];
+        dom(`${tag}[${attribute}]`).each((_index, element) => {
+            const link = dom(element).attr(attribute);
+            if (link !== undefined)
+                links.push(link);
+            return true;
+        });
+        return commonsArrayUnique(links);
+    }
+    static getBaseHref(dom, url) {
+        try {
+            const base = dom('base[href]');
+            if (base.length === 0)
+                return undefined;
+            const href = dom(base[0]).attr('href');
+            if (!href)
+                return undefined;
+            const whatwg = new URL(href, url);
+            return whatwg.toString();
+        }
+        catch (_ex) { /* do nothing */ }
+        return undefined;
+    }
+    hyperlinksConfig;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'hyperlinks');
+        this.url = url;
+        this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
+    }
+    async parse(database) {
+        if (!this.dom || !this.url || !this.hyperlinksConfig)
+            return; // invalid parse
+        let title;
+        try {
+            title = this.dom('title').text();
+        }
+        catch (_ex) { /* do nothing */ }
+        if (title !== undefined && title !== null && title !== '') {
+            await database.setData(this.url, 'title', title);
+        }
+        const links = {};
+        for (const source of this.hyperlinksConfig.sources) {
+            const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
+            if (pattern)
+                links[pattern[1]] = HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2]);
+        }
+        await database.setData(this.url, 'links', links);
+    }
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async links() {
+        if (!this.dom || !this.url || !this.hyperlinksConfig)
+            return []; // invalid parse
+        let links = [];
+        for (const source of this.hyperlinksConfig.sources) {
+            const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
+            if (pattern) {
+                if (pattern[1] === 'form')
+                    continue;
+                links = [...links, ...HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2])];
+            }
+        }
+        const unique = [...new Set(links)];
+        let baseHref = HyperlinksParser.getBaseHref(this.dom, this.url);
+        if (baseHref === undefined)
+            baseHref = this.url;
+        const results = [];
+        for (let l of unique) {
+            const anchor = l.split('#');
+            l = anchor[0];
+            if (l === '')
+                continue;
+            try {
+                const u = new URL(l, baseHref);
+                if (u.toString().indexOf('#') !== -1) {
+                    process.exit(1);
+                }
+                if (!u.protocol.match(/^http(s?):$/))
+                    continue;
+                if (u.toString() === this.url || u.toString() === baseHref) {
+                    // skip self-links, or presumed self-links
+                    continue;
+                }
+                results.push(u.toString());
+            }
+            catch (_ex) { /* do nothing */ }
+        }
+        return results;
+    }
+}
+//# sourceMappingURL=hyperlinks.parser.mjs.map

package/dist/parsers/hyperlinks.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"hyperlinks.parser.mjs","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAI1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAmB,MAAM,oBAAoB,CAAC;AAO5G,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAK/D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;IAmCxD;IAjCH,MAAM,CAAC,aAAa,CACzB,GAAuB,EACvB,GAAW,EACX,SAAiB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAY,EAAW,EAAE;YAC1E,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5D,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,GAAuB,EAAE,GAAW;QAC7D,IAAI,CAAC;YACJ,MAAM,IAAI,GAAyB,GAAG,CAAC,YAAY,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAExC,MAAM,IAAI,GAAqB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAE5B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAClC,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,gBAAgB,CAA8B;IAEtD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAJ5B,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAC7D,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,CAAC,gBAAgB;QAE9E,IAAI,KAAuB,CAAC;QAC5B,IAAI,CAAC;YACJ,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QAElC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,KAAK,GAAkC,EAAE,CAAC;QAChD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YACpD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACvF,IAAI,OAAO;gBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACnG,CAAC;QAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED,4DAA4D;IAC5C,KAAK,CAAC,KAAK;QAC1B,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,EAAE,CAAC,CAAC,gBAAgB;QAEjF,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YACpD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACvF,IAAI,OAAO,EAAE,CAAC;gBACb,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;oBAAE,SAAS;gBACpC,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAa,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAE7C,IAAI,QAAQ,GAAqB,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAClF,IAAI,QAAQ,KAAK,SAAS;YAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;QAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;YACtB,MAAM,MAAM,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACtC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,KAAK,EAAE;gBAAE,SAAS;YAEvB,IAAI,CAAC;gBACJ,MAAM,CAAC,GAAQ,IAAI,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjB,CAAC;gBACD,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;oBAAE,SAAS;gBAC/C,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,QAAQ,EAAE,CAAC;oBAC5D,0CAA0C;oBAC1C,SAAS;gBACV,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,OAAO,CAAC;IAChB,CAAC;CACD"}

package/dist/parsers/image-tags.parser.d.mts

@@ -0,0 +1,19 @@
+import * as cheerio from 'cheerio';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { HtmlParser } from './html.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
+type TImageTag = {
+    src: string;
+    alt?: string;
+    title?: string;
+};
+export declare class ImageTagsParser extends HtmlParser<IDataConfig> {
+    private url?;
+    static parseImageTags(dom: cheerio.CheerioAPI): TImageTag[];
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    parse(database: DatabaseService): Promise<void>;
+}
+export {};

package/dist/parsers/image-tags.parser.mjs

@@ -0,0 +1,42 @@
+import { commonsLogDebug } from 'nodecommons-esm-log';
+import { HtmlParser } from './html.parser.mjs';
+export class ImageTagsParser extends HtmlParser {
+    url;
+    static parseImageTags(dom) {
+        const images = [];
+        dom('img').each((_index, element) => {
+            const src = dom(element).attr('src');
+            if (src === null || src === undefined)
+                return true;
+            const alt = dom(element).attr('alt');
+            const title = dom(element).attr('title');
+            const image = {
+                src: src
+            };
+            if (alt !== undefined && alt !== null)
+                image.alt = alt;
+            if (title !== undefined && title !== null)
+                image.title = title;
+            images.push(image);
+            return true;
+        });
+        return images;
+    }
+    constructor(url, outcome, config) {
+        super(outcome, config, 'imageTags');
+        this.url = url;
+    }
+    async parse(database) {
+        if (!this.dom || !this.url)
+            return; // invalid parse
+        const images = ImageTagsParser.parseImageTags(this.dom);
+        if (images.length > 0) {
+            commonsLogDebug(`Detected images for ${this.url}`);
+            await database.setData(this.url, 'images', images);
+        }
+        else {
+            await database.unsetData(this.url, 'images');
+        }
+    }
+}
+//# sourceMappingURL=image-tags.parser.mjs.map

package/dist/parsers/image-tags.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-tags.parser.mjs","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.mts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAOtD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAS/C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;IA4BjD;IA3BH,MAAM,CAAC,cAAc,CAC1B,GAAuB;QAExB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAY,EAAW,EAAE;YACzD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEnD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEhE,MAAM,KAAK,GAAc;gBACvB,GAAG,EAAE,GAAG;aACT,CAAC;YACF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAE/D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAJ3B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO,CAAC,gBAAgB;QAEpD,MAAM,MAAM,GAAgB,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,eAAe,CAAC,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACnD,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC9C,CAAC;IACF,CAAC;CACD"}

package/dist/parsers/{interest.parser.d.ts → interest.parser.d.mts}

@@ -1,9 +1,9 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IOllamaConfig, OllamaParser } from './ollama.parser';
-import { IDataConfig } from './data.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IOllamaConfig, OllamaParser } from './ollama.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IInterestConfig extends IDataConfig {
     model: string;
     permittedDomains: string[];
@@ -13,6 +13,6 @@ export declare class InterestParser extends OllamaParser<IOllamaConfig> {
     private url?;
     private model;
     private permittedDomains;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseText(database: DatabaseService, text: string): Promise<void>;
 }

package/dist/parsers/interest.parser.mjs

@@ -0,0 +1,60 @@
+import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service.mjs';
+import { OllamaParser } from './ollama.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIInterestConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'model'))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
+        return false;
+    return true;
+}
+export class InterestParser extends OllamaParser {
+    url;
+    model;
+    permittedDomains = [];
+    constructor(url, outcome, config) {
+        super(outcome, config, 'interest');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'interest', isIInterestConfig)) {
+            commonsLogError('Invalid config for InterestParser');
+        }
+        this.model = config['interest'].model;
+        this.permittedDomains = config['interest'].permittedDomains;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (!this.ollamaUrl || !this.ollamaKey || !this.model)
+            return;
+        const hostname = new URL(this.url).hostname;
+        if (!this.permittedDomains.includes(hostname))
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'interest');
+            return;
+        }
+        const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
+        await this.debugGetLength();
+        // run this outside of await so that it doesn't block other things
+        void (async () => {
+            if (!this.url)
+                return;
+            const score = await rigService.generateInterestScore(text, this.ollamaKeepAlive);
+            if (score === undefined)
+                return;
+            if (score.score < 2)
+                delete score.reason;
+            commonsLogDebug(`Interest score is: ${score.score.toString(10)}`);
+            if (score.reason)
+                commonsLogDebug(`Reason for score is: ${score.reason}`);
+            await database.setData(this.url, 'interest', score);
+        })();
+    }
+}
+//# sourceMappingURL=interest.parser.mjs.map

package/dist/parsers/interest.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"interest.parser.mjs","sourceRoot":"","sources":["../../src/parsers/interest.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE/I,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGvE,OAAO,EAAE,2BAA2B,EAAuB,MAAM,gDAAgD,CAAC;AAKlH,OAAO,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAM/D,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC9C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,cAAe,SAAQ,YAA2B;IAKpD;IAJF,KAAK,CAAmB;IACxB,gBAAgB,GAAa,EAAE,CAAC;IAExC,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,UAAU,CACX,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAUrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAkB,MAAM,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE,CAAC;YACtF,eAAe,CAAC,mCAAmC,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,gBAAgB,CAAC;IAClF,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO;QAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAC/C,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;QAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,kEAAkE;QAClE,KAAK,CAAC,KAAK,IAAmB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,qBAAqB,CACjF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;YACF,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO;YAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;YAEzC,eAAe,CAAC,sBAAsB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAClE,IAAI,KAAK,CAAC,MAAM;gBAAE,eAAe,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAE1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;CACD"}

package/dist/parsers/jpeg.parser.d.mts

@@ -0,0 +1,11 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { DataParser, IDataConfig } from './data.parser.mjs';
+export declare class JpegParser extends DataParser<IDataConfig> {
+    private url?;
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
+    supports(contentType: string, isAllow: boolean): boolean;
+    parse(database: DatabaseService): Promise<void>;
+}

package/dist/parsers/jpeg.parser.mjs

@@ -0,0 +1,29 @@
+import * as jpeg from 'jpeg-js';
+import { DataParser } from './data.parser.mjs';
+export class JpegParser extends DataParser {
+    url;
+    constructor(url, outcome, config) {
+        super(outcome, config, 'jpeg');
+        this.url = url;
+    }
+    supports(contentType, isAllow) {
+        if (!isAllow)
+            return false;
+        return contentType.match(/^image\/(jpeg|jpg)$/) ? true : false;
+    }
+    async parse(database) {
+        if (!this.outcome || !this.url)
+            return;
+        if (!this.outcome.data)
+            return;
+        if (this.outcome.exceeded)
+            return; // don't try and parse vast JPEGs than haven't been downloaded properly, regardless of what the IDataConfig.allowExceeded is set to
+        try {
+            const rawImageData = jpeg.decode(this.outcome.data);
+            await database.setData(this.url, 'width', rawImageData.width);
+            await database.setData(this.url, 'height', rawImageData.height);
+        }
+        catch (_ex) { /* do nothing */ }
+    }
+}
+//# sourceMappingURL=jpeg.parser.mjs.map

package/dist/parsers/jpeg.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"jpeg.parser.mjs","sourceRoot":"","sources":["../../src/parsers/jpeg.parser.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAShC,OAAO,EAAE,UAAU,EAAe,MAAM,mBAAmB,CAAC;AAE5D,MAAM,OAAO,UAAW,SAAQ,UAAuB;IAE5C;IADV,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAJtB,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAChE,CAAC;IAEe,KAAK,CAAC,KAAK,CAAC,QAAyB;QACpD,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEvC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ;YAAE,OAAO,CAAC,mIAAmI;QAEtK,IAAI,CAAC;YACJ,MAAM,YAAY,GAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAEpE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACnC,CAAC;CACD"}

package/dist/parsers/{llama-guard.parser.d.ts → llama-guard.parser.d.mts}

@@ -1,9 +1,9 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IOllamaConfig, OllamaParser } from './ollama.parser';
-import { IDataConfig } from './data.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IOllamaConfig, OllamaParser } from './ollama.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface ILlamaGuardConfig extends IDataConfig {
     model: string;
     permittedDomains: string[];
@@ -13,6 +13,6 @@ export declare class LlamaGuardParser extends OllamaParser<IOllamaConfig> {
     private url?;
     private model;
     private permittedDomains;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseText(database: DatabaseService, text: string): Promise<void>;
 }

package/dist/parsers/llama-guard.parser.mjs

@@ -0,0 +1,56 @@
+import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { ELlamaGuardClassification, RigLlamaGuardService } from '../services/rig-llama-guard.service.mjs';
+import { OllamaParser } from './ollama.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isILlamaGuardConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'model'))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
+        return false;
+    return true;
+}
+export class LlamaGuardParser extends OllamaParser {
+    url;
+    model;
+    permittedDomains = [];
+    constructor(url, outcome, config) {
+        super(outcome, config, 'llamaguard');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'llamaguard', isILlamaGuardConfig)) {
+            commonsLogError('Invalid config for LlamaGuardParser');
+        }
+        this.model = config['llamaguard'].model;
+        this.permittedDomains = config['llamaguard'].permittedDomains;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (!this.ollamaUrl || !this.ollamaKey || !this.model)
+            return;
+        const hostname = new URL(this.url).hostname;
+        if (!this.permittedDomains.includes(hostname))
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'llamaguard');
+            return;
+        }
+        const rigService = new RigLlamaGuardService(this.ollamaUrl, this.ollamaKey, this.model);
+        await this.debugGetLength();
+        // run this outside of await so that it doesn't block other things
+        void (async () => {
+            if (!this.url)
+                return;
+            const result = await rigService.generate(text, this.ollamaKeepAlive);
+            if (result === undefined)
+                return;
+            commonsLogDebug(`LlamaGuard result is: ${result.classification.toString()}${result.classification === ELlamaGuardClassification.UNSAFE ? ` (${result.categories.join(',')})` : ''}`);
+            await database.setData(this.url, 'llamaguard', result);
+        })();
+    }
+}
+//# sourceMappingURL=llama-guard.parser.mjs.map

package/dist/parsers/llama-guard.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-guard.parser.mjs","sourceRoot":"","sources":["../../src/parsers/llama-guard.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE/I,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGvE,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAqB,MAAM,yCAAyC,CAAC;AAK7H,OAAO,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAM/D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAA2B;IAKtD;IAJF,KAAK,CAAmB;IACxB,gBAAgB,GAAa,EAAE,CAAC;IAExC,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAUrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,YAAY,EAAE,mBAAmB,CAAC,EAAE,CAAC;YAC5F,eAAe,CAAC,qCAAqC,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,KAAK,CAAC;QAC/D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,gBAAgB,CAAC;IACtF,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO;QAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YACjD,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAyB,IAAI,oBAAoB,CAC/D,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;QAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,kEAAkE;QAClE,KAAK,CAAC,KAAK,IAAmB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,MAAM,MAAM,GAAgC,MAAM,UAAU,CAAC,QAAQ,CACnE,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;YACF,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO;YAEjC,eAAe,CAAC,yBAAyB,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,cAAc,KAAK,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAErL,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;CACD"}

package/dist/parsers/{offence.parser.d.ts → offence.parser.d.mts}

@@ -1,9 +1,9 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IOllamaConfig, OllamaParser } from './ollama.parser';
-import { IDataConfig } from './data.parser';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IOllamaConfig, OllamaParser } from './ollama.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IOffenceConfig extends IDataConfig {
     model: string;
     permittedDomains: string[];
@@ -13,6 +13,6 @@ export declare class OffenceParser extends OllamaParser<IOllamaConfig> {
     private url?;
     private model;
     private permittedDomains;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
+    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>);
     protected parseText(database: DatabaseService, text: string): Promise<void>;
 }

package/dist/parsers/offence.parser.mjs

@@ -0,0 +1,60 @@
+import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service.mjs';
+import { OllamaParser } from './ollama.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIOffenceConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'model'))
+        return false;
+    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
+        return false;
+    return true;
+}
+export class OffenceParser extends OllamaParser {
+    url;
+    model;
+    permittedDomains = [];
+    constructor(url, outcome, config) {
+        super(outcome, config, 'offence');
+        this.url = url;
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'offence', isIOffenceConfig)) {
+            commonsLogError('Invalid config for OffenceParser');
+        }
+        this.model = config['offence'].model;
+        this.permittedDomains = config['offence'].permittedDomains;
+    }
+    async parseText(database, text) {
+        if (!this.url)
+            return;
+        if (!this.ollamaUrl || !this.ollamaKey || !this.model)
+            return;
+        const hostname = new URL(this.url).hostname;
+        if (!this.permittedDomains.includes(hostname))
+            return;
+        if (text.length === 0) {
+            await database.unsetData(this.url, 'offence');
+            return;
+        }
+        const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
+        await this.debugGetLength();
+        // run this outside of await so that it doesn't block other things
+        void (async () => {
+            if (!this.url)
+                return;
+            const score = await rigService.generateOffenceScore(text, this.ollamaKeepAlive);
+            if (score === undefined)
+                return;
+            if (score.score < 1)
+                delete score.reason;
+            commonsLogDebug(`Offence score is: ${score.score.toString(10)}`);
+            if (score.reason)
+                commonsLogDebug(`Reason for score is: ${score.reason}`);
+            await database.setData(this.url, 'offence', score);
+        })();
+    }
+}
+//# sourceMappingURL=offence.parser.mjs.map

package/dist/parsers/offence.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"offence.parser.mjs","sourceRoot":"","sources":["../../src/parsers/offence.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE/I,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGvE,OAAO,EAAE,2BAA2B,EAAuB,MAAM,gDAAgD,CAAC;AAKlH,OAAO,EAAiB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAClE,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAM/D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAKnD;IAJF,KAAK,CAAmB;IACxB,gBAAgB,GAAa,EAAE,CAAC;IAExC,YACU,GAAY,EACpB,OAAyB,EACzB,MAAuC;QAExC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAUrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACnF,eAAe,CAAC,kCAAkC,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,gBAAgB,CAAC;IAChF,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,QAAyB,EAAE,IAAY;QAChE,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO;QAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAC9C,OAAO;QACR,CAAC;QAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;QAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,kEAAkE;QAClE,KAAK,CAAC,KAAK,IAAmB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,oBAAoB,CAChF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;YACF,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO;YAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;YAEzC,eAAe,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACjE,IAAI,KAAK,CAAC,MAAM;gBAAE,eAAe,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAE1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACpD,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;CACD"}

package/dist/parsers/{ollama.parser.d.ts → ollama.parser.d.mts}

@@ -1,8 +1,8 @@
-import { CommonsFixedDuration, TKeyObject } from 'tscommons-es-core';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { TextParser } from './text.parser';
-import { IDataConfig } from './data.parser';
+import { CommonsFixedDuration, TPropertyObject } from 'tscommons-esm-core';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { TextParser } from './text.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IOllamaConfig extends IDataConfig {
     url: string;
     key: string;
@@ -13,6 +13,6 @@ export declare abstract class OllamaParser<T extends IOllamaConfig> extends Text
     protected readonly ollamaUrl: string | undefined;
     protected readonly ollamaKey: string | undefined;
     protected readonly ollamaKeepAlive: CommonsFixedDuration | undefined;
-    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
+    constructor(outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>, configKey?: string);
     debugGetLength(): Promise<void>;
 }

package/dist/parsers/ollama.parser.mjs

@@ -0,0 +1,43 @@
+import { CommonsFixedDuration, commonsTypeHasPropertyNumberOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
+import { RigQueueLengthService } from '../services/rig-queue-length.service.mjs';
+import { TextParser } from './text.parser.mjs';
+import { isIDataConfig } from './data.parser.mjs';
+export function isIOllamaConfig(test) {
+    if (!isIDataConfig(test))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'url'))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'key'))
+        return false;
+    if (!commonsTypeHasPropertyNumberOrUndefined(test, 'keepAlive'))
+        return false;
+    return true;
+}
+export class OllamaParser extends TextParser {
+    ollamaUrl;
+    ollamaKey;
+    ollamaKeepAlive;
+    constructor(outcome, config, configKey) {
+        super(outcome, config, configKey);
+        if (!config)
+            return;
+        if (!commonsTypeHasPropertyT(config, 'ollama', isIOllamaConfig)) {
+            commonsLogError('Invalid config for OllamaParser');
+        }
+        const ollamaConfig = config['ollama'];
+        this.ollamaUrl = ollamaConfig.url;
+        this.ollamaKey = ollamaConfig.key;
+        if (ollamaConfig.keepAlive) {
+            this.ollamaKeepAlive = CommonsFixedDuration.fromMillis(ollamaConfig.keepAlive);
+        }
+    }
+    async debugGetLength() {
+        if (!this.ollamaUrl || !this.ollamaKey)
+            return;
+        const rigService = new RigQueueLengthService(this.ollamaUrl, this.ollamaKey);
+        const length = await rigService.getLength();
+        commonsLogDebug(`Rig queue length is: ${length.active} active, ${length.queue} queued: ${length.line} total line length.`);
+    }
+}
+//# sourceMappingURL=ollama.parser.mjs.map

package/dist/parsers/ollama.parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama.parser.mjs","sourceRoot":"","sources":["../../src/parsers/ollama.parser.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,uCAAuC,EAAE,4BAA4B,EAAE,uBAAuB,EAAmB,MAAM,oBAAoB,CAAC;AAE3K,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,0CAA0C,CAAC;AAOjF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAe,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAO/D,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,YAAsC,SAAQ,UAAa;IAC7D,SAAS,CAAmB;IAC5B,SAAS,CAAmB;IAC5B,eAAe,CAAiC;IAEnE,YACE,OAAyB,EACzB,MAAuC,EACvC,SAAkB;QAEnB,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QAEF,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAgB,MAAM,EAAE,QAAQ,EAAE,eAAe,CAAC,EAAE,CAAC;YAChF,eAAe,CAAC,iCAAiC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,YAAY,GAAkB,MAAM,CAAC,QAAQ,CAAkB,CAAC;QAEtE,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAElC,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;YAC5B,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAChF,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,cAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE/C,MAAM,UAAU,GAA0B,IAAI,qBAAqB,CACjE,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,CACf,CAAC;QAEF,MAAM,MAAM,GAAiB,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1D,eAAe,CAAC,wBAAwB,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,IAAI,qBAAqB,CAAC,CAAC;IAC5H,CAAC;CACD"}

package/dist/parsers/{paragraphs.parser.d.ts → paragraphs.parser.d.mts}

@@ -1,13 +1,13 @@
-/// <reference types="cheerio" />
-import { DatabaseService } from '../services/database.service';
-import { HtmlParser } from './html.parser';
-import { IDataConfig } from './data.parser';
+import * as cheerio from 'cheerio';
+import { DatabaseService } from '../services/database.service.mjs';
+import { HtmlParser } from './html.parser.mjs';
+import { IDataConfig } from './data.parser.mjs';
 export interface IParagraphsConfig extends IDataConfig {
     paragraphWordsThreshold?: number;
 }
 export declare function isIParagraphsConfig(test: unknown): test is IParagraphsConfig;
 export declare abstract class ParagraphsParser<T extends IParagraphsConfig> extends HtmlParser<T> {
-    static parseParagraphs(dom: cheerio.Root, paragraphsConfig: IParagraphsConfig): string[];
+    static parseParagraphs(dom: cheerio.CheerioAPI, paragraphsConfig: IParagraphsConfig): string[];
     protected abstract parseParagraphs(database: DatabaseService, paragraphs: string[]): Promise<void>;
     parse(database: DatabaseService): Promise<void>;
 }