hydra-crawler 2.8.4 → 3.0.2

package/dist/parsers/hyperlinks.parser.js

@@ -1,115 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { URL } from 'url';
-import { commonsArrayUnique, commonsTypeHasPropertyStringArray } from 'tscommons-es-core';
-import { HtmlParser } from './html.parser';
-import { isIDataConfig } from './data.parser';
-export function isIHyperlinksConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'sources'))
-        return false;
-    return true;
-}
-export class HyperlinksParser extends HtmlParser {
-    static parseTagLinks(dom, tag, attribute) {
-        const links = [];
-        dom(`${tag}[${attribute}]`).each((_index, element) => {
-            const link = dom(element).attr(attribute);
-            if (link !== undefined)
-                links.push(link);
-            return true;
-        });
-        return commonsArrayUnique(links);
-    }
-    static getBaseHref(dom, url) {
-        try {
-            const base = dom('base[href]');
-            if (base.length === 0)
-                return undefined;
-            const href = dom(base[0]).attr('href');
-            if (!href)
-                return undefined;
-            const whatwg = new URL(href, url);
-            return whatwg.toString();
-        }
-        catch (ex) { /* do nothing */ }
-        return undefined;
-    }
-    constructor(url, outcome, config) {
-        super(outcome, config, 'hyperlinks');
-        this.url = url;
-        this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
-    }
-    parse(database) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.dom || !this.url || !this.hyperlinksConfig)
-                return; // invalid parse
-            let title;
-            try {
-                title = this.dom('title').text();
-            }
-            catch (ex) { /* do nothing */ }
-            if (title !== undefined && title !== null && title !== '') {
-                yield database.setData(this.url, 'title', title);
-            }
-            const links = {};
-            for (const source of this.hyperlinksConfig.sources) {
-                const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
-                if (pattern)
-                    links[pattern[1]] = HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2]);
-            }
-            yield database.setData(this.url, 'links', links);
-        });
-    }
-    // eslint-disable-next-line @typescript-eslint/require-await
-    links() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.dom || !this.url || !this.hyperlinksConfig)
-                return []; // invalid parse
-            let links = [];
-            for (const source of this.hyperlinksConfig.sources) {
-                const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
-                if (pattern) {
-                    if (pattern[1] === 'form')
-                        continue;
-                    links = [...links, ...HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2])];
-                }
-            }
-            const unique = [...new Set(links)];
-            let baseHref = HyperlinksParser.getBaseHref(this.dom, this.url);
-            if (baseHref === undefined)
-                baseHref = this.url;
-            const results = [];
-            for (let l of unique) {
-                const anchor = l.split('#');
-                l = anchor[0];
-                if (l === '')
-                    continue;
-                try {
-                    const u = new URL(l, baseHref);
-                    if (u.toString().indexOf('#') !== -1) {
-                        process.exit(1);
-                    }
-                    if (!u.protocol.match(/^http(s?):$/))
-                        continue;
-                    if (u.toString() === this.url || u.toString() === baseHref) {
-                        // skip self-links, or presumed self-links
-                        continue;
-                    }
-                    results.push(u.toString());
-                }
-                catch (ex) { /* do nothing */ }
-            }
-            return results;
-        });
-    }
-}
-//# sourceMappingURL=hyperlinks.parser.js.map

package/dist/parsers/hyperlinks.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"hyperlinks.parser.js","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAOtG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;IAE3D,MAAM,CAAC,aAAa,CACzB,GAAiB,EACjB,GAAW,EACX,SAAiB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACtF,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5D,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,GAAiB,EAAE,GAAW;QACvD,IAAI;YACH,MAAM,IAAI,GAAoB,GAAG,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAExC,MAAM,IAAI,GAAqB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAE5B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;SACzB;QAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QACjC,OAAO,SAAS,CAAC;IAClB,CAAC;IAID,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAJ5B,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAC7D,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,CAAC,gBAAgB;YAE9E,IAAI,KAAuB,CAAC;YAC5B,IAAI;gBACH,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACjC;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;YAEjC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE;gBAC1D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;aACjD;YAED,MAAM,KAAK,GAAkC,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO;oBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;aAClG;YAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;KAAA;IAED,4DAA4D;IAC/C,KAAK;;YACjB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,EAAE,CAAC,CAAC,gBAAgB;YAEjF,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO,EAAE;oBACZ,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;wBAAE,SAAS;oBACpC,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxF;aACD;YAED,MAAM,MAAM,GAAa,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAE7C,IAAI,QAAQ,GAAqB,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAClF,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,IAAI,MAAM,EAAE;gBACrB,MAAM,MAAM,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,KAAK,EAAE;oBAAE,SAAS;gBAEvB,IAAI;oBACH,MAAM,CAAC,GAAQ,IAAI,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;wBACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;qBAChB;oBACD,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;wBAAE,SAAS;oBAC/C,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,QAAQ,EAAE;wBAC3D,0CAA0C;wBAC1C,SAAS;qBACT;oBAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;iBAC3B;gBAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;aACjC;YAED,OAAO,OAAO,CAAC;QAChB,CAAC;KAAA;CACD"}

package/dist/parsers/image-tags.parser.d.ts

@@ -1,19 +0,0 @@
-/// <reference types="cheerio" />
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { HtmlParser } from './html.parser';
-import { IDataConfig } from './data.parser';
-type TImageTag = {
-    src: string;
-    alt?: string;
-    title?: string;
-};
-export declare class ImageTagsParser extends HtmlParser<IDataConfig> {
-    private url?;
-    static parseImageTags(dom: cheerio.Root): TImageTag[];
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
-    parse(database: DatabaseService): Promise<void>;
-}
-export {};

package/dist/parsers/image-tags.parser.js

@@ -1,52 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsOutputDebug } from 'nodecommons-es-cli';
-import { HtmlParser } from './html.parser';
-export class ImageTagsParser extends HtmlParser {
-    static parseImageTags(dom) {
-        const images = [];
-        dom('img').each((_index, element) => {
-            const src = dom(element).attr('src');
-            if (src === null || src === undefined)
-                return true;
-            const alt = dom(element).attr('alt');
-            const title = dom(element).attr('title');
-            const image = {
-                src: src
-            };
-            if (alt !== undefined && alt !== null)
-                image.alt = alt;
-            if (title !== undefined && title !== null)
-                image.title = title;
-            images.push(image);
-            return true;
-        });
-        return images;
-    }
-    constructor(url, outcome, config) {
-        super(outcome, config, 'imageTags');
-        this.url = url;
-    }
-    parse(database) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.dom || !this.url)
-                return; // invalid parse
-            const images = ImageTagsParser.parseImageTags(this.dom);
-            if (images.length > 0) {
-                commonsOutputDebug(`Detected images for ${this.url}`);
-                yield database.setData(this.url, 'images', images);
-            }
-            else {
-                yield database.unsetData(this.url, 'images');
-            }
-        });
-    }
-}
-//# sourceMappingURL=image-tags.parser.js.map

package/dist/parsers/image-tags.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"image-tags.parser.js","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAS3C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;IACpD,MAAM,CAAC,cAAc,CAC1B,GAAiB;QAElB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACrE,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEnD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEhE,MAAM,KAAK,GAAc;gBACvB,GAAG,EAAE,GAAG;aACT,CAAC;YACF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAE/D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAJ3B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gBAAgB;YAEpD,MAAM,MAAM,GAAgB,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACtB,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACtD,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;aACnD;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;aAC7C;QACF,CAAC;KAAA;CACD"}

package/dist/parsers/interest.parser.js

@@ -1,69 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service';
-import { OllamaParser } from './ollama.parser';
-import { isIDataConfig } from './data.parser';
-export function isIInterestConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'model'))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
-        return false;
-    return true;
-}
-export class InterestParser extends OllamaParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'interest');
-        this.url = url;
-        this.permittedDomains = [];
-        if (!config)
-            return;
-        if (!commonsTypeHasPropertyT(config, 'interest', isIInterestConfig)) {
-            commonsOutputError('Invalid config for InterestParser');
-        }
-        this.model = config['interest'].model;
-        this.permittedDomains = config['interest'].permittedDomains;
-    }
-    parseText(database, text) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.url)
-                return;
-            if (!this.ollamaUrl || !this.ollamaKey || !this.model)
-                return;
-            const hostname = new URL(this.url).hostname;
-            if (!this.permittedDomains.includes(hostname))
-                return;
-            if (text.length === 0) {
-                yield database.unsetData(this.url, 'interest');
-                return;
-            }
-            const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
-            yield this.debugGetLength();
-            // run this outside of await so that it doesn't block other things
-            void (() => __awaiter(this, void 0, void 0, function* () {
-                if (!this.url)
-                    return;
-                const score = yield rigService.generateInterestScore(text, this.ollamaKeepAlive);
-                if (score === undefined)
-                    return;
-                if (score.score < 2)
-                    delete score.reason;
-                commonsOutputDebug(`Interest score is: ${score.score.toString(10)}`);
-                if (score.reason)
-                    commonsOutputDebug(`Reason for score is: ${score.reason}`);
-                yield database.setData(this.url, 'interest', score);
-            }))();
-        });
-    }
-}
-//# sourceMappingURL=interest.parser.js.map

package/dist/parsers/interest.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"interest.parser.js","sourceRoot":"","sources":["../../src/parsers/interest.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAuB,MAAM,4CAA4C,CAAC;AAK9G,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC9C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,cAAe,SAAQ,YAA2B;IAI9D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,UAAU,CACX,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAkB,MAAM,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE;YACrF,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;SACxD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,gBAAgB,CAAC;IAClF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC/C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAE5B,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,qBAAqB,CACjF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;gBAEzC,kBAAkB,CAAC,sBAAsB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,CAAC,MAAM;oBAAE,kBAAkB,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE7E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}

package/dist/parsers/jpeg.parser.d.ts

@@ -1,11 +0,0 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { DataParser, IDataConfig } from './data.parser';
-export declare class JpegParser extends DataParser<IDataConfig> {
-    private url?;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
-    supports(contentType: string, isAllow: boolean): boolean;
-    parse(database: DatabaseService): Promise<void>;
-}

package/dist/parsers/jpeg.parser.js

@@ -1,39 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import * as jpeg from 'jpeg-js';
-import { DataParser } from './data.parser';
-export class JpegParser extends DataParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'jpeg');
-        this.url = url;
-    }
-    supports(contentType, isAllow) {
-        if (!isAllow)
-            return false;
-        return contentType.match(/^image\/(jpeg|jpg)$/) ? true : false;
-    }
-    parse(database) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.outcome || !this.url)
-                return;
-            if (!this.outcome.data)
-                return;
-            if (this.outcome.exceeded)
-                return; // don't try and parse vast JPEGs than haven't been downloaded properly, regardless of what the IDataConfig.allowExceeded is set to
-            try {
-                const rawImageData = jpeg.decode(this.outcome.data);
-                yield database.setData(this.url, 'width', rawImageData.width);
-                yield database.setData(this.url, 'height', rawImageData.height);
-            }
-            catch (ex) { /* do nothing */ }
-        });
-    }
-}
-//# sourceMappingURL=jpeg.parser.js.map

package/dist/parsers/jpeg.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"jpeg.parser.js","sourceRoot":"","sources":["../../src/parsers/jpeg.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAShC,OAAO,EAAE,UAAU,EAAe,MAAM,eAAe,CAAC;AAExD,MAAM,OAAO,UAAW,SAAQ,UAAuB;IACtD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAJtB,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAChE,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEvC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI;gBAAE,OAAO;YAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,OAAO,CAAC,mIAAmI;YAEtK,IAAI;gBACH,MAAM,YAAY,GAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAEpE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;gBAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;aAChE;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QAClC,CAAC;KAAA;CACD"}

package/dist/parsers/llama-guard.parser.js

@@ -1,65 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { ELlamaGuardClassification, RigLlamaGuardService } from '../services/rig-llama-guard.service';
-import { OllamaParser } from './ollama.parser';
-import { isIDataConfig } from './data.parser';
-export function isILlamaGuardConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'model'))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
-        return false;
-    return true;
-}
-export class LlamaGuardParser extends OllamaParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'llamaguard');
-        this.url = url;
-        this.permittedDomains = [];
-        if (!config)
-            return;
-        if (!commonsTypeHasPropertyT(config, 'llamaguard', isILlamaGuardConfig)) {
-            commonsOutputError('Invalid config for LlamaGuardParser');
-        }
-        this.model = config['llamaguard'].model;
-        this.permittedDomains = config['llamaguard'].permittedDomains;
-    }
-    parseText(database, text) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.url)
-                return;
-            if (!this.ollamaUrl || !this.ollamaKey || !this.model)
-                return;
-            const hostname = new URL(this.url).hostname;
-            if (!this.permittedDomains.includes(hostname))
-                return;
-            if (text.length === 0) {
-                yield database.unsetData(this.url, 'llamaguard');
-                return;
-            }
-            const rigService = new RigLlamaGuardService(this.ollamaUrl, this.ollamaKey, this.model);
-            yield this.debugGetLength();
-            // run this outside of await so that it doesn't block other things
-            void (() => __awaiter(this, void 0, void 0, function* () {
-                if (!this.url)
-                    return;
-                const result = yield rigService.generate(text, this.ollamaKeepAlive);
-                if (result === undefined)
-                    return;
-                commonsOutputDebug(`LlamaGuard result is: ${result.classification.toString()}${result.classification === ELlamaGuardClassification.UNSAFE ? ` (${result.categories.join(',')})` : ''}`);
-                yield database.setData(this.url, 'llamaguard', result);
-            }))();
-        });
-    }
-}
-//# sourceMappingURL=llama-guard.parser.js.map

package/dist/parsers/llama-guard.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"llama-guard.parser.js","sourceRoot":"","sources":["../../src/parsers/llama-guard.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAqB,MAAM,qCAAqC,CAAC;AAKzH,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAA2B;IAIhE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,YAAY,EAAE,mBAAmB,CAAC,EAAE;YAC3F,kBAAkB,CAAC,qCAAqC,CAAC,CAAC;SAC1D;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,KAAK,CAAC;QAC/D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,gBAAgB,CAAC;IACtF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;gBACjD,OAAO;aACP;YAED,MAAM,UAAU,GAAyB,IAAI,oBAAoB,CAC/D,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAE5B,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,MAAM,GAAgC,MAAM,UAAU,CAAC,QAAQ,CACnE,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,MAAM,KAAK,SAAS;oBAAE,OAAO;gBAEjC,kBAAkB,CAAC,yBAAyB,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,cAAc,KAAK,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAExL,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;YACxD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}

package/dist/parsers/offence.parser.js

@@ -1,69 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service';
-import { OllamaParser } from './ollama.parser';
-import { isIDataConfig } from './data.parser';
-export function isIOffenceConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'model'))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
-        return false;
-    return true;
-}
-export class OffenceParser extends OllamaParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'offence');
-        this.url = url;
-        this.permittedDomains = [];
-        if (!config)
-            return;
-        if (!commonsTypeHasPropertyT(config, 'offence', isIOffenceConfig)) {
-            commonsOutputError('Invalid config for OffenceParser');
-        }
-        this.model = config['offence'].model;
-        this.permittedDomains = config['offence'].permittedDomains;
-    }
-    parseText(database, text) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.url)
-                return;
-            if (!this.ollamaUrl || !this.ollamaKey || !this.model)
-                return;
-            const hostname = new URL(this.url).hostname;
-            if (!this.permittedDomains.includes(hostname))
-                return;
-            if (text.length === 0) {
-                yield database.unsetData(this.url, 'offence');
-                return;
-            }
-            const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
-            yield this.debugGetLength();
-            // run this outside of await so that it doesn't block other things
-            void (() => __awaiter(this, void 0, void 0, function* () {
-                if (!this.url)
-                    return;
-                const score = yield rigService.generateOffenceScore(text, this.ollamaKeepAlive);
-                if (score === undefined)
-                    return;
-                if (score.score < 1)
-                    delete score.reason;
-                commonsOutputDebug(`Offence score is: ${score.score.toString(10)}`);
-                if (score.reason)
-                    commonsOutputDebug(`Reason for score is: ${score.reason}`);
-                yield database.setData(this.url, 'offence', score);
-            }))();
-        });
-    }
-}
-//# sourceMappingURL=offence.parser.js.map

package/dist/parsers/offence.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"offence.parser.js","sourceRoot":"","sources":["../../src/parsers/offence.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAuB,MAAM,4CAA4C,CAAC;AAK9G,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAI7D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE;YAClF,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;SACvD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,gBAAgB,CAAC;IAChF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC9C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAE5B,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,oBAAoB,CAChF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;gBAEzC,kBAAkB,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACpE,IAAI,KAAK,CAAC,MAAM;oBAAE,kBAAkB,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE7E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}

package/dist/parsers/ollama.parser.js

@@ -1,51 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { CommonsFixedDuration, commonsTypeHasPropertyNumberOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { RigQueueLengthService } from '../services/rig-queue-length.service';
-import { TextParser } from './text.parser';
-import { isIDataConfig } from './data.parser';
-export function isIOllamaConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'url'))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'key'))
-        return false;
-    if (!commonsTypeHasPropertyNumberOrUndefined(test, 'keepAlive'))
-        return false;
-    return true;
-}
-export class OllamaParser extends TextParser {
-    constructor(outcome, config, configKey) {
-        super(outcome, config, configKey);
-        if (!config)
-            return;
-        if (!commonsTypeHasPropertyT(config, 'ollama', isIOllamaConfig)) {
-            commonsOutputError('Invalid config for OllamaParser');
-        }
-        const ollamaConfig = config['ollama'];
-        this.ollamaUrl = ollamaConfig.url;
-        this.ollamaKey = ollamaConfig.key;
-        if (ollamaConfig.keepAlive) {
-            this.ollamaKeepAlive = CommonsFixedDuration.fromMillis(ollamaConfig.keepAlive);
-        }
-    }
-    debugGetLength() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.ollamaUrl || !this.ollamaKey)
-                return;
-            const rigService = new RigQueueLengthService(this.ollamaUrl, this.ollamaKey);
-            const length = yield rigService.getLength();
-            commonsOutputDebug(`Rig queue length is: ${length.active} active, ${length.queue} queued: ${length.line} total line length.`);
-        });
-    }
-}
-//# sourceMappingURL=ollama.parser.js.map

package/dist/parsers/ollama.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ollama.parser.js","sourceRoot":"","sources":["../../src/parsers/ollama.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,oBAAoB,EAAE,uCAAuC,EAAE,4BAA4B,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAErK,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE5E,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAO7E,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAO3D,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,YAAsC,SAAQ,UAAa;IAKhF,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QAEF,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAgB,MAAM,EAAE,QAAQ,EAAE,eAAe,CAAC,EAAE;YAC/E,kBAAkB,CAAC,iCAAiC,CAAC,CAAC;SACtD;QACD,MAAM,YAAY,GAAkB,MAAM,CAAC,QAAQ,CAAkB,CAAC;QAEtE,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAElC,IAAI,YAAY,CAAC,SAAS,EAAE;YAC3B,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;SAC/E;IACF,CAAC;IAEY,cAAc;;YAC1B,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS;gBAAE,OAAO;YAE/C,MAAM,UAAU,GAA0B,IAAI,qBAAqB,CACjE,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,CACf,CAAC;YAEF,MAAM,MAAM,GAAiB,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;YAC1D,kBAAkB,CAAC,wBAAwB,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,IAAI,qBAAqB,CAAC,CAAC;QAC/H,CAAC;KAAA;CACD"}

package/dist/parsers/paragraphs.parser.js

@@ -1,49 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsStringSplitWords, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-es-core';
-import { HtmlParser } from './html.parser';
-import { isIDataConfig } from './data.parser';
-export function isIParagraphsConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyNumberOrUndefined(test, 'paragraphWordsThreshold'))
-        return false;
-    return true;
-}
-export class ParagraphsParser extends HtmlParser {
-    static parseParagraphs(dom, paragraphsConfig) {
-        const nodes = dom('p,dd')
-            .contents()
-            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-            .filter((_index, element) => element.nodeType === 3);
-        const paragraphs = [];
-        dom(nodes).each((_index, element) => {
-            const paragraph = dom(element).text();
-            if (paragraphsConfig.paragraphWordsThreshold && commonsStringSplitWords(paragraph).length >= paragraphsConfig.paragraphWordsThreshold) {
-                paragraphs.push(paragraph);
-            }
-        });
-        return paragraphs
-            .map((p) => p.trim())
-            .filter((p) => p !== '');
-    }
-    parse(database) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.dom)
-                return; // invalid parse
-            const paragraphsConfig = this.getConfig(isIParagraphsConfig);
-            if (!paragraphsConfig)
-                return;
-            const paragraphs = ParagraphsParser.parseParagraphs(this.dom, paragraphsConfig);
-            yield this.parseParagraphs(database, paragraphs);
-        });
-    }
-}
-//# sourceMappingURL=paragraphs.parser.js.map

package/dist/parsers/paragraphs.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"paragraphs.parser.js","sourceRoot":"","sources":["../../src/parsers/paragraphs.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,uCAAuC,EAAE,MAAM,mBAAmB,CAAC;AAIrG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,yBAAyB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,UAAa;IACjF,MAAM,CAAC,eAAe,CAC3B,GAAiB,EACjB,gBAAmC;QAEpC,MAAM,KAAK,GAAoB,GAAG,CAAC,MAAM,CAAC;aACvC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAElG,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CACb,CAAC,MAAc,EAAE,OAAwB,EAAQ,EAAE;YAClD,MAAM,SAAS,GAAW,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9C,IAAI,gBAAgB,CAAC,uBAAuB,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC,MAAM,IAAI,gBAAgB,CAAC,uBAAuB,EAAE;gBACtI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aAC3B;QACF,CAAC,CACF,CAAC;QAEF,OAAO,UAAU;aACd,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7C,CAAC;IAIY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gBAAgB;YAEvC,MAAM,gBAAgB,GAAgC,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC1F,IAAI,CAAC,gBAAgB;gBAAE,OAAO;YAE9B,MAAM,UAAU,GAAa,gBAAgB,CAAC,eAAe,CAC3D,IAAI,CAAC,GAAG,EACR,gBAAgB,CACjB,CAAC;YAEF,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAClD,CAAC;KAAA;CACD"}