hydra-crawler 2.8.3 → 3.0.1

package/dist/parsers/dictionary.parser.js

@@ -1,73 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT, commonsTypeIsStringArray } from 'tscommons-es-core';
-import { commonsOutputError } from 'nodecommons-es-cli';
-import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-es-file';
-import { WordsParser, isIWordsConfig } from './words.parser';
-export function isIDictionaryConfig(test) {
-    if (!isIWordsConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'dictionary'))
-        return false;
-    if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'caseSensitive'))
-        return false;
-    return true;
-}
-const CACHE = new Map();
-function loadDictionary(config) {
-    const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
-    const json = commonsFileReadJsonFile(filename);
-    if (json === undefined)
-        throw new Error('Unable to read dictionary file');
-    if (!commonsTypeIsStringArray(json))
-        throw new Error('Dictionary file is not a JSON array');
-    return json;
-}
-export class DictionaryParser extends WordsParser {
-    constructor(outcome, config, configKey) {
-        super(outcome, config, configKey);
-        this.caseSensitive = false;
-        this.dictionary = [];
-        if (!config || !configKey)
-            return;
-        this.cacheId = configKey;
-        if (!commonsTypeHasPropertyT(config, configKey, isIDictionaryConfig)) {
-            commonsOutputError(`Invalid config for DictionaryParser (${configKey})`);
-        }
-        const dictionaryConfig = config[configKey];
-        this.caseSensitive = dictionaryConfig.caseSensitive || false;
-        if (!CACHE.has(this.cacheId)) {
-            const loaded = loadDictionary(dictionaryConfig);
-            CACHE.set(this.cacheId, loaded);
-        }
-    }
-    getDictionary() {
-        if (!this.cacheId)
-            return [];
-        return CACHE.get(this.cacheId) || [];
-    }
-    parseWords(database, words) {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.dictionary = this.getDictionary()
-                .map((word) => this.caseSensitive ? word : word.toLowerCase());
-            const matches = [];
-            const nonMatches = [];
-            for (const word of words) {
-                const compare = this.caseSensitive ? word : word.toLowerCase();
-                if (this.dictionary.includes(compare))
-                    matches.push(word);
-                else
-                    nonMatches.push(word);
-            }
-            yield this.parseMatches(database, matches, nonMatches);
-        });
-    }
-}
-//# sourceMappingURL=dictionary.parser.js.map

package/dist/parsers/dictionary.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"dictionary.parser.js","sourceRoot":"","sources":["../../src/parsers/dictionary.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,wCAAwC,EAAE,4BAA4B,EAAE,uBAAuB,EAAE,wBAAwB,EAAc,MAAM,mBAAmB,CAAC;AAE1K,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAE,WAAW,EAAgB,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAM3E,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAExC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,YAAY,CAAC;QAAE,OAAO,KAAK,CAAC;IACpE,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,KAAK,GAA0B,IAAI,GAAG,EAAoB,CAAC;AAEjE,SAAS,cAAc,CAAC,MAAyB;IAChD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAsB,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAClE,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE1E,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAE5F,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,gBAA8C,SAAQ,WAAc;IAKzF,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QATzB,kBAAa,GAAY,KAAK,CAAC;QACjC,eAAU,GAAa,EAAE,CAAC;QAUjC,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS;YAAE,OAAO;QAElC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAEzB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,SAAS,EAAE,mBAAmB,CAAC,EAAE;YACxF,kBAAkB,CAAC,wCAAwC,SAAS,GAAG,CAAC,CAAC;SACzE;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,SAAS,CAAsB,CAAC;QAEnF,IAAI,CAAC,aAAa,GAAG,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC;QAE7D,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC7B,MAAM,MAAM,GAAa,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC1D,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;SAChC;IACF,CAAC;IAES,aAAa;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAE7B,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;IAIe,UAAU,CAAC,QAAyB,EAAE,KAAe;;YACpE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE;iBACnC,GAAG,CAAC,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAEjF,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAa,EAAE,CAAC;YAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACzB,MAAM,OAAO,GAAW,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACvE,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;oBACrD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC3B;YAED,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;CACD"}

package/dist/parsers/html.parser.d.ts

@@ -1,13 +0,0 @@
-/// <reference types="cheerio" />
-import { TKeyObject } from 'tscommons-es-core';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { StringParser } from './string.parser';
-import { IDataConfig } from './data.parser';
-export declare abstract class HtmlParser<T extends IDataConfig> extends StringParser<T> {
-    static collapseWhitespace(raw: string): string;
-    static loadDom(html: string): cheerio.Root | undefined;
-    protected dom: cheerio.Root | undefined;
-    constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
-    supports(contentType: string, isAllow: boolean): boolean;
-}

package/dist/parsers/html.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"html.parser.js","sourceRoot":"","sources":["../../src/parsers/html.parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,OAAgB,UAAkC,SAAQ,YAAe;IACvE,MAAM,CAAC,kBAAkB,CAAC,GAAW;QAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,IAAY;QACjC,IAAI;YACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC1B;QAAC,OAAO,EAAE,EAAE;YACZ,SAAS;YACT,OAAO,SAAS,CAAC;SACjB;IACF,CAAC;IAID,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAC9C,MAAM,SAAS,GAAW,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;SACzC;IACF,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,4BAA4B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;CACD"}

package/dist/parsers/hyperlinks.parser.d.ts

@@ -1,20 +0,0 @@
-/// <reference types="cheerio" />
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { HtmlParser } from './html.parser';
-import { IDataConfig } from './data.parser';
-export interface IHyperlinksConfig extends IDataConfig {
-    sources: string[];
-}
-export declare function isIHyperlinksConfig(test: unknown): test is IHyperlinksConfig;
-export declare class HyperlinksParser extends HtmlParser<IHyperlinksConfig> {
-    private url?;
-    static parseTagLinks(dom: cheerio.Root, tag: string, attribute: string): string[];
-    static getBaseHref(dom: cheerio.Root, url: string): string | undefined;
-    private hyperlinksConfig;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
-    parse(database: DatabaseService): Promise<void>;
-    links(): Promise<string[]>;
-}

package/dist/parsers/hyperlinks.parser.js

@@ -1,115 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { URL } from 'url';
-import { commonsArrayUnique, commonsTypeHasPropertyStringArray } from 'tscommons-es-core';
-import { HtmlParser } from './html.parser';
-import { isIDataConfig } from './data.parser';
-export function isIHyperlinksConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'sources'))
-        return false;
-    return true;
-}
-export class HyperlinksParser extends HtmlParser {
-    static parseTagLinks(dom, tag, attribute) {
-        const links = [];
-        dom(`${tag}[${attribute}]`).each((_index, element) => {
-            const link = dom(element).attr(attribute);
-            if (link !== undefined)
-                links.push(link);
-            return true;
-        });
-        return commonsArrayUnique(links);
-    }
-    static getBaseHref(dom, url) {
-        try {
-            const base = dom('base[href]');
-            if (base.length === 0)
-                return undefined;
-            const href = dom(base[0]).attr('href');
-            if (!href)
-                return undefined;
-            const whatwg = new URL(href, url);
-            return whatwg.toString();
-        }
-        catch (ex) { /* do nothing */ }
-        return undefined;
-    }
-    constructor(url, outcome, config) {
-        super(outcome, config, 'hyperlinks');
-        this.url = url;
-        this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
-    }
-    parse(database) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.dom || !this.url || !this.hyperlinksConfig)
-                return; // invalid parse
-            let title;
-            try {
-                title = this.dom('title').text();
-            }
-            catch (ex) { /* do nothing */ }
-            if (title !== undefined && title !== null && title !== '') {
-                yield database.setData(this.url, 'title', title);
-            }
-            const links = {};
-            for (const source of this.hyperlinksConfig.sources) {
-                const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
-                if (pattern)
-                    links[pattern[1]] = HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2]);
-            }
-            yield database.setData(this.url, 'links', links);
-        });
-    }
-    // eslint-disable-next-line @typescript-eslint/require-await
-    links() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.dom || !this.url || !this.hyperlinksConfig)
-                return []; // invalid parse
-            let links = [];
-            for (const source of this.hyperlinksConfig.sources) {
-                const pattern = source.match(/^([a-z]+)\[(src|href|action)\]$/);
-                if (pattern) {
-                    if (pattern[1] === 'form')
-                        continue;
-                    links = [...links, ...HyperlinksParser.parseTagLinks(this.dom, pattern[1], pattern[2])];
-                }
-            }
-            const unique = [...new Set(links)];
-            let baseHref = HyperlinksParser.getBaseHref(this.dom, this.url);
-            if (baseHref === undefined)
-                baseHref = this.url;
-            const results = [];
-            for (let l of unique) {
-                const anchor = l.split('#');
-                l = anchor[0];
-                if (l === '')
-                    continue;
-                try {
-                    const u = new URL(l, baseHref);
-                    if (u.toString().indexOf('#') !== -1) {
-                        process.exit(1);
-                    }
-                    if (!u.protocol.match(/^http(s?):$/))
-                        continue;
-                    if (u.toString() === this.url || u.toString() === baseHref) {
-                        // skip self-links, or presumed self-links
-                        continue;
-                    }
-                    results.push(u.toString());
-                }
-                catch (ex) { /* do nothing */ }
-            }
-            return results;
-        });
-    }
-}
-//# sourceMappingURL=hyperlinks.parser.js.map

package/dist/parsers/hyperlinks.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"hyperlinks.parser.js","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAOtG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;IAE3D,MAAM,CAAC,aAAa,CACzB,GAAiB,EACjB,GAAW,EACX,SAAiB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACtF,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5D,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,GAAiB,EAAE,GAAW;QACvD,IAAI;YACH,MAAM,IAAI,GAAoB,GAAG,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAExC,MAAM,IAAI,GAAqB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAE5B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;SACzB;QAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QACjC,OAAO,SAAS,CAAC;IAClB,CAAC;IAID,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAJ5B,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAC7D,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,CAAC,gBAAgB;YAE9E,IAAI,KAAuB,CAAC;YAC5B,IAAI;gBACH,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACjC;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;YAEjC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE;gBAC1D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;aACjD;YAED,MAAM,KAAK,GAAkC,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO;oBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;aAClG;YAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;KAAA;IAED,4DAA4D;IAC/C,KAAK;;YACjB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,EAAE,CAAC,CAAC,gBAAgB;YAEjF,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO,EAAE;oBACZ,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;wBAAE,SAAS;oBACpC,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxF;aACD;YAED,MAAM,MAAM,GAAa,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAE7C,IAAI,QAAQ,GAAqB,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAClF,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,IAAI,MAAM,EAAE;gBACrB,MAAM,MAAM,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,KAAK,EAAE;oBAAE,SAAS;gBAEvB,IAAI;oBACH,MAAM,CAAC,GAAQ,IAAI,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;wBACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;qBAChB;oBACD,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;wBAAE,SAAS;oBAC/C,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,QAAQ,EAAE;wBAC3D,0CAA0C;wBAC1C,SAAS;qBACT;oBAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;iBAC3B;gBAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;aACjC;YAED,OAAO,OAAO,CAAC;QAChB,CAAC;KAAA;CACD"}

package/dist/parsers/image-tags.parser.d.ts

@@ -1,19 +0,0 @@
-/// <reference types="cheerio" />
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { HtmlParser } from './html.parser';
-import { IDataConfig } from './data.parser';
-type TImageTag = {
-    src: string;
-    alt?: string;
-    title?: string;
-};
-export declare class ImageTagsParser extends HtmlParser<IDataConfig> {
-    private url?;
-    static parseImageTags(dom: cheerio.Root): TImageTag[];
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
-    parse(database: DatabaseService): Promise<void>;
-}
-export {};

package/dist/parsers/image-tags.parser.js

@@ -1,52 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsOutputDebug } from 'nodecommons-es-cli';
-import { HtmlParser } from './html.parser';
-export class ImageTagsParser extends HtmlParser {
-    static parseImageTags(dom) {
-        const images = [];
-        dom('img').each((_index, element) => {
-            const src = dom(element).attr('src');
-            if (src === null || src === undefined)
-                return true;
-            const alt = dom(element).attr('alt');
-            const title = dom(element).attr('title');
-            const image = {
-                src: src
-            };
-            if (alt !== undefined && alt !== null)
-                image.alt = alt;
-            if (title !== undefined && title !== null)
-                image.title = title;
-            images.push(image);
-            return true;
-        });
-        return images;
-    }
-    constructor(url, outcome, config) {
-        super(outcome, config, 'imageTags');
-        this.url = url;
-    }
-    parse(database) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.dom || !this.url)
-                return; // invalid parse
-            const images = ImageTagsParser.parseImageTags(this.dom);
-            if (images.length > 0) {
-                commonsOutputDebug(`Detected images for ${this.url}`);
-                yield database.setData(this.url, 'images', images);
-            }
-            else {
-                yield database.unsetData(this.url, 'images');
-            }
-        });
-    }
-}
-//# sourceMappingURL=image-tags.parser.js.map

package/dist/parsers/image-tags.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"image-tags.parser.js","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAS3C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;IACpD,MAAM,CAAC,cAAc,CAC1B,GAAiB;QAElB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACrE,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEnD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEhE,MAAM,KAAK,GAAc;gBACvB,GAAG,EAAE,GAAG;aACT,CAAC;YACF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAE/D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAJ3B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gBAAgB;YAEpD,MAAM,MAAM,GAAgB,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACtB,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACtD,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;aACnD;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;aAC7C;QACF,CAAC;KAAA;CACD"}

package/dist/parsers/interest.parser.js

@@ -1,69 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service';
-import { OllamaParser } from './ollama.parser';
-import { isIDataConfig } from './data.parser';
-export function isIInterestConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'model'))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
-        return false;
-    return true;
-}
-export class InterestParser extends OllamaParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'interest');
-        this.url = url;
-        this.permittedDomains = [];
-        if (!config)
-            return;
-        if (!commonsTypeHasPropertyT(config, 'interest', isIInterestConfig)) {
-            commonsOutputError('Invalid config for InterestParser');
-        }
-        this.model = config['interest'].model;
-        this.permittedDomains = config['interest'].permittedDomains;
-    }
-    parseText(database, text) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.url)
-                return;
-            if (!this.ollamaUrl || !this.ollamaKey || !this.model)
-                return;
-            const hostname = new URL(this.url).hostname;
-            if (!this.permittedDomains.includes(hostname))
-                return;
-            if (text.length === 0) {
-                yield database.unsetData(this.url, 'interest');
-                return;
-            }
-            const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
-            yield this.debugGetLength();
-            // run this outside of await so that it doesn't block other things
-            void (() => __awaiter(this, void 0, void 0, function* () {
-                if (!this.url)
-                    return;
-                const score = yield rigService.generateInterestScore(text, this.ollamaKeepAlive);
-                if (score === undefined)
-                    return;
-                if (score.score < 2)
-                    delete score.reason;
-                commonsOutputDebug(`Interest score is: ${score.score.toString(10)}`);
-                if (score.reason)
-                    commonsOutputDebug(`Reason for score is: ${score.reason}`);
-                yield database.setData(this.url, 'interest', score);
-            }))();
-        });
-    }
-}
-//# sourceMappingURL=interest.parser.js.map

package/dist/parsers/interest.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"interest.parser.js","sourceRoot":"","sources":["../../src/parsers/interest.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAuB,MAAM,4CAA4C,CAAC;AAK9G,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC9C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,cAAe,SAAQ,YAA2B;IAI9D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,UAAU,CACX,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAkB,MAAM,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE;YACrF,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;SACxD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,gBAAgB,CAAC;IAClF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC/C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAE5B,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,qBAAqB,CACjF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;gBAEzC,kBAAkB,CAAC,sBAAsB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,CAAC,MAAM;oBAAE,kBAAkB,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE7E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}

package/dist/parsers/jpeg.parser.d.ts

@@ -1,11 +0,0 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { DatabaseService } from '../services/database.service';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { DataParser, IDataConfig } from './data.parser';
-export declare class JpegParser extends DataParser<IDataConfig> {
-    private url?;
-    constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
-    supports(contentType: string, isAllow: boolean): boolean;
-    parse(database: DatabaseService): Promise<void>;
-}

package/dist/parsers/jpeg.parser.js

@@ -1,39 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import * as jpeg from 'jpeg-js';
-import { DataParser } from './data.parser';
-export class JpegParser extends DataParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'jpeg');
-        this.url = url;
-    }
-    supports(contentType, isAllow) {
-        if (!isAllow)
-            return false;
-        return contentType.match(/^image\/(jpeg|jpg)$/) ? true : false;
-    }
-    parse(database) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.outcome || !this.url)
-                return;
-            if (!this.outcome.data)
-                return;
-            if (this.outcome.exceeded)
-                return; // don't try and parse vast JPEGs than haven't been downloaded properly, regardless of what the IDataConfig.allowExceeded is set to
-            try {
-                const rawImageData = jpeg.decode(this.outcome.data);
-                yield database.setData(this.url, 'width', rawImageData.width);
-                yield database.setData(this.url, 'height', rawImageData.height);
-            }
-            catch (ex) { /* do nothing */ }
-        });
-    }
-}
-//# sourceMappingURL=jpeg.parser.js.map

package/dist/parsers/jpeg.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"jpeg.parser.js","sourceRoot":"","sources":["../../src/parsers/jpeg.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAShC,OAAO,EAAE,UAAU,EAAe,MAAM,eAAe,CAAC;AAExD,MAAM,OAAO,UAAW,SAAQ,UAAuB;IACtD,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAJtB,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAChE,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEvC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI;gBAAE,OAAO;YAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,OAAO,CAAC,mIAAmI;YAEtK,IAAI;gBACH,MAAM,YAAY,GAAmB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAEpE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;gBAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;aAChE;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QAClC,CAAC;KAAA;CACD"}

package/dist/parsers/llama-guard.parser.js

@@ -1,65 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { ELlamaGuardClassification, RigLlamaGuardService } from '../services/rig-llama-guard.service';
-import { OllamaParser } from './ollama.parser';
-import { isIDataConfig } from './data.parser';
-export function isILlamaGuardConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'model'))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
-        return false;
-    return true;
-}
-export class LlamaGuardParser extends OllamaParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'llamaguard');
-        this.url = url;
-        this.permittedDomains = [];
-        if (!config)
-            return;
-        if (!commonsTypeHasPropertyT(config, 'llamaguard', isILlamaGuardConfig)) {
-            commonsOutputError('Invalid config for LlamaGuardParser');
-        }
-        this.model = config['llamaguard'].model;
-        this.permittedDomains = config['llamaguard'].permittedDomains;
-    }
-    parseText(database, text) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.url)
-                return;
-            if (!this.ollamaUrl || !this.ollamaKey || !this.model)
-                return;
-            const hostname = new URL(this.url).hostname;
-            if (!this.permittedDomains.includes(hostname))
-                return;
-            if (text.length === 0) {
-                yield database.unsetData(this.url, 'llamaguard');
-                return;
-            }
-            const rigService = new RigLlamaGuardService(this.ollamaUrl, this.ollamaKey, this.model);
-            yield this.debugGetLength();
-            // run this outside of await so that it doesn't block other things
-            void (() => __awaiter(this, void 0, void 0, function* () {
-                if (!this.url)
-                    return;
-                const result = yield rigService.generate(text, this.ollamaKeepAlive);
-                if (result === undefined)
-                    return;
-                commonsOutputDebug(`LlamaGuard result is: ${result.classification.toString()}${result.classification === ELlamaGuardClassification.UNSAFE ? ` (${result.categories.join(',')})` : ''}`);
-                yield database.setData(this.url, 'llamaguard', result);
-            }))();
-        });
-    }
-}
-//# sourceMappingURL=llama-guard.parser.js.map

package/dist/parsers/llama-guard.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"llama-guard.parser.js","sourceRoot":"","sources":["../../src/parsers/llama-guard.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAqB,MAAM,qCAAqC,CAAC;AAKzH,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,YAA2B;IAIhE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,YAAY,CACb,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,YAAY,EAAE,mBAAmB,CAAC,EAAE;YAC3F,kBAAkB,CAAC,qCAAqC,CAAC,CAAC;SAC1D;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,KAAK,CAAC;QAC/D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,YAAY,CAAuB,CAAC,gBAAgB,CAAC;IACtF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;gBACjD,OAAO;aACP;YAED,MAAM,UAAU,GAAyB,IAAI,oBAAoB,CAC/D,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAE5B,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,MAAM,GAAgC,MAAM,UAAU,CAAC,QAAQ,CACnE,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,MAAM,KAAK,SAAS;oBAAE,OAAO;gBAEjC,kBAAkB,CAAC,yBAAyB,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,cAAc,KAAK,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAExL,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;YACxD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}

package/dist/parsers/offence.parser.js

@@ -1,69 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service';
-import { OllamaParser } from './ollama.parser';
-import { isIDataConfig } from './data.parser';
-export function isIOffenceConfig(test) {
-    if (!isIDataConfig(test))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'model'))
-        return false;
-    if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
-        return false;
-    return true;
-}
-export class OffenceParser extends OllamaParser {
-    constructor(url, outcome, config) {
-        super(outcome, config, 'offence');
-        this.url = url;
-        this.permittedDomains = [];
-        if (!config)
-            return;
-        if (!commonsTypeHasPropertyT(config, 'offence', isIOffenceConfig)) {
-            commonsOutputError('Invalid config for OffenceParser');
-        }
-        this.model = config['offence'].model;
-        this.permittedDomains = config['offence'].permittedDomains;
-    }
-    parseText(database, text) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.url)
-                return;
-            if (!this.ollamaUrl || !this.ollamaKey || !this.model)
-                return;
-            const hostname = new URL(this.url).hostname;
-            if (!this.permittedDomains.includes(hostname))
-                return;
-            if (text.length === 0) {
-                yield database.unsetData(this.url, 'offence');
-                return;
-            }
-            const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
-            yield this.debugGetLength();
-            // run this outside of await so that it doesn't block other things
-            void (() => __awaiter(this, void 0, void 0, function* () {
-                if (!this.url)
-                    return;
-                const score = yield rigService.generateOffenceScore(text, this.ollamaKeepAlive);
-                if (score === undefined)
-                    return;
-                if (score.score < 1)
-                    delete score.reason;
-                commonsOutputDebug(`Offence score is: ${score.score.toString(10)}`);
-                if (score.reason)
-                    commonsOutputDebug(`Reason for score is: ${score.reason}`);
-                yield database.setData(this.url, 'offence', score);
-            }))();
-        });
-    }
-}
-//# sourceMappingURL=offence.parser.js.map

package/dist/parsers/offence.parser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"offence.parser.js","sourceRoot":"","sources":["../../src/parsers/offence.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAuB,MAAM,4CAA4C,CAAC;AAK9G,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAI7D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE;YAClF,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;SACvD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,gBAAgB,CAAC;IAChF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC9C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAE5B,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,oBAAoB,CAChF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;gBAEzC,kBAAkB,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACpE,IAAI,KAAK,CAAC,MAAM;oBAAE,kBAAkB,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE7E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}