hydra-crawler 2.8.3 → 3.0.1

package/dist/services/{ollama-rest.service.d.ts → ollama-rest.service.d.mts}

@@ -1,5 +1,5 @@
-import { CommonsKeyRestClientService } from 'tscommons-es-rest';
-import { CommonsInternalHttpClientImplementation } from 'nodecommons-es-http';
+import { CommonsKeyRestClientService } from 'tscommons-esm-rest';
+import { CommonsInternalHttpClientImplementation } from 'nodecommons-esm-http';
 export declare class OllamaRestService extends CommonsKeyRestClientService {
     constructor(implementation: CommonsInternalHttpClientImplementation, url: string, key: string);
     listModels(): Promise<string[]>;

package/dist/services/ollama-rest.service.mjs

@@ -0,0 +1,27 @@
+import { commonsTypeIsString, commonsTypeIsStringArray } from 'tscommons-esm-core';
+import { ECommonsHttpContentType } from 'tscommons-esm-http';
+import { CommonsKeyRestClientService } from 'tscommons-esm-rest';
+export class OllamaRestService extends CommonsKeyRestClientService {
+    constructor(implementation, url, key) {
+        super(implementation, url, key, ECommonsHttpContentType.JSON);
+    }
+    async listModels() {
+        const result = await super.getRest('/models');
+        if (!commonsTypeIsStringArray(result))
+            throw new Error('Result was not a string array');
+        return result;
+    }
+    async generate(model, prompt) {
+        const body = {
+            prompt: prompt
+        };
+        const result = await super.postRest(`/generate/${model}`, body, undefined, undefined, {
+            maxReattempts: 0,
+            timeout: 30000
+        });
+        if (!commonsTypeIsString(result))
+            throw new Error('Result was not a string');
+        return result;
+    }
+}
+//# sourceMappingURL=ollama-rest.service.mjs.map

package/dist/services/ollama-rest.service.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama-rest.service.mjs","sourceRoot":"","sources":["../../src/services/ollama-rest.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AAIjE,MAAM,OAAO,iBAAkB,SAAQ,2BAA2B;IACjE,YACE,cAAuD,EACvD,GAAW,EACX,GAAW;QAEZ,KAAK,CACH,cAAc,EACd,GAAG,EACH,GAAG,EACH,uBAAuB,CAAC,IAAI,CAC7B,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,UAAU;QACtB,MAAM,MAAM,GAAY,MAAM,KAAK,CAAC,OAAO,CAAW,SAAS,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAExF,OAAO,MAAM,CAAC;IACf,CAAC;IAEM,KAAK,CAAC,QAAQ,CACnB,KAAa,EACb,MAAc;QAEf,MAAM,IAAI,GAAmB;YAC3B,MAAM,EAAE,MAAM;SACf,CAAC;QAEF,MAAM,MAAM,GAAY,MAAM,KAAK,CAAC,QAAQ,CAC1C,aAAa,KAAK,EAAE,EACpB,IAAI,EACJ,SAAS,EACT,SAAS,EACT;YACE,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,KAAK;SACf,CACF,CAAC;QACF,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE7E,OAAO,MAAM,CAAC;IACf,CAAC;CACD"}

package/dist/services/{rig-llama-guard.service.d.ts → rig-llama-guard.service.d.mts}

@@ -1,5 +1,5 @@
-import { CommonsFixedDuration } from 'tscommons-es-core';
-import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
+import { CommonsFixedDuration } from 'tscommons-esm-core';
+import { RigTicketedPromiseService } from './rig-ticketed-promise.service.mjs';
 export declare enum ELlamaGuardClassification {
     SAFE = "safe",
     UNSAFE = "unsafe"

package/dist/services/{rig-llama-guard.service.js → rig-llama-guard.service.mjs}

@@ -1,16 +1,7 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsArrayRemoveUndefineds, commonsArrayUnique, commonsStringSplitString, commonsTypeHasPropertyEnum, commonsTypeHasPropertyEnumArray, commonsTypeIsString } from 'tscommons-es-core';
-import { CommonsInternalHttpClientImplementation } from 'nodecommons-es-http';
-import { OllamaRestService } from './ollama-rest.service';
-import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
+import { commonsArrayRemoveUndefineds, commonsArrayUnique, commonsStringSplitString, commonsTypeHasPropertyEnum, commonsTypeHasPropertyEnumArray, commonsTypeIsString } from 'tscommons-esm-core';
+import { CommonsInternalHttpClientImplementation } from 'nodecommons-esm-http';
+import { OllamaRestService } from './ollama-rest.service.mjs';
+import { RigTicketedPromiseService } from './rig-ticketed-promise.service.mjs';
 export var ELlamaGuardClassification;
 (function (ELlamaGuardClassification) {
     ELlamaGuardClassification["SAFE"] = "safe";
@@ -220,13 +211,13 @@ function parseToTLlamaGuardResult(s) {
     };
 }
 export class RigLlamaGuardService extends RigTicketedPromiseService {
+    model;
     constructor(url, key, model) {
         super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url, 100);
         this.model = model;
     }
-    generate(text, keepAlive) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const prompt = `
+    async generate(text, keepAlive) {
+        const prompt = `
 <|begin_of_text|><|start_header_id|>user<|end_header_id|>
 
 Task: Check if there is unsafe content in 'Agent' messages in conversations according to our safety policy with the below categories.
@@ -258,33 +249,32 @@ Provide your safety assessment for ONLY THE LAST Agent in the above conversation
 - First line must read 'safe' or 'unsafe'.
 - If unsafe, a second line must include a comma-separated list of violated categories.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 		`
-                .trim()
-                .replace('<<<TEXT>>>', text);
-            const body = { prompt: prompt };
-            if (keepAlive) {
-                const mins = keepAlive.minutes;
-                if (mins < 1)
-                    throw new Error('keepAlive cannot be less than 1 minute');
-                body.keepLoaded = `${Math.round(mins)}m`;
-            }
-            try {
-                const response = yield this.submitAndWaitAsPromise(`/generate/${this.model}`, body, (data) => commonsTypeIsString(data), undefined, undefined, {
-                    timeout: 30000,
-                    maxReattempts: 0
-                });
-                if (!commonsTypeIsString(response))
-                    return undefined;
-                const parsed = parseToTLlamaGuardResult(response);
-                if (!parsed)
-                    return undefined;
-                if (!isTLlamaGuardResult(parsed))
-                    return undefined;
-                return parsed;
-            }
-            catch (e) {
+            .trim()
+            .replace('<<<TEXT>>>', text);
+        const body = { prompt: prompt };
+        if (keepAlive) {
+            const mins = keepAlive.minutes;
+            if (mins < 1)
+                throw new Error('keepAlive cannot be less than 1 minute');
+            body.keepLoaded = `${Math.round(mins)}m`;
+        }
+        try {
+            const response = await this.submitAndWaitAsPromise(`/generate/${this.model}`, body, (data) => commonsTypeIsString(data), undefined, undefined, {
+                timeout: 30000,
+                maxReattempts: 0
+            });
+            if (!commonsTypeIsString(response))
                 return undefined;
-            }
-        });
+            const parsed = parseToTLlamaGuardResult(response);
+            if (!parsed)
+                return undefined;
+            if (!isTLlamaGuardResult(parsed))
+                return undefined;
+            return parsed;
+        }
+        catch (_e) {
+            return undefined;
+        }
     }
 }
-//# sourceMappingURL=rig-llama-guard.service.js.map
+//# sourceMappingURL=rig-llama-guard.service.mjs.map

package/dist/services/rig-llama-guard.service.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rig-llama-guard.service.mjs","sourceRoot":"","sources":["../../src/services/rig-llama-guard.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,kBAAkB,EAAwB,wBAAwB,EAAE,0BAA0B,EAAE,+BAA+B,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAExN,OAAO,EAAE,uCAAuC,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,oCAAoC,CAAC;AAErG,MAAM,CAAN,IAAY,yBAGX;AAHD,WAAY,yBAAyB;IACnC,0CAAa,CAAA;IACb,8CAAiB,CAAA;AACnB,CAAC,EAHW,yBAAyB,KAAzB,yBAAyB,QAGpC;AAED,MAAM,UAAU,2BAA2B,CAAC,IAAY;IACvD,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE;YAC7C,OAAO,yBAAyB,CAAC,IAAI,CAAC;QACvC,KAAK,yBAAyB,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC/C,OAAO,yBAAyB,CAAC,MAAM,CAAC;IAC1C,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,IAA+B;IAC5E,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,yBAAyB,CAAC,IAAI;YAClC,OAAO,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClD,KAAK,yBAAyB,CAAC,MAAM;YACpC,OAAO,yBAAyB,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;IACrD,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,IAAa;IACxD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7C,OAAO,2BAA2B,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,GAAW;IACzD,QAAQ,GAAG,EAAE,CAAC;QACb,KAAK,MAAM;YACV,OAAO,yBAAyB,CAAC,IAAI,CAAC;QACvC,KAAK,QAAQ;YACZ,OAAO,yBAAyB,CAAC,MAAM,CAAC;IAC1C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,uDAAuD,GAAG,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,CAAC,MAAM,4BAA4B,GAAgC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC;KAC5G,GAAG,CAAC,CAAC,CAAS,EAA6B,EAAE,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;AAEpF,MAAM,CAAN,IAAY,mBAeX;AAfD,WAAY,mBAAmB;IAC7B,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,kCAAW,CAAA;IACX,kCAAW,CAAA;IACX,kCAAW,CAAA;IACX,kCAAW,CAAA;IACX,kCAAW,CAAA;AACb,CAAC,EAfW,mBAAmB,KAAnB,mBAAmB,QAe9B;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IACjD,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;IACjC,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAyB;IAChE,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAa;IAClD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7C,OAAO,qBAAqB,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,GAAW;IACnD,QAAQ,GAAG,EAAE,CAAC;QACb,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;IACjC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,iDAAiD,GAAG,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAA6B;IACrE,QAAQ,QAAQ,EAAE,CAAC;QAClB,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACrD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACzD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,YAAY,CAAC;QACjD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACzD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,YAAY,CAAC;QACjD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACzD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,SAAS,CAAC;QAC9C,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,uBAAuB,CAAC;QAC5D,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,wBAAwB,CAAC;QAC7D,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,MAAM,CAAC;QAC5C,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,WAAW,CAAC;QACjD,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACtD,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,WAAW,CAAC;QACjD,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,wBAAwB,CAAC;IAC/D,CAAC;AACF,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAA0B,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC;KAC1F,GAAG,CAAC,CAAC,CAAS,EAAuB,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,CAAC;AAaxE,SAAS,mBAAmB,CAAC,IAAa;IACzC,IAAI,CAAC,0BAA0B,CAA4B,IAAI,EAAE,gBAAgB,EAAE,2BAA2B,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9H,IAAI,IAAI,CAAC,cAAc,KAAK,yBAAyB,CAAC,MAAM,EAAE,CAAC;QAC9D,IAAI,CAAC,+BAA+B,CAAsB,IAAI,EAAE,YAAY,EAAE,qBAAqB,CAAC;YAAE,OAAO,KAAK,CAAC;IACpH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,wBAAwB,CAAC,CAAS;IAC1C,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACb,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;QAClB,OAAO;YACL,cAAc,EAAE,yBAAyB,CAAC,IAAI;SAC/C,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAyB,0EAA0E,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvH,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IAEzB,MAAM,UAAU,GAA0B,kBAAkB,CAC1D,4BAA4B,CAC1B,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC;SAChD,GAAG,CAAC,CAAC,EAAU,EAAiC,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CACjF,EACD,SAAS,EACT,CAAC,CAAsB,EAAE,CAAsB,EAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CACrE,CAAC;IAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1C,OAAO;QACL,cAAc,EAAE,yBAAyB,CAAC,MAAM;QAChD,UAAU,EAAE,UAAU;KACvB,CAAC;AACH,CAAC;AAED,MAAM,OAAO,oBAAqB,SAAQ,yBAAyB;IAIxD;IAHV,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,EACH,GAAG,CACJ,CAAC;QAVO,UAAK,GAAL,KAAK,CAAQ;IAWvB,CAAC;IAEM,KAAK,CAAC,QAAQ,CACnB,IAAY,EACZ,SAAgC;QAEjC,MAAM,MAAM,GAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BtB;aACE,IAAI,EAAE;aACN,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QACtD,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;YACvC,IAAI,IAAI,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;gBACE,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,CAAC;aACjB,CACF,CAAC;YACF,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC;gBAAE,OAAO,SAAS,CAAC;YAErD,MAAM,MAAM,GAA4B,wBAAwB,CAAC,QAAQ,CAAC,CAAC;YAC3E,IAAI,CAAC,MAAM;gBAAE,OAAO,SAAS,CAAC;YAC9B,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;gBAAE,OAAO,SAAS,CAAC;YAEnD,OAAO,MAAM,CAAC;QACf,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}

package/dist/services/{rig-queue-length.service.d.ts → rig-queue-length.service.d.mts}

@@ -1,4 +1,4 @@
-import { TQueueLength } from '../types/tqueue-length';
+import { TQueueLength } from '../types/tqueue-length.mjs';
 export declare class RigQueueLengthService {
     private restService;
     constructor(url: string, key: string);

package/dist/services/rig-queue-length.service.mjs

@@ -0,0 +1,12 @@
+import { CommonsInternalHttpClientImplementation } from 'nodecommons-esm-http';
+import { RigTicketRestService } from './rig-ticket-rest.service.mjs';
+export class RigQueueLengthService {
+    restService;
+    constructor(url, key) {
+        this.restService = new RigTicketRestService(new CommonsInternalHttpClientImplementation(), `${url}/tickets`, key);
+    }
+    async getLength() {
+        return await this.restService.length();
+    }
+}
+//# sourceMappingURL=rig-queue-length.service.mjs.map

package/dist/services/rig-queue-length.service.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rig-queue-length.service.mjs","sourceRoot":"","sources":["../../src/services/rig-queue-length.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uCAAuC,EAAE,MAAM,sBAAsB,CAAC;AAI/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAErE,MAAM,OAAO,qBAAqB;IACzB,WAAW,CAAuB;IAE1C,YACE,GAAW,EACX,GAAW;QAEZ,IAAI,CAAC,WAAW,GAAG,IAAI,oBAAoB,CACzC,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,UAAU,EAChB,GAAG,CACJ,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,SAAS;QACrB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;IACxC,CAAC;CACD"}

package/dist/services/{rig-subjectivity-scale.service.d.ts → rig-subjectivity-scale.service.d.mts}

@@ -1,5 +1,5 @@
-import { CommonsFixedDuration } from 'tscommons-es-core';
-import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
+import { CommonsFixedDuration } from 'tscommons-esm-core';
+import { RigTicketedPromiseService } from './rig-ticketed-promise.service.mjs';
 export type TSubjectivityResult = {
     score: number;
     reason?: string;

package/dist/services/rig-subjectivity-scale.service.mjs

@@ -0,0 +1,82 @@
+import { commonsTypeAttemptObject, commonsTypeHasPropertyNumber, commonsTypeHasPropertyStringOrUndefined, commonsTypeIsString } from 'tscommons-esm-core';
+import { CommonsInternalHttpClientImplementation } from 'nodecommons-esm-http';
+import { OllamaRestService } from './ollama-rest.service.mjs';
+import { RigTicketedPromiseService } from './rig-ticketed-promise.service.mjs';
+function isTSubjectivityResult(test) {
+    if (!commonsTypeHasPropertyNumber(test, 'score'))
+        return false;
+    if (!commonsTypeHasPropertyStringOrUndefined(test, 'reason'))
+        return false;
+    return true;
+}
+const GENERIC_PROMPT = `
+The following text was extracted from a webpage as plain text:
+
+\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`
+<<<TEXT>>>
+\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`
+
+Generate a single numeric digit score <<<REASON>>>, using the following scale:
+<<<SCALE>>>
+
+Output the score and a single sentence justifying the reason for that score, using the JSON format:
+{ "score": <score>, "reason": "<reason>"}
+
+Do not include any other prefix or suffix explanatory text.
+`.trim();
+const OFFENCE_PROMPT = GENERIC_PROMPT
+    .replace('<<<REASON>>>', 'for how offensive or controversial the text is')
+    .replace('<<<SCALE>>>', `
+0 = not offensive or controversial at all
+1 = mildly offensive or controversial to a few people
+2 = significantly offensive or controversial to most people
+3 = extremely offensive or highly controversial, potentially racist, sexist, homophobic or promoting hatred.
+				`.trim());
+const INTEREST_PROMPT = GENERIC_PROMPT
+    .replace('<<<REASON>>>', 'for how interesting or enlightening the text is')
+    .replace('SCALE', `
+0 = purely function information with no further interest, such as opening times or directions
+1 = information that is partially interesting or intriguing, such as informative descriptions
+2 = opinion and commentary which is likely to be interesting to many people
+3 = highly informative and interesting or enlightening commentary, which most people would rate highly.
+				`.trim());
+export class RigSubjectivityScaleService extends RigTicketedPromiseService {
+    model;
+    constructor(url, key, model) {
+        super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url, 100);
+        this.model = model;
+    }
+    async generateSubjectivityScore(text, prompt, keepAlive) {
+        prompt = prompt
+            .replace('<<<TEXT>>>', text);
+        const body = { prompt: prompt };
+        if (keepAlive) {
+            const mins = keepAlive.minutes;
+            if (mins < 1)
+                throw new Error('keepAlive cannot be less than 1 minute');
+            body.keepLoaded = `${Math.round(mins)}m`;
+        }
+        try {
+            const response = await this.submitAndWaitAsPromise(`/generate/${this.model}`, body, (data) => commonsTypeIsString(data), undefined, undefined, {
+                timeout: 30000,
+                maxReattempts: 0
+            });
+            const attempt = commonsTypeAttemptObject(response);
+            if (!attempt)
+                return undefined;
+            if (!isTSubjectivityResult(attempt))
+                return undefined;
+            return attempt;
+        }
+        catch (_e) {
+            return undefined;
+        }
+    }
+    async generateOffenceScore(text, keepAlive) {
+        return await this.generateSubjectivityScore(text, OFFENCE_PROMPT, keepAlive);
+    }
+    async generateInterestScore(text, keepAlive) {
+        return await this.generateSubjectivityScore(text, INTEREST_PROMPT, keepAlive);
+    }
+}
+//# sourceMappingURL=rig-subjectivity-scale.service.mjs.map

package/dist/services/rig-subjectivity-scale.service.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rig-subjectivity-scale.service.mjs","sourceRoot":"","sources":["../../src/services/rig-subjectivity-scale.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAyC,wBAAwB,EAAE,4BAA4B,EAAE,uCAAuC,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEjM,OAAO,EAAE,uCAAuC,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,oCAAoC,CAAC;AAMrG,SAAS,qBAAqB,CAAC,IAAa;IAC3C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,cAAc,GAAW;;;;;;;;;;;;;;CAc9B,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,cAAc,GAAW,cAAc;KAC1C,OAAO,CACN,cAAc,EACd,gDAAgD,CACjD;KACA,OAAO,CACN,aAAa,EACb;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,eAAe,GAAW,cAAc;KAC3C,OAAO,CACN,cAAc,EACd,iDAAiD,CAClD;KACA,OAAO,CACN,OAAO,EACP;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,OAAO,2BAA4B,SAAQ,yBAAyB;IAI/D;IAHV,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,EACH,GAAG,CACJ,CAAC;QAVO,UAAK,GAAL,KAAK,CAAQ;IAWvB,CAAC;IAEO,KAAK,CAAC,yBAAyB,CACrC,IAAY,EACZ,MAAc,EACd,SAAgC;QAEjC,MAAM,GAAG,MAAM;aACZ,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QACtD,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;YACvC,IAAI,IAAI,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;gBACE,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,CAAC;aACjB,CACF,CAAC;YACF,MAAM,OAAO,GAAuC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;YACvF,IAAI,CAAC,OAAO;gBAAE,OAAO,SAAS,CAAC;YAE/B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC;gBAAE,OAAO,SAAS,CAAC;YAEtD,OAAO,OAAO,CAAC;QAChB,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAC/B,IAAY,EACZ,SAAgC;QAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,cAAc,EACd,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB,CAChC,IAAY,EACZ,SAAgC;QAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,eAAe,EACf,SAAS,CACV,CAAC;IACH,CAAC;CACD"}

package/dist/services/{rig-ticket-rest.service.d.ts → rig-ticket-rest.service.d.mts}

@@ -1,6 +1,6 @@
-import { CommonsKeyRestClientService } from 'tscommons-es-rest';
-import { CommonsInternalHttpClientImplementation } from 'nodecommons-es-http';
-import { TQueueLength } from '../types/tqueue-length';
+import { CommonsKeyRestClientService } from 'tscommons-esm-rest';
+import { CommonsInternalHttpClientImplementation } from 'nodecommons-esm-http';
+import { TQueueLength } from '../types/tqueue-length.mjs';
 export declare class RigTicketRestService extends CommonsKeyRestClientService {
     constructor(implementation: CommonsInternalHttpClientImplementation, url: string, key: string);
     length(): Promise<TQueueLength>;

package/dist/services/rig-ticket-rest.service.mjs

@@ -0,0 +1,15 @@
+import { commonsTypeIsNumberPropertyObjectT } from 'tscommons-esm-core';
+import { ECommonsHttpContentType } from 'tscommons-esm-http';
+import { CommonsKeyRestClientService } from 'tscommons-esm-rest';
+export class RigTicketRestService extends CommonsKeyRestClientService {
+    constructor(implementation, url, key) {
+        super(implementation, url, key, ECommonsHttpContentType.JSON);
+    }
+    async length() {
+        const result = await super.getRest('/length');
+        if (!commonsTypeIsNumberPropertyObjectT(result))
+            throw new Error('Result was not a number key object');
+        return result;
+    }
+}
+//# sourceMappingURL=rig-ticket-rest.service.mjs.map

package/dist/services/rig-ticket-rest.service.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rig-ticket-rest.service.mjs","sourceRoot":"","sources":["../../src/services/rig-ticket-rest.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kCAAkC,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AAMjE,MAAM,OAAO,oBAAqB,SAAQ,2BAA2B;IACpE,YACE,cAAuD,EACvD,GAAW,EACX,GAAW;QAEZ,KAAK,CACH,cAAc,EACd,GAAG,EACH,GAAG,EACH,uBAAuB,CAAC,IAAI,CAC7B,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,MAAM;QAClB,MAAM,MAAM,GAAY,MAAM,KAAK,CAAC,OAAO,CAAe,SAAS,CAAC,CAAC;QACrE,IAAI,CAAC,kCAAkC,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAEvG,OAAO,MAAsB,CAAC;IAC/B,CAAC;CACD"}

package/dist/services/{rig-ticketed-promise.service.d.ts → rig-ticketed-promise.service.d.mts}

@@ -1,6 +1,6 @@
-import { TEncoded, TEncodedObject } from 'tscommons-es-core';
-import { CommonsRestClientService } from 'tscommons-es-rest';
-import { TCommonsHttpRequestOptions } from 'tscommons-es-http';
+import { TEncoded, TEncodedObject } from 'tscommons-esm-core';
+import { CommonsRestClientService } from 'tscommons-esm-rest';
+import { TCommonsHttpRequestOptions } from 'tscommons-esm-http';
 export type TPromptAndKeepLoaded = {
     prompt: string;
     keepLoaded?: string;
@@ -10,7 +10,5 @@ export declare abstract class RigTicketedPromiseService {
     private socketIoUrl;
     private connectToSubmitDelay;
     constructor(restService: CommonsRestClientService, socketIoUrl: string, connectToSubmitDelay?: number);
-    submitAndWaitAsPromise<ResultT extends TEncoded = TEncoded, BodyT extends (TEncodedObject & TPromptAndKeepLoaded) = (TEncodedObject & TPromptAndKeepLoaded), ParamsT extends TEncodedObject = TEncodedObject, HeadersT extends TEncodedObject = TEncodedObject>(script: string, body: BodyT, checker: (data: unknown) => data is ResultT, params?: ParamsT | undefined, // the params can be explicitly undefined (none) as well as optional
-    headers?: HeadersT | undefined, // the headers can be explicitly undefined (none) as well as optional
-    options?: TCommonsHttpRequestOptions): Promise<ResultT>;
+    submitAndWaitAsPromise<ResultT extends TEncoded = TEncoded, BodyT extends (TEncodedObject & TPromptAndKeepLoaded) = (TEncodedObject & TPromptAndKeepLoaded), ParamsT extends TEncodedObject = TEncodedObject, HeadersT extends TEncodedObject = TEncodedObject>(script: string, body: BodyT, checker: (data: unknown) => data is ResultT, params?: ParamsT, headers?: HeadersT, options?: TCommonsHttpRequestOptions): Promise<ResultT>;
 }

package/dist/services/{rig-ticketed-promise.service.js → rig-ticketed-promise.service.mjs}

@@ -1,15 +1,16 @@
-import { ECommonsHttpMethod } from 'tscommons-es-http';
-import { commonsTicketedNodeGenericSubmitAndWaitAsPromise } from 'nodecommons-es-socket-io-ticketed';
+import { ECommonsHttpMethod } from 'tscommons-esm-http';
+import { commonsTicketedNodeGenericSubmitAndWaitAsPromise } from 'nodecommons-esm-socket-io-ticketed';
 export class RigTicketedPromiseService {
+    restService;
+    socketIoUrl;
+    connectToSubmitDelay;
     constructor(restService, socketIoUrl, connectToSubmitDelay = 100) {
         this.restService = restService;
         this.socketIoUrl = socketIoUrl;
         this.connectToSubmitDelay = connectToSubmitDelay;
     }
-    submitAndWaitAsPromise(script, body, checker, params, // the params can be explicitly undefined (none) as well as optional
-    headers, // the headers can be explicitly undefined (none) as well as optional
-    options = {}) {
+    submitAndWaitAsPromise(script, body, checker, params, headers, options = {}) {
         return commonsTicketedNodeGenericSubmitAndWaitAsPromise(this.socketIoUrl, 'rig-marshall', this.restService, ECommonsHttpMethod.POST, script, body, checker, params, headers, options, this.connectToSubmitDelay);
     }
 }
-//# sourceMappingURL=rig-ticketed-promise.service.js.map
+//# sourceMappingURL=rig-ticketed-promise.service.mjs.map

package/dist/services/rig-ticketed-promise.service.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rig-ticketed-promise.service.mjs","sourceRoot":"","sources":["../../src/services/rig-ticketed-promise.service.mts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAA8B,MAAM,oBAAoB,CAAC;AAEpF,OAAO,EAAE,gDAAgD,EAAE,MAAM,oCAAoC,CAAC;AAOtG,MAAM,OAAgB,yBAAyB;IAEpC;IACA;IACA;IAHV,YACU,WAAqC,EACrC,WAAmB,EACnB,uBAA+B,GAAG;QAFlC,gBAAW,GAAX,WAAW,CAA0B;QACrC,gBAAW,GAAX,WAAW,CAAQ;QACnB,yBAAoB,GAApB,oBAAoB,CAAc;IACzC,CAAC;IAEG,sBAAsB,CAM3B,MAAc,EACd,IAAW,EACX,OAA2C,EAC3C,MAAgB,EAChB,OAAkB,EAClB,UAAsC,EAAE;QAEzC,OAAO,gDAAgD,CAMrD,IAAI,CAAC,WAAW,EAChB,cAAc,EACd,IAAI,CAAC,WAAW,EAChB,kBAAkB,CAAC,IAAI,EACvB,MAAM,EACN,IAAI,EACJ,OAAO,EACP,MAAM,EACN,OAAO,EACP,OAAO,EACP,IAAI,CAAC,oBAAoB,CAC1B,CAAC;IACH,CAAC;CACD"}

package/dist/types/{tcrawl-config.js → tcrawl-config.mjs}

@@ -1,4 +1,4 @@
-import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumber, commonsTypeHasPropertyStringArray } from 'tscommons-es-core';
+import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumber, commonsTypeHasPropertyStringArray } from 'tscommons-esm-core';
 export function isTCrawlConfig(test) {
     if (!commonsTypeHasPropertyNumber(test, 'findNewAvailableDelay'))
         return false;
@@ -28,4 +28,4 @@ export function isTCrawlConfig(test) {
         return false;
     return true;
 }
-//# sourceMappingURL=tcrawl-config.js.map
+//# sourceMappingURL=tcrawl-config.mjs.map

package/dist/types/tcrawl-config.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tcrawl-config.mjs","sourceRoot":"","sources":["../../src/types/tcrawl-config.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,4BAA4B,EAAE,iCAAiC,EAAE,MAAM,oBAAoB,CAAC;AAgB/I,MAAM,UAAU,cAAc,CAAC,IAAa;IAC3C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,uBAAuB,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/E,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IACrE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,oBAAoB,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5E,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,4BAA4B,CAAC;QAAE,OAAO,KAAK,CAAC;IACpF,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,mBAAmB,CAAC;QAAE,OAAO,KAAK,CAAC;IAC3E,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IACvE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,gBAAgB,CAAC;QAAE,OAAO,KAAK,CAAC;IACxE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IACrE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,aAAa,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1E,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACvC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAgB,CAAC;YAAE,OAAO,KAAK,CAAC;IACrD,CAAC;IAED,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,uBAAuB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3F,OAAO,IAAI,CAAC;AACb,CAAC"}

package/dist/types/{thydra-config.js → thydra-config.mjs}

@@ -1,7 +1,7 @@
-import { commonsTypeHasPropertyString } from 'tscommons-es-core';
+import { commonsTypeHasPropertyString } from 'tscommons-esm-core';
 export function isTHydraConfig(test) {
     if (!commonsTypeHasPropertyString(test, 'startUrl'))
         return false;
     return true;
 }
-//# sourceMappingURL=thydra-config.js.map
+//# sourceMappingURL=thydra-config.mjs.map

package/dist/types/thydra-config.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"thydra-config.mjs","sourceRoot":"","sources":["../../src/types/thydra-config.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAMlE,MAAM,UAAU,cAAc,CAAC,IAAa;IAC3C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAElE,OAAO,IAAI,CAAC;AACb,CAAC"}

package/dist/types/tparser-ctor.d.mts

@@ -0,0 +1,7 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { IParser } from '../interfaces/iparser.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+export type TParserCtor = {
+    new (url?: string, outcome?: IRequestOutcome, config?: TPropertyObject<IParserConfig>): IParser;
+};

package/dist/types/tparser-ctor.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tparser-ctor.mjs.map

package/dist/types/tparser-ctor.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tparser-ctor.mjs","sourceRoot":"","sources":["../../src/types/tparser-ctor.mts"],"names":[],"mappings":""}

package/dist/types/tquery.d.mts

@@ -0,0 +1,7 @@
+import { TPropertyObject } from 'tscommons-esm-core';
+import { CommonsArgs } from 'nodecommons-esm-cli';
+import { Lists } from '../classes/lists.mjs';
+import { Expiry } from '../classes/expiry.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+export type TQuery = (args: CommonsArgs, databaseService: DatabaseService, lists: Lists, expiry: Expiry, parsersConfig: TPropertyObject<IParserConfig>) => Promise<void>;

package/dist/types/tquery.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tquery.mjs.map

package/dist/types/tquery.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tquery.mjs","sourceRoot":"","sources":["../../src/types/tquery.mts"],"names":[],"mappings":""}

package/dist/types/tqueue-length.mjs

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tqueue-length.mjs.map

package/dist/types/tqueue-length.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tqueue-length.mjs","sourceRoot":"","sources":["../../src/types/tqueue-length.mts"],"names":[],"mappings":""}

package/dist/types/{trobots-config.js → trobots-config.mjs}

@@ -1,7 +1,7 @@
-import { commonsTypeHasPropertyBoolean } from 'tscommons-es-core';
+import { commonsTypeHasPropertyBoolean } from 'tscommons-esm-core';
 export function isTRobotsConfig(test) {
     if (!commonsTypeHasPropertyBoolean(test, 'ignoreWholeSiteDirectives'))
         return false;
     return true;
 }
-//# sourceMappingURL=trobots-config.js.map
+//# sourceMappingURL=trobots-config.mjs.map

package/dist/types/trobots-config.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"trobots-config.mjs","sourceRoot":"","sources":["../../src/types/trobots-config.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,oBAAoB,CAAC;AAMnE,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,6BAA6B,CAAC,IAAI,EAAE,2BAA2B,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpF,OAAO,IAAI,CAAC;AACb,CAAC"}

package/package.json

@@ -1,64 +1,58 @@
 {
 	"name": "hydra-crawler",
-	"version": "2.8.3",
+	"version": "3.0.1",
 	"description": "Node.js Hydra web crawler",
 	"author": "Pete Morris",
 	"license": "ISC",
-	"main": "dist/cli.js",
 	"type": "module",
-	"types": "dist/index.d.ts",
 	"scripts": {
+		"tsc": "./node_modules/typescript/bin/tsc",
 		"start": "node dist/cli.js",
-		"publish-major": "rm -rf dist; tsc && npx eslint . && npm version major && npm install && npm publish && git add . && git commit -m 'publish'",
-		"publish-minor": "rm -rf dist; tsc && npx eslint . && npm version minor && npm install && npm publish && git add . && git commit -m 'publish'",
-		"publish-patch": "rm -rf dist; tsc && npx eslint . && npm version patch && npm install && npm publish && git add . && git commit -m 'publish'"
+		"publish-major": "rm -rf dist; npm run tsc && npx eslint . && npm version major && npm install && npm publish && git add . && git commit -m 'publish'",
+		"publish-minor": "rm -rf dist; npm run tsc && npx eslint . && npm version minor && npm install && npm publish && git add . && git commit -m 'publish'",
+		"publish-patch": "rm -rf dist; npm run tsc && npx eslint . && npm version patch && npm install && npm publish && git add . && git commit -m 'publish'"
 	},
 	"bin": {
 		"hydra-crawler": "./dist/cli.js"
 	},
+	"main": "dist/cli.mjs",
 	"dependencies": {
-		"chardet": "^1.6.0",
-		"cheerio": "1.0.0-rc.12",
-		"express": "^4.18.2",
-		"hydra-crawler-ts-assets": "2.0.1",
+		"chardet": "^2.1.0",
+		"cheerio": "^1.0.0",
+		"express": "^5.1.0",
+		"hydra-crawler-esm-assets": "^3.0.1",
 		"jpeg-js": "^0.4.4",
 		"memcached-promisify": "^2.0.0",
-		"mongodb": "^5.0.1",
-		"nodecommons-es-app": "^1.1.2",
-		"nodecommons-es-app-socket-io": "3.0.3",
-		"nodecommons-es-cli": "^2.1.1",
-		"nodecommons-es-config": "^1.0.8",
-		"nodecommons-es-database": "^2.2.8",
-		"nodecommons-es-database-mongodb": "^1.0.9",
-		"nodecommons-es-express": "^4.0.3",
-		"nodecommons-es-file": "^1.1.5",
-		"nodecommons-es-http": "^2.5.3",
-		"nodecommons-es-process": "^1.5.2",
-		"nodecommons-es-rest": "^2.0.14",
-		"nodecommons-es-security": "^1.0.10",
-		"nodecommons-es-socket-io": "3.0.7",
-		"nodecommons-es-socket-io-ticketed": "^3.1.1",
-		"tscommons-es-async": "^1.2.4",
-		"tscommons-es-config": "^1.0.6",
-		"tscommons-es-core": "^1.25.1",
-		"tscommons-es-format": "1.1.1",
-		"tscommons-es-rest": "^1.1.2",
+		"mongodb": "^6.15.0",
+		"nodecommons-esm-app": "^0.0.10",
+		"nodecommons-esm-app-socket-io": "^0.0.7",
+		"nodecommons-esm-cli": "^0.0.10",
+		"nodecommons-esm-config": "^0.0.9",
+		"nodecommons-esm-database": "^0.0.12",
+		"nodecommons-esm-database-mongodb": "^0.0.5",
+		"nodecommons-esm-express": "^0.0.10",
+		"nodecommons-esm-file": "^0.0.5",
+		"nodecommons-esm-http": "^0.0.8",
+		"nodecommons-esm-log": "^0.0.7",
+		"nodecommons-esm-process": "^0.1.0",
+		"nodecommons-esm-rest": "^0.0.9",
+		"nodecommons-esm-security": "^0.0.5",
+		"nodecommons-esm-socket-io-ticketed": "^0.0.6",
+		"tscommons-esm-async": "^0.0.9",
+		"tscommons-esm-config": "^0.0.5",
+		"tscommons-esm-core": "^0.0.5",
+		"tscommons-esm-format": "^0.0.5",
+		"tscommons-esm-http": "^0.0.6",
+		"tscommons-esm-rest": "^0.0.6",
 		"unfluff": "^3.2.0"
 	},
 	"devDependencies": {
-		"@types/chardet": "^0.8.1",
-		"@types/cheerio": "0.22.32",
-		"@types/express": "^4.17.17",
-		"@types/mongodb": "^4.0.6",
-		"@types/node": "18.6.1",
-		"@typescript-eslint/eslint-plugin": "^5.53.0",
-		"@typescript-eslint/parser": "^5.53.0",
-		"eslint": "^8.35.0",
-		"eslint-config-prettier": "^8.6.0",
-		"eslint-plugin-import": "^2.27.5",
-		"eslint-plugin-jsdoc": "^40.0.0",
-		"eslint-plugin-prefer-arrow": "^1.2.3",
-		"typescript": "^4.8.4"
+		"@stylistic/eslint-plugin-ts": "^2.11.0",
+		"@types/node": "^22.10.0",
+		"eslint-plugin-import": "^2.31.0",
+		"eslint-plugin-prefer-arrow-functions": "^3.4.1",
+		"typescript": "^5.7.2",
+		"typescript-eslint": "^8.16.0"
 	},
 	"files": [
 		"dist/**/*"

package/dist/apis/autocomplete.api.js

@@ -1,22 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { CommonsRestApi } from 'nodecommons-es-rest';
-export class AutocompleteApi extends CommonsRestApi {
-    constructor(restServer, databaseService, path) {
-        super(restServer);
-        this.databaseService = databaseService;
-        super.getHandler(`${path}rest/autocomplete/domain/:term[idname]`, (req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const domains = yield this.databaseService.listDomainsByLike(req.strictParams.term);
-            return domains
-                .map((domain) => domain.domain);
-        }));
-    }
-}
-//# sourceMappingURL=autocomplete.api.js.map

package/dist/apis/autocomplete.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"autocomplete.api.js","sourceRoot":"","sources":["../../src/apis/autocomplete.api.ts"],"names":[],"mappings":";;;;;;;;;AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAMrD,MAAM,OAAO,eAAgB,SAAQ,cAAiE;IACrG,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,wCAAwC,EAC/C,CAAO,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,MAAM,OAAO,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAc,CAAC,CAAC;YAEzG,OAAO,OAAO;iBACX,GAAG,CAAC,CAAC,MAAe,EAAU,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}