hydra-crawler 2.3.2 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/hydra.app.js +3 -1
- package/dist/apps/hydra.app.js.map +1 -1
- package/dist/parsers/offence.parser.d.ts +16 -0
- package/dist/parsers/offence.parser.js +57 -0
- package/dist/parsers/offence.parser.js.map +1 -0
- package/dist/parsers/ollama.parser.d.ts +17 -0
- package/dist/parsers/ollama.parser.js +32 -0
- package/dist/parsers/ollama.parser.js.map +1 -0
- package/dist/services/ollama-rest.service.d.ts +7 -0
- package/dist/services/ollama-rest.service.js +46 -0
- package/dist/services/ollama-rest.service.js.map +1 -0
- package/dist/services/rig-offence-scale.service.d.ts +7 -0
- package/dist/services/rig-offence-scale.service.js +58 -0
- package/dist/services/rig-offence-scale.service.js.map +1 -0
- package/dist/services/rig-ticketed-promise.service.d.ts +16 -0
- package/dist/services/rig-ticketed-promise.service.js +14 -0
- package/dist/services/rig-ticketed-promise.service.js.map +1 -0
- package/package.json +11 -9
package/dist/apps/hydra.app.js
CHANGED
|
@@ -36,6 +36,7 @@ import { PhpErrorParser } from '../parsers/php-error.parser';
|
|
|
36
36
|
import { AspErrorParser } from '../parsers/asp-error.parser';
|
|
37
37
|
import { ComplexEnglishParser } from '../parsers/complex-english.parser';
|
|
38
38
|
import { AccessibilityMetricsParser } from '../parsers/accessibility-metrics.parser';
|
|
39
|
+
import { OffenceParser } from '../parsers/offence.parser';
|
|
39
40
|
import { isIParserConfig } from '../interfaces/iparser-config';
|
|
40
41
|
import { isTHydraConfig } from '../types/thydra-config';
|
|
41
42
|
import { isTCrawlConfig } from '../types/tcrawl-config';
|
|
@@ -141,7 +142,8 @@ export class HydraApp extends CommonsSocketIoApp {
|
|
|
141
142
|
PhpErrorParser,
|
|
142
143
|
AspErrorParser,
|
|
143
144
|
ComplexEnglishParser,
|
|
144
|
-
AccessibilityMetricsParser
|
|
145
|
+
AccessibilityMetricsParser,
|
|
146
|
+
OffenceParser
|
|
145
147
|
]) {
|
|
146
148
|
commonsOutputDebug(`Installing parser ${parser.name}`);
|
|
147
149
|
this.crawlServer.addParser(parser);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAGxE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACrH,OAAO,EAAE,gCAAgC,EAA0D,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAG3J,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAGxE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACrH,OAAO,EAAE,gCAAgC,EAA0D,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAG3J,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yCAAyC,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI9E,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAUzE,MAAM,OAAO,QAAS,SAAQ,kBAAkC;IAkB/D;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QANhB,qBAAgB,GAA2B,EAAE,CAAC;QAQrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QAEnC,MAAM,YAAY,GAAY,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IAEkB,eAAe;QACjC,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAEvF,OAAO,gCAAgC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAC9F,CAAC;IAES,mBAAmB,CAC3B,aAAyC,EACzC,aAAoC;QAErC,OAAO,IAAI,cAAc,CACvB,aAAa,EACb,aAAa,CACd,CAAC;IACH,CAAC;IAEM,UAAU;QAChB,OAAO,OAAO,CAAC;IAChB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEM,sBAAsB,CAC3B,KAA6B;QAE9B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,CAAC,OAAO,CACV,CAAC,UAAgF,EAAE,IAAY,EAAQ,EAAE;gBACxG,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAExF,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;gBACnD,IAAI,eAAe,CACjB,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;gBAC3C,IAAI,QAAQ,CACV,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;gBAC7C,IAAI,UAAU,CACZ,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;gBAC5C,IAAI,SAAS,CACX,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAChD,IAAI,aAAa,CACf,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACxB,CAAC,CACF,CAAC;YAEF,MAAM,OAAM,IAAI,WAAE,CAAC;YAEnB,MAAM,cAAc,GAA6B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAE7E,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,IAAI,CAAC,eAAe,EACpB,cAAc,CACf,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;YAEF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI;gBACnB,YAAY;gBACZ,gBAAgB;gBAChB,eAAe;gBACf,UAAU;gBACV,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,oBAAoB;gBACpB,0BAA0B;gBAC1B,aAAa;aACd,EAAE;gBACF,kBAAkB,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC9C;YAED,MAAM,oBAAoB,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,wBAAwB,CAAC,CAAC;YAC7G,IAAI,oBAAoB;gBAAE,kBAAkB,CAAC,qCAAqC,oBAAoB,EAAE,CAAC,CAAC;YAE1G,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,oBAAoB,CAAC,2DAA2D;aACjF,CAAC;QACH,CAAC;KAAA;IAES,SAAS;QAClB,KAAK,CAAC,GAAwB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAA,CAAC,EAAE,CAAC;IACN,CAAC;IAEY,GAAG;;;;;YACf,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,SAAS,EAAE;gBACd,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAE5E,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;gBAEvC,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,CAAC,QAAQ,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAC9B,IAAI,CAAC,KAAK,CACX,CAAC;gBAEF,iBAAiB,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAW,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;gBAE7D,OAAO;aACP;YAED,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAChG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;YAErH,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAE1B,MAAM,OAAM,GAAG,WAAE,CAAC;QACnB,CAAC;KAAA;IAEe,QAAQ;;;;;YACvB,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAElC,MAAM,OAAM,QAAQ,WAAE,CAAC;QACxB,CAAC;KAAA;CACD"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { TKeyObject } from 'tscommons-es-core';
|
|
2
|
+
import { DatabaseService } from '../services/database.service';
|
|
3
|
+
import { IParserConfig } from '../interfaces/iparser-config';
|
|
4
|
+
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
5
|
+
import { IOllamaConfig, OllamaParser } from './ollama.parser';
|
|
6
|
+
import { IDataConfig } from './data.parser';
|
|
7
|
+
export interface IOffenceConfig extends IDataConfig {
|
|
8
|
+
model: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function isIOffenceConfig(test: unknown): test is IOffenceConfig;
|
|
11
|
+
export declare class OffenceParser extends OllamaParser<IOllamaConfig> {
|
|
12
|
+
private url?;
|
|
13
|
+
private model;
|
|
14
|
+
constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
|
|
15
|
+
protected parseText(database: DatabaseService, text: string): Promise<void>;
|
|
16
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { commonsTypeHasPropertyString, commonsTypeHasPropertyT } from 'tscommons-es-core';
|
|
11
|
+
import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
|
|
12
|
+
import { RigOffenceScaleService } from '../services/rig-offence-scale.service';
|
|
13
|
+
import { OllamaParser } from './ollama.parser';
|
|
14
|
+
import { isIDataConfig } from './data.parser';
|
|
15
|
+
export function isIOffenceConfig(test) {
|
|
16
|
+
if (!isIDataConfig(test))
|
|
17
|
+
return false;
|
|
18
|
+
if (!commonsTypeHasPropertyString(test, 'model'))
|
|
19
|
+
return false;
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
export class OffenceParser extends OllamaParser {
|
|
23
|
+
constructor(url, outcome, config) {
|
|
24
|
+
super(outcome, config, 'offence');
|
|
25
|
+
this.url = url;
|
|
26
|
+
if (!config)
|
|
27
|
+
return;
|
|
28
|
+
if (!commonsTypeHasPropertyT(config, 'offence', isIOffenceConfig)) {
|
|
29
|
+
commonsOutputError('Invalid config for OffenceParser');
|
|
30
|
+
}
|
|
31
|
+
this.model = config['offence'].model;
|
|
32
|
+
}
|
|
33
|
+
parseText(database, text) {
|
|
34
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
35
|
+
if (!this.url)
|
|
36
|
+
return;
|
|
37
|
+
if (!this.ollamaUrl || !this.ollamaKey || !this.model)
|
|
38
|
+
return;
|
|
39
|
+
if (text.length === 0) {
|
|
40
|
+
yield database.unsetData(this.url, 'offence');
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const rigOffenceService = new RigOffenceScaleService(this.ollamaUrl, this.ollamaKey, this.model);
|
|
44
|
+
// run this outside of await so that it doesn't block other things
|
|
45
|
+
void (() => __awaiter(this, void 0, void 0, function* () {
|
|
46
|
+
if (!this.url)
|
|
47
|
+
return;
|
|
48
|
+
const score = yield rigOffenceService.generateOffenceScore(text, this.ollamaKeepAlive);
|
|
49
|
+
if (!score)
|
|
50
|
+
return;
|
|
51
|
+
commonsOutputDebug(`Offence score is: ${score.toString(10)}`);
|
|
52
|
+
yield database.setData(this.url, 'offence', score);
|
|
53
|
+
}));
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=offence.parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offence.parser.js","sourceRoot":"","sources":["../../src/parsers/offence.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEtG,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAK/E,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAG7D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAUrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE;YAClF,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;SACvD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;IAC1D,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC9C,OAAO;aACP;YAED,MAAM,iBAAiB,GAA2B,IAAI,sBAAsB,CAC1E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAqB,MAAM,iBAAiB,CAAC,oBAAoB,CAC1E,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,CAAC,KAAK;oBAAE,OAAO;gBAEnB,kBAAkB,CAAC,qBAAqB,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC,CAAA,CAAC,CAAC;QACJ,CAAC;KAAA;CACD"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { CommonsFixedDuration, TKeyObject } from 'tscommons-es-core';
|
|
2
|
+
import { IParserConfig } from '../interfaces/iparser-config';
|
|
3
|
+
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
4
|
+
import { TextParser } from './text.parser';
|
|
5
|
+
import { IDataConfig } from './data.parser';
|
|
6
|
+
export interface IOllamaConfig extends IDataConfig {
|
|
7
|
+
url: string;
|
|
8
|
+
key: string;
|
|
9
|
+
keepAlive?: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function isIOllamaConfig(test: unknown): test is IOllamaConfig;
|
|
12
|
+
export declare abstract class OllamaParser<T extends IOllamaConfig> extends TextParser<T> {
|
|
13
|
+
protected readonly ollamaUrl: string | undefined;
|
|
14
|
+
protected readonly ollamaKey: string | undefined;
|
|
15
|
+
protected readonly ollamaKeepAlive: CommonsFixedDuration | undefined;
|
|
16
|
+
constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
|
|
17
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { CommonsFixedDuration, commonsTypeHasPropertyNumberOrUndefined, commonsTypeHasPropertyString, commonsTypeHasPropertyT } from 'tscommons-es-core';
|
|
2
|
+
import { commonsOutputError } from 'nodecommons-es-cli';
|
|
3
|
+
import { TextParser } from './text.parser';
|
|
4
|
+
import { isIDataConfig } from './data.parser';
|
|
5
|
+
export function isIOllamaConfig(test) {
|
|
6
|
+
if (!isIDataConfig(test))
|
|
7
|
+
return false;
|
|
8
|
+
if (!commonsTypeHasPropertyString(test, 'url'))
|
|
9
|
+
return false;
|
|
10
|
+
if (!commonsTypeHasPropertyString(test, 'key'))
|
|
11
|
+
return false;
|
|
12
|
+
if (!commonsTypeHasPropertyNumberOrUndefined(test, 'keepAlive'))
|
|
13
|
+
return false;
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
export class OllamaParser extends TextParser {
|
|
17
|
+
constructor(outcome, config, configKey) {
|
|
18
|
+
super(outcome, config, configKey);
|
|
19
|
+
if (!config)
|
|
20
|
+
return;
|
|
21
|
+
if (!commonsTypeHasPropertyT(config, 'ollama', isIOllamaConfig)) {
|
|
22
|
+
commonsOutputError('Invalid config for OllamaParser');
|
|
23
|
+
}
|
|
24
|
+
const ollamaConfig = config['ollama'];
|
|
25
|
+
this.ollamaUrl = ollamaConfig.url;
|
|
26
|
+
this.ollamaKey = ollamaConfig.key;
|
|
27
|
+
if (ollamaConfig.keepAlive) {
|
|
28
|
+
this.ollamaKeepAlive = CommonsFixedDuration.fromMillis(ollamaConfig.keepAlive);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=ollama.parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama.parser.js","sourceRoot":"","sources":["../../src/parsers/ollama.parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,uCAAuC,EAAE,4BAA4B,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAErK,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAKxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAO3D,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,WAAW,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAgB,YAAsC,SAAQ,UAAa;IAKhF,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QAEF,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAgB,MAAM,EAAE,QAAQ,EAAE,eAAe,CAAC,EAAE;YAC/E,kBAAkB,CAAC,iCAAiC,CAAC,CAAC;SACtD;QACD,MAAM,YAAY,GAAkB,MAAM,CAAC,QAAQ,CAAkB,CAAC;QAEtE,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QAElC,IAAI,YAAY,CAAC,SAAS,EAAE;YAC3B,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;SAC/E;IACF,CAAC;CACD"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { CommonsKeyRestClientService } from 'tscommons-es-rest';
|
|
2
|
+
import { CommonsInternalHttpClientImplementation } from 'nodecommons-es-http';
|
|
3
|
+
export declare class OllamaRestService extends CommonsKeyRestClientService {
|
|
4
|
+
constructor(implementation: CommonsInternalHttpClientImplementation, url: string, key: string);
|
|
5
|
+
listModels(): Promise<string[]>;
|
|
6
|
+
generate(model: string, prompt: string): Promise<string>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { commonsTypeIsString, commonsTypeIsStringArray } from 'tscommons-es-core';
|
|
11
|
+
import { ECommonsHttpContentType } from 'tscommons-es-http';
|
|
12
|
+
import { CommonsKeyRestClientService } from 'tscommons-es-rest';
|
|
13
|
+
export class OllamaRestService extends CommonsKeyRestClientService {
|
|
14
|
+
constructor(implementation, url, key) {
|
|
15
|
+
super(implementation, url, key, ECommonsHttpContentType.JSON);
|
|
16
|
+
}
|
|
17
|
+
listModels() {
|
|
18
|
+
const _super = Object.create(null, {
|
|
19
|
+
getRest: { get: () => super.getRest }
|
|
20
|
+
});
|
|
21
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
22
|
+
const result = yield _super.getRest.call(this, '/models');
|
|
23
|
+
if (!commonsTypeIsStringArray(result))
|
|
24
|
+
throw new Error('Result was not a string array');
|
|
25
|
+
return result;
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
generate(model, prompt) {
|
|
29
|
+
const _super = Object.create(null, {
|
|
30
|
+
postRest: { get: () => super.postRest }
|
|
31
|
+
});
|
|
32
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
33
|
+
const body = {
|
|
34
|
+
prompt: prompt
|
|
35
|
+
};
|
|
36
|
+
const result = yield _super.postRest.call(this, `/generate/${model}`, body, undefined, undefined, {
|
|
37
|
+
maxReattempts: 0,
|
|
38
|
+
timeout: 30000
|
|
39
|
+
});
|
|
40
|
+
if (!commonsTypeIsString(result))
|
|
41
|
+
throw new Error('Result was not a string');
|
|
42
|
+
return result;
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=ollama-rest.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama-rest.service.js","sourceRoot":"","sources":["../../src/services/ollama-rest.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAkB,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAClG,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAIhE,MAAM,OAAO,iBAAkB,SAAQ,2BAA2B;IACjE,YACE,cAAuD,EACvD,GAAW,EACX,GAAW;QAEZ,KAAK,CACH,cAAc,EACd,GAAG,EACH,GAAG,EACH,uBAAuB,CAAC,IAAI,CAC7B,CAAC;IACH,CAAC;IAEY,UAAU;;;;;YACtB,MAAM,MAAM,GAAY,MAAM,OAAM,OAAO,YAAW,SAAS,CAAC,CAAC;YACjE,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAExF,OAAO,MAAM,CAAC;QACf,CAAC;KAAA;IAEY,QAAQ,CACnB,KAAa,EACb,MAAc;;;;;YAEf,MAAM,IAAI,GAAmB;gBAC3B,MAAM,EAAE,MAAM;aACf,CAAC;YAEF,MAAM,MAAM,GAAY,MAAM,OAAM,QAAQ,YAC1C,aAAa,KAAK,EAAE,EACpB,IAAI,EACJ,SAAS,EACT,SAAS,EACT;gBACE,aAAa,EAAE,CAAC;gBAChB,OAAO,EAAE,KAAK;aACf,CACF,CAAC;YACF,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAE7E,OAAO,MAAM,CAAC;QACf,CAAC;KAAA;CACD"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { CommonsFixedDuration } from 'tscommons-es-core';
|
|
2
|
+
import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
|
|
3
|
+
export declare class RigOffenceScaleService extends RigTicketedPromiseService {
|
|
4
|
+
private model;
|
|
5
|
+
constructor(url: string, key: string, model: string);
|
|
6
|
+
generateOffenceScore(text: string, keepAlive?: CommonsFixedDuration): Promise<number | undefined>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { commonsTypeAttemptNumber, commonsTypeIsString } from 'tscommons-es-core';
|
|
11
|
+
import { CommonsInternalHttpClientImplementation } from 'nodecommons-es-http';
|
|
12
|
+
import { OllamaRestService } from './ollama-rest.service';
|
|
13
|
+
import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
|
|
14
|
+
const PROMPT = `
|
|
15
|
+
The following text was extracted from a webpage as plain text:
|
|
16
|
+
|
|
17
|
+
\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`
|
|
18
|
+
<<<TEXT>>>
|
|
19
|
+
\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`
|
|
20
|
+
|
|
21
|
+
Generate a single numeric digit score for how offensive the text is, using the following scale:
|
|
22
|
+
0 = not offensive at all
|
|
23
|
+
1 = mildly offensive to a few people
|
|
24
|
+
2 = significantly offensive to most people
|
|
25
|
+
3 = extremely offensive, racist, sexist, homophobic or promoting hatred.
|
|
26
|
+
|
|
27
|
+
Output a single number by itself, with no other explanatory or introductory text.
|
|
28
|
+
`.trim();
|
|
29
|
+
export class RigOffenceScaleService extends RigTicketedPromiseService {
|
|
30
|
+
constructor(url, key, model) {
|
|
31
|
+
super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url, 100);
|
|
32
|
+
this.model = model;
|
|
33
|
+
}
|
|
34
|
+
generateOffenceScore(text, keepAlive) {
|
|
35
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
36
|
+
const prompt = PROMPT
|
|
37
|
+
.replace('<<<TEXT>>>', text);
|
|
38
|
+
const body = { prompt: prompt };
|
|
39
|
+
if (keepAlive) {
|
|
40
|
+
const mins = keepAlive.minutes;
|
|
41
|
+
if (mins < 1)
|
|
42
|
+
throw new Error('keepAlive cannot be less than 1 minute');
|
|
43
|
+
body.keepLoaded = `${Math.round(mins)}m`;
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
const response = yield this.submitAndWaitAsPromise(`/generate/${this.model}`, body, (data) => commonsTypeIsString(data), undefined, undefined, {
|
|
47
|
+
timeout: 30000,
|
|
48
|
+
maxReattempts: 0
|
|
49
|
+
});
|
|
50
|
+
return commonsTypeAttemptNumber(response);
|
|
51
|
+
}
|
|
52
|
+
catch (e) {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=rig-offence-scale.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rig-offence-scale.service.js","sourceRoot":"","sources":["../../src/services/rig-offence-scale.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAwB,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExG,OAAO,EAAE,uCAAuC,EAAE,MAAM,qBAAqB,CAAC;AAE9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,gCAAgC,CAAC;AAEjG,MAAM,MAAM,GAAW;;;;;;;;;;;;;;CActB,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,OAAO,sBAAuB,SAAQ,yBAAyB;IACpE,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,EACH,GAAG,CACJ,CAAC;QAVO,UAAK,GAAL,KAAK,CAAQ;IAWvB,CAAC;IAEY,oBAAoB,CAC/B,IAAY,EACZ,SAAgC;;YAEjC,MAAM,MAAM,GAAW,MAAM;iBAC1B,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YACtD,IAAI,SAAS,EAAE;gBACd,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;gBACvC,IAAI,IAAI,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;gBAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;aACzC;YAED,IAAI;gBACH,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;oBACE,OAAO,EAAE,KAAK;oBACd,aAAa,EAAE,CAAC;iBACjB,CACF,CAAC;gBAEF,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAC;aAC1C;YAAC,OAAO,CAAC,EAAE;gBACX,OAAO,SAAS,CAAC;aACjB;QACF,CAAC;KAAA;CACD"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { TEncoded, TEncodedObject } from 'tscommons-es-core';
|
|
2
|
+
import { CommonsRestClientService } from 'tscommons-es-rest';
|
|
3
|
+
import { TCommonsHttpRequestOptions } from 'tscommons-es-http';
|
|
4
|
+
export type TPromptAndKeepLoaded = {
|
|
5
|
+
prompt: string;
|
|
6
|
+
keepLoaded?: string;
|
|
7
|
+
};
|
|
8
|
+
export declare abstract class RigTicketedPromiseService {
|
|
9
|
+
private restService;
|
|
10
|
+
private socketIoUrl;
|
|
11
|
+
private connectToSubmitDelay;
|
|
12
|
+
constructor(restService: CommonsRestClientService, socketIoUrl: string, connectToSubmitDelay?: number);
|
|
13
|
+
submitAndWaitAsPromise<ResultT extends TEncoded = TEncoded, BodyT extends (TEncodedObject & TPromptAndKeepLoaded) = (TEncodedObject & TPromptAndKeepLoaded), ParamsT extends TEncodedObject = TEncodedObject, HeadersT extends TEncodedObject = TEncodedObject>(script: string, body: BodyT, checker: (data: unknown) => data is ResultT, params?: ParamsT | undefined, // the params can be explicitly undefined (none) as well as optional
|
|
14
|
+
headers?: HeadersT | undefined, // the headers can be explicitly undefined (none) as well as optional
|
|
15
|
+
options?: TCommonsHttpRequestOptions): Promise<ResultT>;
|
|
16
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { commonsTicketedGenericSubmitAndWaitAsPromise } from 'nodecommons-es-socket-io-ticketed';
|
|
2
|
+
export class RigTicketedPromiseService {
|
|
3
|
+
constructor(restService, socketIoUrl, connectToSubmitDelay = 100) {
|
|
4
|
+
this.restService = restService;
|
|
5
|
+
this.socketIoUrl = socketIoUrl;
|
|
6
|
+
this.connectToSubmitDelay = connectToSubmitDelay;
|
|
7
|
+
}
|
|
8
|
+
submitAndWaitAsPromise(script, body, checker, params, // the params can be explicitly undefined (none) as well as optional
|
|
9
|
+
headers, // the headers can be explicitly undefined (none) as well as optional
|
|
10
|
+
options = {}) {
|
|
11
|
+
return commonsTicketedGenericSubmitAndWaitAsPromise(this.socketIoUrl, 'rig-marshall', this.restService, script, body, checker, params, headers, options, this.connectToSubmitDelay);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=rig-ticketed-promise.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rig-ticketed-promise.service.js","sourceRoot":"","sources":["../../src/services/rig-ticketed-promise.service.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,4CAA4C,EAAE,MAAM,mCAAmC,CAAC;AAOjG,MAAM,OAAgB,yBAAyB;IAC9C,YACU,WAAqC,EACrC,WAAmB,EACnB,uBAA+B,GAAG;QAFlC,gBAAW,GAAX,WAAW,CAA0B;QACrC,gBAAW,GAAX,WAAW,CAAQ;QACnB,yBAAoB,GAApB,oBAAoB,CAAc;IACzC,CAAC;IAEG,sBAAsB,CAM3B,MAAc,EACd,IAAW,EACX,OAA2C,EAC3C,MAA0B,EAAE,oEAAoE;IAChG,OAA4B,EAAE,qEAAqE;IACnG,UAAsC,EAAE;QAEzC,OAAO,4CAA4C,CAMjD,IAAI,CAAC,WAAW,EAChB,cAAc,EACd,IAAI,CAAC,WAAW,EAChB,MAAM,EACN,IAAI,EACJ,OAAO,EACP,MAAM,EACN,OAAO,EACP,OAAO,EACP,IAAI,CAAC,oBAAoB,CAC1B,CAAC;IACH,CAAC;CACD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hydra-crawler",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Node.js Hydra web crawler",
|
|
5
5
|
"author": "Pete Morris",
|
|
6
6
|
"license": "ISC",
|
|
@@ -25,29 +25,31 @@
|
|
|
25
25
|
"memcached-promisify": "^2.0.0",
|
|
26
26
|
"mongodb": "^5.0.1",
|
|
27
27
|
"nodecommons-es-app": "^1.1.2",
|
|
28
|
-
"nodecommons-es-app-socket-io": "3.0.
|
|
28
|
+
"nodecommons-es-app-socket-io": "3.0.3",
|
|
29
29
|
"nodecommons-es-cli": "^2.1.1",
|
|
30
30
|
"nodecommons-es-config": "^1.0.8",
|
|
31
31
|
"nodecommons-es-database": "^2.2.8",
|
|
32
32
|
"nodecommons-es-database-mongodb": "^1.0.9",
|
|
33
|
-
"nodecommons-es-express": "^4.0.
|
|
33
|
+
"nodecommons-es-express": "^4.0.3",
|
|
34
34
|
"nodecommons-es-file": "^1.1.5",
|
|
35
|
-
"nodecommons-es-http": "^2.4.
|
|
35
|
+
"nodecommons-es-http": "^2.4.9",
|
|
36
36
|
"nodecommons-es-process": "^1.4.1",
|
|
37
|
-
"nodecommons-es-rest": "^2.0.
|
|
37
|
+
"nodecommons-es-rest": "^2.0.13",
|
|
38
38
|
"nodecommons-es-security": "^1.0.10",
|
|
39
|
-
"nodecommons-es-socket-io": "3.0.
|
|
39
|
+
"nodecommons-es-socket-io": "3.0.7",
|
|
40
|
+
"nodecommons-es-socket-io-ticketed": "^0.0.11",
|
|
40
41
|
"tscommons-es-async": "^1.2.4",
|
|
41
42
|
"tscommons-es-config": "^1.0.6",
|
|
42
|
-
"tscommons-es-core": "^1.
|
|
43
|
+
"tscommons-es-core": "^1.25.1",
|
|
43
44
|
"tscommons-es-format": "1.1.1",
|
|
45
|
+
"tscommons-es-rest": "^1.0.11",
|
|
44
46
|
"unfluff": "^3.2.0"
|
|
45
47
|
},
|
|
46
48
|
"devDependencies": {
|
|
47
|
-
"@types/cheerio": "0.22.32",
|
|
48
|
-
"@types/mongodb": "^4.0.6",
|
|
49
49
|
"@types/chardet": "^0.8.1",
|
|
50
|
+
"@types/cheerio": "0.22.32",
|
|
50
51
|
"@types/express": "^4.17.17",
|
|
52
|
+
"@types/mongodb": "^4.0.6",
|
|
51
53
|
"@types/node": "18.6.1",
|
|
52
54
|
"@typescript-eslint/eslint-plugin": "^5.53.0",
|
|
53
55
|
"@typescript-eslint/parser": "^5.53.0",
|