hydra-crawler 2.4.6 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -37,6 +37,7 @@ import { AspErrorParser } from '../parsers/asp-error.parser';
37
37
  import { ComplexEnglishParser } from '../parsers/complex-english.parser';
38
38
  import { AccessibilityMetricsParser } from '../parsers/accessibility-metrics.parser';
39
39
  import { OffenceParser } from '../parsers/offence.parser';
40
+ import { InterestParser } from '../parsers/interest.parser';
40
41
  import { isIParserConfig } from '../interfaces/iparser-config';
41
42
  import { isTHydraConfig } from '../types/thydra-config';
42
43
  import { isTCrawlConfig } from '../types/tcrawl-config';
@@ -143,7 +144,8 @@ export class HydraApp extends CommonsSocketIoApp {
143
144
  AspErrorParser,
144
145
  ComplexEnglishParser,
145
146
  AccessibilityMetricsParser,
146
- OffenceParser
147
+ OffenceParser,
148
+ InterestParser
147
149
  ]) {
148
150
  commonsOutputDebug(`Installing parser ${parser.name}`);
149
151
  this.crawlServer.addParser(parser);
@@ -1 +1 @@
1
- {"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAGxE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACrH,OAAO,EAAE,gCAAgC,EAA0D,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAG3J,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yCAAyC,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI9E,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAUzE,MAAM,OAAO,QAAS,SAAQ,kBAAkC;IAkB/D;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QANhB,qBAAgB,GAA2B,EAAE,CAAC;QAQrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QAEnC,MAAM,YAAY,GAAY,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IAEkB,eAAe;QACjC,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAEvF,OAAO,gCAAgC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAC9F,CAAC;IAES,mBAAmB,CAC3B,aAAyC,EACzC,aAAoC;QAErC,OAAO,IAAI,cAAc,CACvB,aAAa,EACb,aAAa,CACd,CAAC;IACH,CAAC;IAEM,UAAU;QAChB,OAAO,OAAO,CAAC;IAChB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEM,sBAAsB,CAC3B,KAA6B;QAE9B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,CAAC,OAAO,CACV,CAAC,UAAgF,EAAE,IAAY,EAAQ,EAAE;gBACxG,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAExF,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;gBACnD,IAAI,eAAe,CACjB,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;gBAC3C,IAAI,QAAQ,CACV,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;gBAC7C,IAAI,UAAU,CACZ,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;gBAC5C,IAAI,SAAS,CACX,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAChD,IAAI,aAAa,CACf,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACxB,CAAC,CACF,CAAC;YAEF,MAAM,OAAM,IAAI,WAAE,CAAC;YAEnB,MAAM,cAAc,GAA6B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAE7E,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,IAAI,CAAC,eAAe,EACpB,cAAc,CACf,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;YAEF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI;gBACnB,YAAY;gBACZ,gBAAgB;gBAChB,eAAe;gBACf,UAAU;gBACV,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,oBAAoB;gBACpB,0BAA0B;gBAC1B,aAAa;aACd,EAAE;gBACF,kBAAkB,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC9C;YAED,MAAM,oBAAoB,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,wBAAwB,CAAC,CAAC;YAC7G,IAAI,oBAAoB;gBAAE,kBAAkB,CAAC,qCAAqC,oBAAoB,EAAE,CAAC,CAAC;YAE1G,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,oBAAoB,CAAC,2DAA2D;aACjF,CAAC;QACH,CAAC;KAAA;IAES,SAAS;QAClB,KAAK,CAAC,GAAwB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAA,CAAC,EAAE,CAAC;IACN,CAAC;IAEY,GAAG;;;;;YACf,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,SAAS,EAAE;gBACd,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAE5E,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;gBAEvC,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,CAAC,QAAQ,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAC9B,IAAI,CAAC,KAAK,CACX,CAAC;gBAEF,iBAAiB,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAW,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;gBAE7D,OAAO;aACP;YAED,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAChG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;YAErH,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAE1B,MAAM,OAAM,GAAG,WAAE,CAAC;QACnB,CAAC;KAAA;IAEe,QAAQ;;;;;YACvB,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAElC,MAAM,OAAM,QAAQ,WAAE,CAAC;QACxB,CAAC;KAAA;CACD"}
1
+ {"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAGxE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACrH,OAAO,EAAE,gCAAgC,EAA0D,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAG3J,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yCAAyC,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAE5D,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI9E,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAUzE,MAAM,OAAO,QAAS,SAAQ,kBAAkC;IAkB/D;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QANhB,qBAAgB,GAA2B,EAAE,CAAC;QAQrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QAEnC,MAAM,YAAY,GAAY,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IAEkB,eAAe;QACjC,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAEvF,OAAO,gCAAgC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAC9F,CAAC;IAES,mBAAmB,CAC3B,aAAyC,EACzC,aAAoC;QAErC,OAAO,IAAI,cAAc,CACvB,aAAa,EACb,aAAa,CACd,CAAC;IACH,CAAC;IAEM,UAAU;QAChB,OAAO,OAAO,CAAC;IAChB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEM,sBAAsB,CAC3B,KAA6B;QAE9B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,CAAC,OAAO,CACV,CAAC,UAAgF,EAAE,IAAY,EAAQ,EAAE;gBACxG,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAExF,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;gBACnD,IAAI,eAAe,CACjB,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;gBAC3C,IAAI,QAAQ,CACV,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;gBAC7C,IAAI,UAAU,CACZ,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;gBAC5C,IAAI,SAAS,CACX,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAChD,IAAI,aAAa,CACf,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACxB,CAAC,CACF,CAAC;YAEF,MAAM,OAAM,IAAI,WAAE,CAAC;YAEnB,MAAM,cAAc,GAA6B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAE7E,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,IAAI,CAAC,eAAe,EACpB,cAAc,CACf,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;YAEF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI;gBACnB,YAAY;gBACZ,gBAAgB;gBAChB,eAAe;gBACf,UAAU;gBACV,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,oBAAoB;gBACpB,0BAA0B;gBAC1B,aAAa;gBACb,cAAc;aACf,EAAE;gBACF,kBAAkB,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC9C;YAED,MAAM,oBAAoB,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,wBAAwB,CAAC,CAAC;YAC7G,IAAI,oBAAoB;gBAAE,kBAAkB,CAAC,qCAAqC,oBAAoB,EAAE,CAAC,CAAC;YAE1G,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,oBAAoB,CAAC,2DAA2D;aACjF,CAAC;QACH,CAAC;KAAA;IAES,SAAS;QAClB,KAAK,CAAC,GAAwB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAA,CAAC,EAAE,CAAC;IACN,CAAC;IAEY,GAAG;;;;;YACf,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,SAAS,EAAE;gBACd,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAE5E,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;gBAEvC,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,CAAC,QAAQ,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAC9B,IAAI,CAAC,KAAK,CACX,CAAC;gBAEF,iBAAiB,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAW,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;gBAE7D,OAAO;aACP;YAED,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAChG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;YAErH,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAE1B,MAAM,OAAM,GAAG,WAAE,CAAC;QACnB,CAAC;KAAA;IAEe,QAAQ;;;;;YACvB,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAElC,MAAM,OAAM,QAAQ,WAAE,CAAC;QACxB,CAAC;KAAA;CACD"}
@@ -0,0 +1,18 @@
1
+ import { TKeyObject } from 'tscommons-es-core';
2
+ import { DatabaseService } from '../services/database.service';
3
+ import { IParserConfig } from '../interfaces/iparser-config';
4
+ import { IRequestOutcome } from '../interfaces/irequest-outcome';
5
+ import { IOllamaConfig, OllamaParser } from './ollama.parser';
6
+ import { IDataConfig } from './data.parser';
7
+ export interface IInterestConfig extends IDataConfig {
8
+ model: string;
9
+ permittedDomains: string[];
10
+ }
11
+ export declare function isIInterestConfig(test: unknown): test is IInterestConfig;
12
+ export declare class InterestParser extends OllamaParser<IOllamaConfig> {
13
+ private url?;
14
+ private model;
15
+ private permittedDomains;
16
+ constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
17
+ protected parseText(database: DatabaseService, text: string): Promise<void>;
18
+ }
@@ -0,0 +1,64 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
11
+ import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
12
+ import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service';
13
+ import { OllamaParser } from './ollama.parser';
14
+ import { isIDataConfig } from './data.parser';
15
+ export function isIInterestConfig(test) {
16
+ if (!isIDataConfig(test))
17
+ return false;
18
+ if (!commonsTypeHasPropertyString(test, 'model'))
19
+ return false;
20
+ if (!commonsTypeHasPropertyStringArray(test, 'permittedDomains'))
21
+ return false;
22
+ return true;
23
+ }
24
+ export class InterestParser extends OllamaParser {
25
+ constructor(url, outcome, config) {
26
+ super(outcome, config, 'interest');
27
+ this.url = url;
28
+ this.permittedDomains = [];
29
+ if (!config)
30
+ return;
31
+ if (!commonsTypeHasPropertyT(config, 'interest', isIInterestConfig)) {
32
+ commonsOutputError('Invalid config for InterestParser');
33
+ }
34
+ this.model = config['interest'].model;
35
+ this.permittedDomains = config['interest'].permittedDomains;
36
+ }
37
+ parseText(database, text) {
38
+ return __awaiter(this, void 0, void 0, function* () {
39
+ if (!this.url)
40
+ return;
41
+ if (!this.ollamaUrl || !this.ollamaKey || !this.model)
42
+ return;
43
+ const hostname = new URL(this.url).hostname;
44
+ if (!this.permittedDomains.includes(hostname))
45
+ return;
46
+ if (text.length === 0) {
47
+ yield database.unsetData(this.url, 'interest');
48
+ return;
49
+ }
50
+ const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
51
+ // run this outside of await so that it doesn't block other things
52
+ void (() => __awaiter(this, void 0, void 0, function* () {
53
+ if (!this.url)
54
+ return;
55
+ const score = yield rigService.generateInterestScore(text, this.ollamaKeepAlive);
56
+ if (score === undefined)
57
+ return;
58
+ commonsOutputDebug(`Interest score is: ${score.toString(10)}`);
59
+ yield database.setData(this.url, 'interest', score);
60
+ }))();
61
+ });
62
+ }
63
+ }
64
+ //# sourceMappingURL=interest.parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"interest.parser.js","sourceRoot":"","sources":["../../src/parsers/interest.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAE,MAAM,4CAA4C,CAAC;AAKzF,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC9C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,cAAe,SAAQ,YAA2B;IAI9D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,UAAU,CACX,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAkB,MAAM,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE;YACrF,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;SACxD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,gBAAgB,CAAC;IAClF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC/C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAqB,MAAM,UAAU,CAAC,qBAAqB,CACpE,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,kBAAkB,CAAC,sBAAsB,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC/D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}
@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
9
9
  };
10
10
  import { commonsTypeHasPropertyString, commonsTypeHasPropertyStringArray, commonsTypeHasPropertyT } from 'tscommons-es-core';
11
11
  import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
12
- import { RigOffenceScaleService } from '../services/rig-offence-scale.service';
12
+ import { RigSubjectivityScaleService } from '../services/rig-subjectivity-scale.service';
13
13
  import { OllamaParser } from './ollama.parser';
14
14
  import { isIDataConfig } from './data.parser';
15
15
  export function isIOffenceConfig(test) {
@@ -47,12 +47,12 @@ export class OffenceParser extends OllamaParser {
47
47
  yield database.unsetData(this.url, 'offence');
48
48
  return;
49
49
  }
50
- const rigOffenceService = new RigOffenceScaleService(this.ollamaUrl, this.ollamaKey, this.model);
50
+ const rigService = new RigSubjectivityScaleService(this.ollamaUrl, this.ollamaKey, this.model);
51
51
  // run this outside of await so that it doesn't block other things
52
52
  void (() => __awaiter(this, void 0, void 0, function* () {
53
53
  if (!this.url)
54
54
  return;
55
- const score = yield rigOffenceService.generateOffenceScore(text, this.ollamaKeepAlive);
55
+ const score = yield rigService.generateOffenceScore(text, this.ollamaKeepAlive);
56
56
  if (score === undefined)
57
57
  return;
58
58
  commonsOutputDebug(`Offence score is: ${score.toString(10)}`);
@@ -1 +1 @@
1
- {"version":3,"file":"offence.parser.js","sourceRoot":"","sources":["../../src/parsers/offence.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAK/E,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAI7D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE;YAClF,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;SACvD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,gBAAgB,CAAC;IAChF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC9C,OAAO;aACP;YAED,MAAM,iBAAiB,GAA2B,IAAI,sBAAsB,CAC1E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAqB,MAAM,iBAAiB,CAAC,oBAAoB,CAC1E,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,kBAAkB,CAAC,qBAAqB,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}
1
+ {"version":3,"file":"offence.parser.js","sourceRoot":"","sources":["../../src/parsers/offence.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAE,MAAM,4CAA4C,CAAC;AAKzF,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAI7D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE;YAClF,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;SACvD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,gBAAgB,CAAC;IAChF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC9C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAqB,MAAM,UAAU,CAAC,oBAAoB,CACnE,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,kBAAkB,CAAC,qBAAqB,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC9D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}
@@ -1,7 +1,9 @@
1
1
  import { CommonsFixedDuration } from 'tscommons-es-core';
2
2
  import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
3
- export declare class RigOffenceScaleService extends RigTicketedPromiseService {
3
+ export declare class RigSubjectivityScaleService extends RigTicketedPromiseService {
4
4
  private model;
5
5
  constructor(url: string, key: string, model: string);
6
+ private generateSubjectivityScore;
6
7
  generateOffenceScore(text: string, keepAlive?: CommonsFixedDuration): Promise<number | undefined>;
8
+ generateInterestScore(text: string, keepAlive?: CommonsFixedDuration): Promise<number | undefined>;
7
9
  }
@@ -11,29 +11,42 @@ import { commonsTypeAttemptNumber, commonsTypeIsString } from 'tscommons-es-core
11
11
  import { CommonsInternalHttpClientImplementation } from 'nodecommons-es-http';
12
12
  import { OllamaRestService } from './ollama-rest.service';
13
13
  import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
14
- const PROMPT = `
14
+ const GENERIC_PROMPT = `
15
15
  The following text was extracted from a webpage as plain text:
16
16
 
17
17
  \`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`
18
18
  <<<TEXT>>>
19
19
  \`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`\`
20
20
 
21
- Generate a single numeric digit score for how offensive or controversial the text is, using the following scale:
21
+ Generate a single numeric digit score <<REASON>>, using the following scale:
22
+ <<SCALE>>
23
+
24
+ Output a single number by itself, with no other explanatory or introductory text.
25
+ `.trim();
26
+ const OFFENCE_PROMPT = GENERIC_PROMPT
27
+ .replace('<<<REASON>>>', 'for how offensive or controversial the text is')
28
+ .replace('SCALE', `
22
29
  0 = not offensive or controversial at all
23
30
  1 = mildly offensive or controversial to a few people
24
31
  2 = significantly offensive or controversial to most people
25
32
  3 = extremely offensive or highly controversial, potentially racist, sexist, homophobic or promoting hatred.
26
-
27
- Output a single number by itself, with no other explanatory or introductory text.
28
- `.trim();
29
- export class RigOffenceScaleService extends RigTicketedPromiseService {
33
+ `.trim());
34
+ const INTEREST_PROMPT = GENERIC_PROMPT
35
+ .replace('<<<REASON>>>', 'for how interesting or entertaining the text is')
36
+ .replace('SCALE', `
37
+ 0 = purely function information with no further interest, such as opening times or directions
38
+ 1 = information with a little interest or intrigue, such as opinions and informative descriptions
39
+ 2 = opinion and commentary which is likely to be very interesting to many people
40
+ 3 = highly informative and interesting or entertaining commentary, which most people would rate highly.
41
+ `.trim());
42
+ export class RigSubjectivityScaleService extends RigTicketedPromiseService {
30
43
  constructor(url, key, model) {
31
44
  super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url, 100);
32
45
  this.model = model;
33
46
  }
34
- generateOffenceScore(text, keepAlive) {
47
+ generateSubjectivityScore(text, prompt, keepAlive) {
35
48
  return __awaiter(this, void 0, void 0, function* () {
36
- const prompt = PROMPT
49
+ prompt = prompt
37
50
  .replace('<<<TEXT>>>', text);
38
51
  const body = { prompt: prompt };
39
52
  if (keepAlive) {
@@ -54,5 +67,15 @@ export class RigOffenceScaleService extends RigTicketedPromiseService {
54
67
  }
55
68
  });
56
69
  }
70
+ generateOffenceScore(text, keepAlive) {
71
+ return __awaiter(this, void 0, void 0, function* () {
72
+ return yield this.generateSubjectivityScore(text, OFFENCE_PROMPT, keepAlive);
73
+ });
74
+ }
75
+ generateInterestScore(text, keepAlive) {
76
+ return __awaiter(this, void 0, void 0, function* () {
77
+ return yield this.generateSubjectivityScore(text, INTEREST_PROMPT, keepAlive);
78
+ });
79
+ }
57
80
  }
58
- //# sourceMappingURL=rig-offence-scale.service.js.map
81
+ //# sourceMappingURL=rig-subjectivity-scale.service.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rig-subjectivity-scale.service.js","sourceRoot":"","sources":["../../src/services/rig-subjectivity-scale.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAwB,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExG,OAAO,EAAE,uCAAuC,EAAE,MAAM,qBAAqB,CAAC;AAE9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,gCAAgC,CAAC;AAEjG,MAAM,cAAc,GAAW;;;;;;;;;;;CAW9B,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,cAAc,GAAW,cAAc;KAC1C,OAAO,CACN,cAAc,EACd,gDAAgD,CACjD;KACA,OAAO,CACN,OAAO,EACP;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,eAAe,GAAW,cAAc;KAC3C,OAAO,CACN,cAAc,EACd,iDAAiD,CAClD;KACA,OAAO,CACN,OAAO,EACP;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,OAAO,2BAA4B,SAAQ,yBAAyB;IACzE,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,EACH,GAAG,CACJ,CAAC;QAVO,UAAK,GAAL,KAAK,CAAQ;IAWvB,CAAC;IAEa,yBAAyB,CACrC,IAAY,EACZ,MAAc,EACd,SAAgC;;YAEjC,MAAM,GAAG,MAAM;iBACZ,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YACtD,IAAI,SAAS,EAAE;gBACd,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;gBACvC,IAAI,IAAI,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;gBAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;aACzC;YAED,IAAI;gBACH,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;oBACE,OAAO,EAAE,KAAK;oBACd,aAAa,EAAE,CAAC;iBACjB,CACF,CAAC;gBAEF,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAC;aAC1C;YAAC,OAAO,CAAC,EAAE;gBACX,OAAO,SAAS,CAAC;aACjB;QACF,CAAC;KAAA;IAEY,oBAAoB,CAC/B,IAAY,EACZ,SAAgC;;YAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,cAAc,EACd,SAAS,CACV,CAAC;QACH,CAAC;KAAA;IAEY,qBAAqB,CAChC,IAAY,EACZ,SAAgC;;YAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,eAAe,EACf,SAAS,CACV,CAAC;QACH,CAAC;KAAA;CACD"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hydra-crawler",
3
- "version": "2.4.6",
3
+ "version": "2.5.0",
4
4
  "description": "Node.js Hydra web crawler",
5
5
  "author": "Pete Morris",
6
6
  "license": "ISC",
@@ -1 +0,0 @@
1
- {"version":3,"file":"rig-offence-scale.service.js","sourceRoot":"","sources":["../../src/services/rig-offence-scale.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAwB,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExG,OAAO,EAAE,uCAAuC,EAAE,MAAM,qBAAqB,CAAC;AAE9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,gCAAgC,CAAC;AAEjG,MAAM,MAAM,GAAW;;;;;;;;;;;;;;CActB,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,OAAO,sBAAuB,SAAQ,yBAAyB;IACpE,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,EACH,GAAG,CACJ,CAAC;QAVO,UAAK,GAAL,KAAK,CAAQ;IAWvB,CAAC;IAEY,oBAAoB,CAC/B,IAAY,EACZ,SAAgC;;YAEjC,MAAM,MAAM,GAAW,MAAM;iBAC1B,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YACtD,IAAI,SAAS,EAAE;gBACd,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;gBACvC,IAAI,IAAI,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;gBAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;aACzC;YAED,IAAI;gBACH,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;oBACE,OAAO,EAAE,KAAK;oBACd,aAAa,EAAE,CAAC;iBACjB,CACF,CAAC;gBAEF,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAC;aAC1C;YAAC,OAAO,CAAC,EAAE;gBACX,OAAO,SAAS,CAAC;aACjB;QACF,CAAC;KAAA;CACD"}