hydra-crawler 2.5.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/parsers/interest.parser.js +5 -1
- package/dist/parsers/interest.parser.js.map +1 -1
- package/dist/parsers/offence.parser.js +5 -1
- package/dist/parsers/offence.parser.js.map +1 -1
- package/dist/services/rig-subjectivity-scale.service.d.ts +6 -2
- package/dist/services/rig-subjectivity-scale.service.js +18 -3
- package/dist/services/rig-subjectivity-scale.service.js.map +1 -1
- package/package.json +1 -1
|
@@ -55,7 +55,11 @@ export class InterestParser extends OllamaParser {
|
|
|
55
55
|
const score = yield rigService.generateInterestScore(text, this.ollamaKeepAlive);
|
|
56
56
|
if (score === undefined)
|
|
57
57
|
return;
|
|
58
|
-
|
|
58
|
+
if (score.score < 2)
|
|
59
|
+
delete score.reason;
|
|
60
|
+
commonsOutputDebug(`Interest score is: ${score.score.toString(10)}`);
|
|
61
|
+
if (score.reason)
|
|
62
|
+
commonsOutputDebug(`Reason for score is: ${score.reason}`);
|
|
59
63
|
yield database.setData(this.url, 'interest', score);
|
|
60
64
|
}))();
|
|
61
65
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"interest.parser.js","sourceRoot":"","sources":["../../src/parsers/interest.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,
|
|
1
|
+
{"version":3,"file":"interest.parser.js","sourceRoot":"","sources":["../../src/parsers/interest.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAuB,MAAM,4CAA4C,CAAC;AAK9G,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC9C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,cAAe,SAAQ,YAA2B;IAI9D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,UAAU,CACX,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAkB,MAAM,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE;YACrF,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;SACxD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,KAAK,CAAC;QAC3D,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,UAAU,CAAqB,CAAC,gBAAgB,CAAC;IAClF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC/C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,qBAAqB,CACjF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;gBAEzC,kBAAkB,CAAC,sBAAsB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,CAAC,MAAM;oBAAE,kBAAkB,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE7E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACrD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}
|
|
@@ -55,7 +55,11 @@ export class OffenceParser extends OllamaParser {
|
|
|
55
55
|
const score = yield rigService.generateOffenceScore(text, this.ollamaKeepAlive);
|
|
56
56
|
if (score === undefined)
|
|
57
57
|
return;
|
|
58
|
-
|
|
58
|
+
if (score.score < 1)
|
|
59
|
+
delete score.reason;
|
|
60
|
+
commonsOutputDebug(`Offence score is: ${score.score.toString(10)}`);
|
|
61
|
+
if (score.reason)
|
|
62
|
+
commonsOutputDebug(`Reason for score is: ${score.reason}`);
|
|
59
63
|
yield database.setData(this.url, 'offence', score);
|
|
60
64
|
}))();
|
|
61
65
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"offence.parser.js","sourceRoot":"","sources":["../../src/parsers/offence.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,
|
|
1
|
+
{"version":3,"file":"offence.parser.js","sourceRoot":"","sources":["../../src/parsers/offence.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAc,4BAA4B,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAEzI,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAAE,2BAA2B,EAAuB,MAAM,4CAA4C,CAAC;AAK9G,OAAO,EAAiB,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAM3D,MAAM,UAAU,gBAAgB,CAAC,IAAa;IAC7C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,YAA2B;IAI7D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,SAAS,CACV,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;QAHd,qBAAgB,GAAa,EAAE,CAAC;QAavC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAiB,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,EAAE;YAClF,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;SACvD;QACD,IAAI,CAAC,KAAK,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,KAAK,CAAC;QACzD,IAAI,CAAC,gBAAgB,GAAI,MAAM,CAAC,SAAS,CAAoB,CAAC,gBAAgB,CAAC;IAChF,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YACtB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,OAAO;YAE9D,MAAM,QAAQ,GAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEtD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC9C,OAAO;aACP;YAED,MAAM,UAAU,GAAgC,IAAI,2BAA2B,CAC7E,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,kEAAkE;YAClE,KAAK,CAAC,GAAwB,EAAE;gBAC/B,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAEtB,MAAM,KAAK,GAAkC,MAAM,UAAU,CAAC,oBAAoB,CAChF,IAAI,EACJ,IAAI,CAAC,eAAe,CACrB,CAAC;gBACF,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO;gBAEhC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;gBAEzC,kBAAkB,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACpE,IAAI,KAAK,CAAC,MAAM;oBAAE,kBAAkB,CAAC,wBAAwB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE7E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC,CAAA,CAAC,EAAE,CAAC;QACN,CAAC;KAAA;CACD"}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { CommonsFixedDuration } from 'tscommons-es-core';
|
|
2
2
|
import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
|
|
3
|
+
export type TSubjectivityResult = {
|
|
4
|
+
score: number;
|
|
5
|
+
reason?: string;
|
|
6
|
+
};
|
|
3
7
|
export declare class RigSubjectivityScaleService extends RigTicketedPromiseService {
|
|
4
8
|
private model;
|
|
5
9
|
constructor(url: string, key: string, model: string);
|
|
6
10
|
private generateSubjectivityScore;
|
|
7
|
-
generateOffenceScore(text: string, keepAlive?: CommonsFixedDuration): Promise<
|
|
8
|
-
generateInterestScore(text: string, keepAlive?: CommonsFixedDuration): Promise<
|
|
11
|
+
generateOffenceScore(text: string, keepAlive?: CommonsFixedDuration): Promise<TSubjectivityResult | undefined>;
|
|
12
|
+
generateInterestScore(text: string, keepAlive?: CommonsFixedDuration): Promise<TSubjectivityResult | undefined>;
|
|
9
13
|
}
|
|
@@ -7,10 +7,17 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
-
import {
|
|
10
|
+
import { commonsTypeAttemptObject, commonsTypeHasPropertyNumber, commonsTypeHasPropertyStringOrUndefined, commonsTypeIsString } from 'tscommons-es-core';
|
|
11
11
|
import { CommonsInternalHttpClientImplementation } from 'nodecommons-es-http';
|
|
12
12
|
import { OllamaRestService } from './ollama-rest.service';
|
|
13
13
|
import { RigTicketedPromiseService } from './rig-ticketed-promise.service';
|
|
14
|
+
function isTSubjectivityResult(test) {
|
|
15
|
+
if (!commonsTypeHasPropertyNumber(test, 'score'))
|
|
16
|
+
return false;
|
|
17
|
+
if (!commonsTypeHasPropertyStringOrUndefined(test, 'reason'))
|
|
18
|
+
return false;
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
14
21
|
const GENERIC_PROMPT = `
|
|
15
22
|
The following text was extracted from a webpage as plain text:
|
|
16
23
|
|
|
@@ -21,7 +28,10 @@ The following text was extracted from a webpage as plain text:
|
|
|
21
28
|
Generate a single numeric digit score <<REASON>>, using the following scale:
|
|
22
29
|
<<SCALE>>
|
|
23
30
|
|
|
24
|
-
Output a single
|
|
31
|
+
Output the score and a single sentence justifying the reason for that score, using the JSON format:
|
|
32
|
+
{ "score": <score>, "reason": "<reason>"}
|
|
33
|
+
|
|
34
|
+
Do not include any other prefix or suffix explanatory text.
|
|
25
35
|
`.trim();
|
|
26
36
|
const OFFENCE_PROMPT = GENERIC_PROMPT
|
|
27
37
|
.replace('<<<REASON>>>', 'for how offensive or controversial the text is')
|
|
@@ -60,7 +70,12 @@ export class RigSubjectivityScaleService extends RigTicketedPromiseService {
|
|
|
60
70
|
timeout: 30000,
|
|
61
71
|
maxReattempts: 0
|
|
62
72
|
});
|
|
63
|
-
|
|
73
|
+
const attempt = commonsTypeAttemptObject(response);
|
|
74
|
+
if (!attempt)
|
|
75
|
+
return undefined;
|
|
76
|
+
if (!isTSubjectivityResult(attempt))
|
|
77
|
+
return undefined;
|
|
78
|
+
return attempt;
|
|
64
79
|
}
|
|
65
80
|
catch (e) {
|
|
66
81
|
return undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rig-subjectivity-scale.service.js","sourceRoot":"","sources":["../../src/services/rig-subjectivity-scale.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"rig-subjectivity-scale.service.js","sourceRoot":"","sources":["../../src/services/rig-subjectivity-scale.service.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAyC,wBAAwB,EAAE,4BAA4B,EAAE,uCAAuC,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAEhM,OAAO,EAAE,uCAAuC,EAAE,MAAM,qBAAqB,CAAC;AAE9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,gCAAgC,CAAC;AAMjG,SAAS,qBAAqB,CAAC,IAAa;IAC3C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,cAAc,GAAW;;;;;;;;;;;;;;CAc9B,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,cAAc,GAAW,cAAc;KAC1C,OAAO,CACN,cAAc,EACd,gDAAgD,CACjD;KACA,OAAO,CACN,OAAO,EACP;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,eAAe,GAAW,cAAc;KAC3C,OAAO,CACN,cAAc,EACd,iDAAiD,CAClD;KACA,OAAO,CACN,OAAO,EACP;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,OAAO,2BAA4B,SAAQ,yBAAyB;IACzE,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,EACH,GAAG,CACJ,CAAC;QAVO,UAAK,GAAL,KAAK,CAAQ;IAWvB,CAAC;IAEa,yBAAyB,CACrC,IAAY,EACZ,MAAc,EACd,SAAgC;;YAEjC,MAAM,GAAG,MAAM;iBACZ,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YACtD,IAAI,SAAS,EAAE;gBACd,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;gBACvC,IAAI,IAAI,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;gBAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;aACzC;YAED,IAAI;gBACH,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;oBACE,OAAO,EAAE,KAAK;oBACd,aAAa,EAAE,CAAC;iBACjB,CACF,CAAC;gBACF,MAAM,OAAO,GAAuC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;gBACvF,IAAI,CAAC,OAAO;oBAAE,OAAO,SAAS,CAAC;gBAE/B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC;oBAAE,OAAO,SAAS,CAAC;gBAEtD,OAAO,OAAO,CAAC;aACf;YAAC,OAAO,CAAC,EAAE;gBACX,OAAO,SAAS,CAAC;aACjB;QACF,CAAC;KAAA;IAEY,oBAAoB,CAC/B,IAAY,EACZ,SAAgC;;YAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,cAAc,EACd,SAAS,CACV,CAAC;QACH,CAAC;KAAA;IAEY,qBAAqB,CAChC,IAAY,EACZ,SAAgC;;YAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,eAAe,EACf,SAAS,CACV,CAAC;QACH,CAAC;KAAA;CACD"}
|