hydra-crawler 3.2.21 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/cleanup.app.mjs +4 -4
- package/dist/apps/cleanup.app.mjs.map +1 -1
- package/dist/apps/cross-populate-export.app.mjs +2 -1
- package/dist/apps/cross-populate-export.app.mjs.map +1 -1
- package/dist/apps/cross-populate-import.app.mjs +2 -2
- package/dist/apps/cross-populate-import.app.mjs.map +1 -1
- package/dist/apps/denylist.app.mjs +5 -5
- package/dist/apps/denylist.app.mjs.map +1 -1
- package/dist/apps/export-domain-urls.mjs +2 -1
- package/dist/apps/export-domain-urls.mjs.map +1 -1
- package/dist/apps/import.app.mjs +2 -1
- package/dist/apps/import.app.mjs.map +1 -1
- package/dist/apps/invalid.app.mjs +3 -3
- package/dist/apps/invalid.app.mjs.map +1 -1
- package/dist/apps/reattempt.app.mjs +3 -2
- package/dist/apps/reattempt.app.mjs.map +1 -1
- package/dist/apps/unarchive-urls.app.mjs +2 -1
- package/dist/apps/unarchive-urls.app.mjs.map +1 -1
- package/dist/classes/cleaner.mjs +5 -5
- package/dist/classes/cleaner.mjs.map +1 -1
- package/dist/classes/expirer.mjs +2 -2
- package/dist/classes/expirer.mjs.map +1 -1
- package/dist/queries/complex-english.query.mjs +2 -2
- package/dist/queries/complex-english.query.mjs.map +1 -1
- package/dist/queries/readability-score.query.mjs +2 -1
- package/dist/queries/readability-score.query.mjs.map +1 -1
- package/dist/services/ollama-rest.service.mjs +2 -2
- package/dist/services/ollama-rest.service.mjs.map +1 -1
- package/dist/services/rig-llama-guard.service.mjs +3 -3
- package/dist/services/rig-llama-guard.service.mjs.map +1 -1
- package/dist/services/rig-subjectivity-scale.service.mjs +3 -3
- package/dist/services/rig-subjectivity-scale.service.mjs.map +1 -1
- package/dist/services/rig-ticketed-promise.service.d.mts +2 -2
- package/dist/services/rig-ticketed-promise.service.mjs +2 -1
- package/dist/services/rig-ticketed-promise.service.mjs.map +1 -1
- package/package.json +21 -21
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commonsTypeHasPropertyNumber } from 'tscommons-esm-core';
|
|
1
|
+
import { CommonsFixedDuration, commonsTypeHasPropertyNumber } from 'tscommons-esm-core';
|
|
2
2
|
import { isIUrl } from 'hydra-crawler-esm-assets';
|
|
3
3
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
4
4
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
@@ -62,7 +62,7 @@ export class CleanupApp extends CommonsApp {
|
|
|
62
62
|
try {
|
|
63
63
|
while (true) {
|
|
64
64
|
tally++;
|
|
65
|
-
log.progress(tally,
|
|
65
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
66
66
|
const row = await result.next();
|
|
67
67
|
if (row === null)
|
|
68
68
|
break;
|
|
@@ -88,7 +88,7 @@ export class CleanupApp extends CommonsApp {
|
|
|
88
88
|
let batch = [];
|
|
89
89
|
while (true) {
|
|
90
90
|
tally++;
|
|
91
|
-
log.progress(`${tally}, ${orphans.length}`,
|
|
91
|
+
log.progress(`${tally}, ${orphans.length}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
92
92
|
const link = await result.next();
|
|
93
93
|
if (link === null)
|
|
94
94
|
break;
|
|
@@ -114,7 +114,7 @@ export class CleanupApp extends CommonsApp {
|
|
|
114
114
|
let tally = 0;
|
|
115
115
|
for (const orphan of orphans) {
|
|
116
116
|
tally++;
|
|
117
|
-
log.progress(`${tally}`,
|
|
117
|
+
log.progress(`${tally}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
118
118
|
await this.databaseService.getLinks().deleteMany({ url: orphan });
|
|
119
119
|
await this.databaseService.getLinks().deleteMany({ outgoing: orphan });
|
|
120
120
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup.app.mjs","sourceRoot":"","sources":["../../src/apps/cleanup.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"cleanup.app.mjs","sourceRoot":"","sources":["../../src/apps/cleanup.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAEzG,OAAO,EAAQ,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAExD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAUjD,MAAM,OAAO,UAAW,SAAQ,UAAU;IAM/B;IACA;IACA;IACA;IARF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK,EAC7B,gBAA0B,EAC1B,aAAsB;QAE/B,KAAK,CAAC,eAAe,CAAC,CAAC;QALd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,qBAAgB,GAAhB,gBAAgB,CAAU;QAC1B,kBAAa,GAAb,aAAa,CAAS;QAI/B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,KAAK;QAClB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,QAAQ,GAAc;YAC1B,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;YAClB,OAAO,CAAC,OAAO;SAChB,CAAC;QACF,IAAI,IAAI,CAAC,gBAAgB;YAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEpJ,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;YAC5D,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CAAC,OAAO,CAAC;iBACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;YAEf,IAAI,CAAC;gBACJ,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;oBAE9D,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC3C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;QACF,CAAC;QAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,KAAK;QAClB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YACzF,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,MAAM,GAA8B,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAE9F,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAEtF,MAAM,IAAI,GAAe,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAEzB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;oBAAE,SAAS;gBAEnC,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3E,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CACxB,CAAC;gBAEF,MAAM,OAAO,GAAa,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;qBACrF,GAAG,CAAC,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEnC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBAED,KAAK,GAAG,EAAE,CAAC;YACZ,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;YACpF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YACxE,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAE1C,MAAM,SAAS,GAAqB,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,YAAY,CAAC,CAAC;QAEtF,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,MAAM,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7D,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;IACnC,CAAC;CACD"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
|
+
import { CommonsFixedDuration } from 'tscommons-esm-core';
|
|
2
3
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
3
4
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
4
5
|
import { CommonsApp } from 'nodecommons-esm-app';
|
|
@@ -40,7 +41,7 @@ export class CrossPopulateExportApp extends CommonsApp {
|
|
|
40
41
|
const log = commonsLogDoing('cross-populate-export', 'Exporting QUEUED, DONE and FAILED urls');
|
|
41
42
|
let tally = 0;
|
|
42
43
|
while (true) {
|
|
43
|
-
log.progress(tally,
|
|
44
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
44
45
|
const row = await results.next();
|
|
45
46
|
if (row === null)
|
|
46
47
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cross-populate-export.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"cross-populate-export.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAIzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAO;YAClF,MAAM,EAAE,EAAE,GAAG,EAAE;oBACb,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;iBACf,EAAE;SACJ,EAAE,EAAE,CAAC,CAAC;QAEP,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,EAAE,GAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wCAAwC,CAAC,CAAC;QAEnH,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,OAAO,IAAI,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAE9D,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,GAAG,KAAK,IAAI;gBAAE,MAAM;YAExB,EAAE,CAAC,SAAS,CACV,EAAE,EACF,IAAI,CAAC,SAAS,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;aACb,CAAC,GAAG,IAAI,CACV,CAAC;YAEF,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;gBAAE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAC7C,KAAK,EAAE,CAAC;QACT,CAAC;QAED,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACrB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEjB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;CACD"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
2
|
import * as readline from 'readline';
|
|
3
|
-
import { commonsTypeHasPropertyString } from 'tscommons-esm-core';
|
|
3
|
+
import { CommonsFixedDuration, commonsTypeHasPropertyString } from 'tscommons-esm-core';
|
|
4
4
|
import { commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
|
|
5
5
|
import { CommonsApp } from 'nodecommons-esm-app';
|
|
6
6
|
function isTRow(test) {
|
|
@@ -44,7 +44,7 @@ export class CrossPopulateImportApp extends CommonsApp {
|
|
|
44
44
|
for await (const line of rl) {
|
|
45
45
|
if (line.trim() === '')
|
|
46
46
|
continue;
|
|
47
|
-
log.progress(tally,
|
|
47
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
48
48
|
const json = JSON.parse(line);
|
|
49
49
|
if (!isTRow(json)) {
|
|
50
50
|
commonsLogError('Invalid row. Skipping');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cross-populate-import.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"cross-populate-import.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAExF,OAAO,EAAsB,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAajD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;YACtD,KAAK,EAAE,UAAU;YACjB,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;QAE3F,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YAEjC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAE9D,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,eAAe,CAAC,uBAAuB,CAAC,CAAC;gBACzC,SAAS;YACV,CAAC;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClD,KAAK,EAAE,CAAC;QACT,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAElB,UAAU,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commonsArrayChunk } from 'tscommons-esm-core';
|
|
1
|
+
import { commonsArrayChunk, CommonsFixedDuration } from 'tscommons-esm-core';
|
|
2
2
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
3
3
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
4
4
|
import { CommonsArgs } from 'nodecommons-esm-cli';
|
|
@@ -46,7 +46,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
46
46
|
.find(clauses, {});
|
|
47
47
|
while (true) {
|
|
48
48
|
tally++;
|
|
49
|
-
log.progress(`${tally}, ${found}`,
|
|
49
|
+
log.progress(`${tally}, ${found}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
50
50
|
if (limit !== undefined && tally > limit)
|
|
51
51
|
break;
|
|
52
52
|
const row = await result.next();
|
|
@@ -76,7 +76,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
76
76
|
ttl: true
|
|
77
77
|
} });
|
|
78
78
|
tally += 100;
|
|
79
|
-
log.progress(tally,
|
|
79
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
80
80
|
}
|
|
81
81
|
log.success();
|
|
82
82
|
}
|
|
@@ -98,7 +98,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
98
98
|
.find(clauses, {});
|
|
99
99
|
while (true) {
|
|
100
100
|
tally++;
|
|
101
|
-
log.progress(`${tally}, ${found}`,
|
|
101
|
+
log.progress(`${tally}, ${found}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
102
102
|
if (limit !== undefined && tally > limit)
|
|
103
103
|
break;
|
|
104
104
|
const row = await result.next();
|
|
@@ -128,7 +128,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
128
128
|
ttl: true
|
|
129
129
|
} });
|
|
130
130
|
tally += 100;
|
|
131
|
-
log.progress(tally,
|
|
131
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
132
132
|
}
|
|
133
133
|
log.success();
|
|
134
134
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"denylist.app.mjs","sourceRoot":"","sources":["../../src/apps/denylist.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAmB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"denylist.app.mjs","sourceRoot":"","sources":["../../src/apps/denylist.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,WAAY,SAAQ,UAAU;IAClC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc,EAAE,MAAe;QACpD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,CAAC,CAAC;YAE3G,MAAM,OAAO,GAGT;gBACF,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;aACtE,CAAC;YACF,IAAI,MAAM;gBAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;YAEpC,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,OAAO,EACP,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC7E,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAErD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,OAAO;QACR,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAC;YAExF,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,IAAI;qBACrB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/D,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc,EAAE,MAAe;QACpD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,CAAC,CAAC;YAE3G,MAAM,OAAO,GAGT;gBACF,MAAM,EAAE,OAAO,CAAC,IAAI;aACrB,CAAC;YACF,IAAI,MAAM;gBAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;YAEpC,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,OAAO,EACP,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC7E,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,OAAO;QACR,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,4BAA4B,CAAC,CAAC;YAE1F,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/D,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;QAE5C,MAAM,KAAK,GAAqB,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QACxE,MAAM,MAAM,GAAqB,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;QAEzE,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;CACD"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CommonsFixedDuration } from 'tscommons-esm-core';
|
|
1
2
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
2
3
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
3
4
|
import { CommonsApp } from 'nodecommons-esm-app';
|
|
@@ -58,7 +59,7 @@ export class ExportDomainUrlsApp extends CommonsApp {
|
|
|
58
59
|
const log = commonsLogDoing('export-domain-urls', `Exporting URLs for domain ${this.domain}`);
|
|
59
60
|
const urls = [];
|
|
60
61
|
while (true) {
|
|
61
|
-
log.progress(urls.length,
|
|
62
|
+
log.progress(urls.length, CommonsFixedDuration.fromHis('00:00:01'));
|
|
62
63
|
const row = await cursor.next();
|
|
63
64
|
if (row === null)
|
|
64
65
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export-domain-urls.mjs","sourceRoot":"","sources":["../../src/apps/export-domain-urls.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAIxC;IAHF,eAAe,CAA4B;IAEnD,YACU,MAAc;QAEvB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,WAAM,GAAN,MAAM,CAAQ;IAGxB,CAAC;IAEM,UAAU;QAChB,OAAO,4BAA4B,CAAC;IACrC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,wBAAwB,CAAC,CAAC;QAChG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEhE,MAAM,aAAa,GAA4B;YAC7C,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;SACpC,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAChB,EAAE,MAAM,EAAE;oBACR,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE;oBACjE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAE,GAAG,EAAE,GAAG,CAAE,EAAE;iBAClC,EAAE,CACJ,CAAC;QACH,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,GAAG;gBACpB,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;qBACrB,EAAE;gBACH,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE;qBAChD,EAAE;aACJ,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAA+C,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;aACtF,SAAS,CAAC;YACT,GAAG,aAAa;YAChB,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;SACxC,CAAC,CAAC;QAEL,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAElH,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,OAAO,IAAI,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"export-domain-urls.mjs","sourceRoot":"","sources":["../../src/apps/export-domain-urls.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAIxC;IAHF,eAAe,CAA4B;IAEnD,YACU,MAAc;QAEvB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,WAAM,GAAN,MAAM,CAAQ;IAGxB,CAAC;IAEM,UAAU;QAChB,OAAO,4BAA4B,CAAC;IACrC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,wBAAwB,CAAC,CAAC;QAChG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEhE,MAAM,aAAa,GAA4B;YAC7C,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;SACpC,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAChB,EAAE,MAAM,EAAE;oBACR,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE;oBACjE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAE,GAAG,EAAE,GAAG,CAAE,EAAE;iBAClC,EAAE,CACJ,CAAC;QACH,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,GAAG;gBACpB,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;qBACrB,EAAE;gBACH,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE;qBAChD,EAAE;aACJ,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAA+C,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;aACtF,SAAS,CAAC;YACT,GAAG,aAAa;YAChB,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;SACxC,CAAC,CAAC;QAEL,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAElH,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,OAAO,IAAI,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAEpE,MAAM,GAAG,GAAyB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACtD,IAAI,GAAG,KAAK,IAAI;gBAAE,MAAM;YAExB,IAAI,GAAG,GAAW,GAAG,CAAC,GAAG;iBACtB,IAAI,EAAE;iBACN,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;iBACrB,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC;iBACrC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;iBAClB,IAAI,EAAE,CAAC;YAEV,IAAI,SAAS,EAAE,CAAC;gBACf,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACrC,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExB,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}
|
package/dist/apps/import.app.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CommonsFixedDuration } from 'tscommons-esm-core';
|
|
1
2
|
import { commonsInputReadAllStdIn } from 'nodecommons-esm-cli';
|
|
2
3
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
3
4
|
import { CommonsApp } from 'nodecommons-esm-app';
|
|
@@ -35,7 +36,7 @@ export class ImportApp extends CommonsApp {
|
|
|
35
36
|
let done = 0;
|
|
36
37
|
for (const url of urls) {
|
|
37
38
|
tally++;
|
|
38
|
-
log.progress(`${tally}, imported ${done}`,
|
|
39
|
+
log.progress(`${tally}, imported ${done}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
39
40
|
const outcome = await this.databaseService.queue(url);
|
|
40
41
|
if (outcome) {
|
|
41
42
|
done++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import.app.mjs","sourceRoot":"","sources":["../../src/apps/import.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,MAAM,OAAO,SAAU,SAAQ,UAAU;IAChC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;QACpF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;YACrF,IAAI,GAAG,MAAM,wBAAwB,EAAE,CAAC;YACxC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;YAChF,IAAI,KAAK,GAAW,CAAC,CAAC;YAAC,IAAI,IAAI,GAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,cAAc,IAAI,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"import.app.mjs","sourceRoot":"","sources":["../../src/apps/import.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,MAAM,OAAO,SAAU,SAAQ,UAAU;IAChC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;QACpF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;YACrF,IAAI,GAAG,MAAM,wBAAwB,EAAE,CAAC;YACxC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;YAChF,IAAI,KAAK,GAAW,CAAC,CAAC;YAAC,IAAI,IAAI,GAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,cAAc,IAAI,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAErF,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE,CAAC;oBACb,IAAI,EAAE,CAAC;gBACR,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commonsArrayChunk } from 'tscommons-esm-core';
|
|
1
|
+
import { commonsArrayChunk, CommonsFixedDuration } from 'tscommons-esm-core';
|
|
2
2
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
3
3
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
4
4
|
import { CommonsApp } from 'nodecommons-esm-app';
|
|
@@ -34,7 +34,7 @@ export class InvalidApp extends CommonsApp {
|
|
|
34
34
|
.find({ status: { $ne: EStatus.INVALID } }, {});
|
|
35
35
|
while (true) {
|
|
36
36
|
tally++;
|
|
37
|
-
log.progress(`${tally}, ${found}`,
|
|
37
|
+
log.progress(`${tally}, ${found}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
38
38
|
const row = await result.next();
|
|
39
39
|
if (row === null)
|
|
40
40
|
break;
|
|
@@ -57,7 +57,7 @@ export class InvalidApp extends CommonsApp {
|
|
|
57
57
|
ttl: true
|
|
58
58
|
} });
|
|
59
59
|
tally += 100;
|
|
60
|
-
log.progress(tally,
|
|
60
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
61
61
|
}
|
|
62
62
|
log.success();
|
|
63
63
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"invalid.app.mjs","sourceRoot":"","sources":["../../src/apps/invalid.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"invalid.app.mjs","sourceRoot":"","sources":["../../src/apps/invalid.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAIrD,MAAM,OAAO,UAAW,SAAQ,UAAU;IACjC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,+CAA+C,CAAC,CAAC;YAE5G,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,EACpC,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAE7E,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,MAAM,OAAO,GAAc,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,OAAO,KAAK,KAAK;oBAAE,SAAS;gBAEhC,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,6BAA6B,CAAC,CAAC;YAE1F,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,OAAO;qBACxB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/D,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CommonsFixedDuration } from 'tscommons-esm-core';
|
|
1
2
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
2
3
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
3
4
|
import { CommonsApp } from 'nodecommons-esm-app';
|
|
@@ -42,7 +43,7 @@ export class ReattemptApp extends CommonsApp {
|
|
|
42
43
|
}, {});
|
|
43
44
|
tally = 0;
|
|
44
45
|
while (true) {
|
|
45
|
-
log.progress(tally,
|
|
46
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
46
47
|
const row = await results.next();
|
|
47
48
|
if (row === null)
|
|
48
49
|
break;
|
|
@@ -60,7 +61,7 @@ export class ReattemptApp extends CommonsApp {
|
|
|
60
61
|
let tally = 0;
|
|
61
62
|
for (const url of urls) {
|
|
62
63
|
tally++;
|
|
63
|
-
log.progress(tally,
|
|
64
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
64
65
|
await this.databaseService.getUrls()
|
|
65
66
|
.updateOne({ url: url }, {
|
|
66
67
|
$set: { status: EStatus.QUEUED },
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reattempt.app.mjs","sourceRoot":"","sources":["../../src/apps/reattempt.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,YAAa,SAAQ,UAAU;IAMjC;IALF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;QACvF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAExH,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;aAC9B,EACD,EAAE,CACH,CAAC;YAEJ,KAAK,GAAG,CAAC,CAAC;YAEV,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"reattempt.app.mjs","sourceRoot":"","sources":["../../src/apps/reattempt.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,YAAa,SAAQ,UAAU;IAMjC;IALF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;QACvF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAExH,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;aAC9B,EACD,EAAE,CACH,CAAC;YAEJ,KAAK,GAAG,CAAC,CAAC;YAEV,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAE9D,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAC;YAClF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAE9D,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACjC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;oBACvB,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;iBACnB,CAAC,CAAC;YACN,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CommonsFixedDuration } from 'tscommons-esm-core';
|
|
1
2
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
2
3
|
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
3
4
|
import { CommonsApp } from 'nodecommons-esm-app';
|
|
@@ -39,7 +40,7 @@ export class UnarchiveUrlsApp extends CommonsApp {
|
|
|
39
40
|
let tally = 0;
|
|
40
41
|
let found = 0;
|
|
41
42
|
while (true) {
|
|
42
|
-
log.progress(`${tally} ${found}`,
|
|
43
|
+
log.progress(`${tally} ${found}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
43
44
|
tally++;
|
|
44
45
|
const row = await results.next();
|
|
45
46
|
if (row === null)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unarchive-urls.app.mjs","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,CAAC;YAE/F,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"unarchive-urls.app.mjs","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,CAAC;YAE/F,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC5E,KAAK,EAAE,CAAC;gBAER,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,0BAA0B,CAAC,CAAC;YAC9F,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACpB,KAAK,IAAI,GAAG,CAAC;gBAEb,MAAM,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;iBACjC,CACF,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}
|
package/dist/classes/cleaner.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commonsTypeHasPropertyNumber } from 'tscommons-esm-core';
|
|
1
|
+
import { CommonsFixedDuration, commonsTypeHasPropertyNumber } from 'tscommons-esm-core';
|
|
2
2
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
3
3
|
import { commonsLogDebug, commonsLogDoing } from 'nodecommons-esm-log';
|
|
4
4
|
import { EList } from '../enums/elist.mjs';
|
|
@@ -37,7 +37,7 @@ export class Cleaner {
|
|
|
37
37
|
{ $project: { links: false } }
|
|
38
38
|
]);
|
|
39
39
|
while (true) {
|
|
40
|
-
log.progress(`${found} orphans`,
|
|
40
|
+
log.progress(`${found} orphans`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
41
41
|
if (hardLimit && found >= hardLimit)
|
|
42
42
|
break;
|
|
43
43
|
const row = await urls.next();
|
|
@@ -139,7 +139,7 @@ export class Cleaner {
|
|
|
139
139
|
try {
|
|
140
140
|
while (true) {
|
|
141
141
|
tally++;
|
|
142
|
-
log.progress(tally,
|
|
142
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
143
143
|
const row = await urls2.next();
|
|
144
144
|
if (row === null)
|
|
145
145
|
break;
|
|
@@ -207,7 +207,7 @@ export class Cleaner {
|
|
|
207
207
|
tally = 0;
|
|
208
208
|
while (true) {
|
|
209
209
|
tally++;
|
|
210
|
-
log.progress(tally,
|
|
210
|
+
log.progress(tally, CommonsFixedDuration.fromHis('00:00:01'));
|
|
211
211
|
const row = await results.next();
|
|
212
212
|
if (row === null)
|
|
213
213
|
break;
|
|
@@ -226,7 +226,7 @@ export class Cleaner {
|
|
|
226
226
|
const log = commonsLogDoing('cleaner', 'Detecting and removing empty domains');
|
|
227
227
|
for (const domain of domains) {
|
|
228
228
|
tally++;
|
|
229
|
-
log.progress(`${tally} domains, ${found} removed`,
|
|
229
|
+
log.progress(`${tally} domains, ${found} removed`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
230
230
|
const urls = await this.databaseService.getUrls()
|
|
231
231
|
.findOne({
|
|
232
232
|
domain: domain,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGzG,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAoC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzE,SAAS,CAAO;gBACf,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACf,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBACjB,EAAE,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,EAAE;gBACnB,gHAAgH;gBAChH,EAAE,OAAO,EAAE;wBACT,IAAI,EAAE,OAAO;wBACb,UAAU,EAAE,KAAK;wBACjB,YAAY,EAAE,UAAU;wBACxB,QAAQ,EAAE;4BACR,EAAE,MAAM,EAAE,CAAC,EAAE;yBACd;wBACD,EAAE,EAAE,OAAO;qBACZ,EAAE;gBACH,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC7C,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;aAC/B,CAAC,CAAC;YAEL,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAE3E,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;oBAAE,MAAM;gBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAE5B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,KAAK,EAAE,CAAC;YACT,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QAChC,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC9B,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,sBAAsB,CAC5B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,sBAAsB,EACtB,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACvB,CACF,CAAC;QACH,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,4BAA4B,CAClC,QAAmB,EACnB,OAA+B,EAC/B,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,sBAAsB,CAChC,uBAAuB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EACxD,OAAO,EACP,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEO,0BAA0B,CAChC,QAAmB,EACnB,OAA+B,EAC/B,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,oBAAoB,CAC9B,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EACnD,OAAO,EACP,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEO,yBAAyB,CAC/B,QAAmB,EACnB,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,oBAAoB,CAC9B,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAC5D,OAAO,EACP,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;IACH,CAAC;IAEO,uBAAuB,CAC7B,GAAW,EACX,EAAU,EACV,OAA+B,EAC/B,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAET,EAAE,IAAI,EAAE;gBACV,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;gBACxB,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;gBAC7B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;aAC5B,EAAE,CAAC;QACJ,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,oBAAoB,CAC9B,sCAAsC,GAAG,IAAI,EAAE,cAAc,EAC7D,OAAO,EACP,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,eAAe,CAC1B,MAAe,EACf,SAAkB;QAEnB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,MAAM,IAAI,CAAC,4BAA4B,CACrC,CAAE,OAAO,CAAC,OAAO,CAAE,EACnB,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,MAAM,CACP,CAAC;QAEF,MAAM,IAAI,CAAC,0BAA0B,CACnC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EACD,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,MAAM,EACN,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,0BAA0B,CACnC;YACE,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;SACnB,EACD,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,MAAM,EACN,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,yBAAyB,CAClC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EACD,MAAM,EACN,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,uBAAuB,CAChC,GAAG,EACH,GAAG,EACH,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,MAAM,EACN,SAAS,CACV,CAAC;QACF,MAAM,IAAI,CAAC,uBAAuB,CAChC,GAAG,EACH,GAAG,EACH,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,MAAM,EACN,SAAS,CACV,CAAC;QACF,MAAM,IAAI,CAAC,uBAAuB,CAChC,GAAG,EACH,GAAG,EACH,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAAE,yBAAyB;QACxD,MAAM,EACN,SAAS,CACV,CAAC;QAEF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAC;YAC7F,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;wBAE9D,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBACpE,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAChF,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;oBACZ,oBAAoB,EAAE,IAAI;oBAC1B,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,QAAQ,EAAE,IAAI;oBACd,cAAc,EAAE,IAAI;oBACpB,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAElF,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,IAAI,CAAC;gBACJ,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;oBAE9D,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,CAAC,CAAC;YACnG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAE7F,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;iBACxD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;CAED"}
|
package/dist/classes/expirer.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import mongodb from 'mongodb';
|
|
2
|
-
import { commonsArrayChunk, commonsDateDateToYmdHis, commonsTypeHasPropertyDate } from 'tscommons-esm-core';
|
|
2
|
+
import { commonsArrayChunk, commonsDateDateToYmdHis, CommonsFixedDuration, commonsTypeHasPropertyDate } from 'tscommons-esm-core';
|
|
3
3
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
4
4
|
import { commonsLogAlert, commonsLogDoing } from 'nodecommons-esm-log';
|
|
5
5
|
import { isTMongoIdRow } from '../services/database.service.mjs';
|
|
@@ -43,7 +43,7 @@ export class Expirer {
|
|
|
43
43
|
let found = 0;
|
|
44
44
|
while (true) {
|
|
45
45
|
tally++;
|
|
46
|
-
log.progress(`${tally}, ${found}`,
|
|
46
|
+
log.progress(`${tally}, ${found}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
47
47
|
if (limit !== undefined && tally > limit)
|
|
48
48
|
break;
|
|
49
49
|
const row = await results.next();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expirer.mjs","sourceRoot":"","sources":["../../src/classes/expirer.mts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,0BAA0B,EAAmB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"expirer.mjs","sourceRoot":"","sources":["../../src/classes/expirer.mts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,0BAA0B,EAAmB,MAAM,oBAAoB,CAAC;AAEnJ,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAI3F,OAAO,EAAgC,aAAa,EAAE,MAAM,kCAAkC,CAAC;AAE/F,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,MAAc,EACd,QAAyB;QADzB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;IAChC,CAAC;IAEI,KAAK,CAAC,mBAAmB;QAChC,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;QAClG,MAAM,OAAO,GAA2C,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,SAAS,CAAc;YACrG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE;YACnE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;SAC/B,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3B,MAAM,MAAM,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAE3F,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE3B,OAAO,MAAM;YACX,gDAAgD;aAC/C,GAAG,CAAC,CAAC,CAAc,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,KAAc;QACjC,eAAe,CAAC,uBAAuB,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE1D,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YAEzF,MAAM,OAAO,GAA6B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACpE,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE;gCAChB,OAAO,CAAC,QAAQ;gCAChB,OAAO,CAAC,OAAO;gCACf,OAAO,CAAC,MAAM;gCACd,OAAO,CAAC,MAAM;6BACf,EAAE,EAAE;oBACL,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;iBACjC,EAAC,EACF,EAAE,CACH,CAAC;YAEF,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YAEhD,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC7E,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,0BAA0B,CAAC,GAAG,EAAE,WAAW,CAAC;oBAAE,SAAS;gBAE5D,IAAI,CAAC;oBACJ,MAAM,QAAQ,GAAW,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAE5D,MAAM,SAAS,GAAa,GAAuB,CAAC,WAAW,CAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;oBAC3F,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,EAAE,CAAC;wBAClC,iEAAiE;wBACjE,OAAO,CAAC,IAAI,CAAE,GAAuB,CAAC,KAAK,CAAC,CAAC,CAAC;wBAC9C,KAAK,EAAE,CAAC;oBACT,CAAC;gBACF,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,SAAS;gBACV,CAAC;YACF,CAAC;YACD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAEnB,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC/B,CAAC;QAED,MAAM,OAAO,GAAe,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAE5D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACjF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAuB,KAAK;qBACvC,GAAG,CAAC,CAAC,EAAU,EAAoB,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEnE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC1B;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;iBACtB,CACF,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,SAAe;QACvC,eAAe,CAAC,kCAAkC,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE1D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mCAAmC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpI,MAAM,KAAK,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACtD,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE,EAAE;oBAC3D,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF,EAAE,CACH,CAAC,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAEnB,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC/B,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE,EAAE;oBAC3D,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;gBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;aACtB,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,IAAI;QACJ,4FAA4F;QAE5F,OAAO,IAAI,CAAC;IACb,CAAC;CACD"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as cheerio from 'cheerio';
|
|
2
|
-
import { commonsArrayUnique } from 'tscommons-esm-core';
|
|
2
|
+
import { commonsArrayUnique, CommonsFixedDuration } from 'tscommons-esm-core';
|
|
3
3
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
4
4
|
import { commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
|
|
5
5
|
import { commonsHttpReadUrlAsBuffer } from 'nodecommons-esm-http';
|
|
@@ -27,7 +27,7 @@ export const QUERY = async (args, databaseService, lists, _expiry, parsersConfig
|
|
|
27
27
|
let tally = 0;
|
|
28
28
|
while (true) {
|
|
29
29
|
tally++;
|
|
30
|
-
log.progress(`${tally}`,
|
|
30
|
+
log.progress(`${tally}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
31
31
|
const row = await result.next();
|
|
32
32
|
if (row === null)
|
|
33
33
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complex-english.query.mjs","sourceRoot":"","sources":["../../src/queries/complex-english.query.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAmB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"complex-english.query.mjs","sourceRoot":"","sources":["../../src/queries/complex-english.query.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE/F,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAInD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,OAAO,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAM7E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAU3C,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAA6C,EAC9B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAwC,MAAM,CAAC,aAAa,EAAE,CAAC;IAClF,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,MAAM,GAAG,GAAuB,eAAe,CAAC,wBAAwB,EAAE,uDAAuD,MAAM,EAAE,CAAC,CAAC;IAE3I,MAAM,MAAM,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACrE;QACE,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO,CAAC,IAAI;QACpB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAClC,EACD,EAAE,CACH,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE,CAAC;QACb,KAAK,EAAE,CAAC;QACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAEnE,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,MAAM,QAAQ,GAAkB,GAA+B,CAAC;QAEhE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,SAAS,EAAE,CAAC;YACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAClB,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAChC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAClC;gBAAE,SAAS;QACb,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,cAAc,EAAE,QAAQ,CAAC,cAAc;SACxC,CAAC,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAqB,MAAM,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,eAAe,CAAC,8BAA8B,CAAC,CAAC;YAChD,SAAS;QACV,CAAC;QAED,MAAM,GAAG,GAAiC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,eAAe,CAAC,gCAAgC,CAAC,CAAC;YAClD,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAyB,GAAG,CAAC,GAAG,CAAC;aACzC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAgB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAE1F,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwC,EAAQ,EAAE;YAClF,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAa,WAAW;aAChC,IAAI,CAAC,IAAI,CAAC;aACV,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAa,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,OAAO,GAAW,IAAI,CAAC;YAE3B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAqB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAEhC,MAAM,WAAW,GAAa,aAAa,CAAC,OAAO,CAAC,CAAC;gBAErD,MAAM,KAAK,GAAa,CAAE,KAAK,OAAO,EAAE,CAAE,CAAC;gBAC3C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAErE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9D,CAAC;YAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CommonsFixedDuration } from 'tscommons-esm-core';
|
|
1
2
|
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
2
3
|
import { commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
|
|
3
4
|
export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
|
|
@@ -54,7 +55,7 @@ export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConf
|
|
|
54
55
|
let tally = 0;
|
|
55
56
|
while (true) {
|
|
56
57
|
tally++;
|
|
57
|
-
log.progress(`${tally}`,
|
|
58
|
+
log.progress(`${tally}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
58
59
|
const row = await results.next();
|
|
59
60
|
if (row === null)
|
|
60
61
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readability-score.query.mjs","sourceRoot":"","sources":["../../src/queries/readability-score.query.mts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"readability-score.query.mjs","sourceRoot":"","sources":["../../src/queries/readability-score.query.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAuB3F,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAA8C,EAC/B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,GAAG,GAAuB,eAAe,CAAC,kBAAkB,EAAE,4DAA4D,MAAM,EAAE,CAAC,CAAC;IAE1I,MAAM,OAAO,GAA+B,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAU;QACvF,EAAE,MAAM,EAAE;gBACR,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;gBACvD,MAAM,EAAE,MAAM;gBACd,sBAAsB,EAAE,WAAW;aACpC,EAAE;QACH,EAAE,MAAM,EAAE;gBACR,2BAA2B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC3C,0BAA0B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;aAC3C,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE;oBACL,UAAU,EAAE,kCAAkC;oBAC9C,SAAS,EAAE,iCAAiC;oBAC5C,KAAK,EAAE,6BAA6B;oBACpC,IAAI,EAAE,4BAA4B;oBAClC,GAAG,EAAE,2BAA2B;iBACjC;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,CAAE,EAAE;oBACtD,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,EAAE,EAAE,YAAY,CAAE,EAAE,CAAE,EAAE;iBACpD;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE;oBACR,IAAI,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,EAAE,GAAG,CAAE,EAAE;oBACjE,GAAG,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,YAAY,EAAE,CAAC,CAAE,EAAE,EAAE,EAAE,CAAE,EAAE;iBAC7D;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,IAAI,EAAE,CAAE,gBAAgB,EAAE,eAAe,CAAE,EAAE,EAAE,CAAC,CAAE,EAAE;aAC3E,EAAE;QACH,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAc,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE,CAAC;QACb,KAAK,EAAE,CAAC;QACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAEnE,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElB,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACjK,CAAC;AACF,CAAC,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commonsTypeIsString, commonsTypeIsStringArray } from 'tscommons-esm-core';
|
|
1
|
+
import { CommonsFixedDuration, commonsTypeIsString, commonsTypeIsStringArray } from 'tscommons-esm-core';
|
|
2
2
|
import { ECommonsHttpContentType } from 'tscommons-esm-http';
|
|
3
3
|
import { CommonsKeyRestClientService } from 'tscommons-esm-rest';
|
|
4
4
|
export class OllamaRestService extends CommonsKeyRestClientService {
|
|
@@ -17,7 +17,7 @@ export class OllamaRestService extends CommonsKeyRestClientService {
|
|
|
17
17
|
};
|
|
18
18
|
const result = await super.postRest(`/generate/${model}`, body, undefined, undefined, {
|
|
19
19
|
maxReattempts: 0,
|
|
20
|
-
timeout:
|
|
20
|
+
timeout: CommonsFixedDuration.fromHis('00:00:30')
|
|
21
21
|
});
|
|
22
22
|
if (!commonsTypeIsString(result))
|
|
23
23
|
throw new Error('Result was not a string');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama-rest.service.mjs","sourceRoot":"","sources":["../../src/services/ollama-rest.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"ollama-rest.service.mjs","sourceRoot":"","sources":["../../src/services/ollama-rest.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAkB,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACzH,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AAIjE,MAAM,OAAO,iBAAkB,SAAQ,2BAA2B;IACjE,YACE,cAAuD,EACvD,GAAW,EACX,GAAW;QAEZ,KAAK,CACH,cAAc,EACd,GAAG,EACH,GAAG,EACH,uBAAuB,CAAC,IAAI,CAC7B,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,UAAU;QACtB,MAAM,MAAM,GAAY,MAAM,KAAK,CAAC,OAAO,CAAW,SAAS,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAExF,OAAO,MAAM,CAAC;IACf,CAAC;IAEM,KAAK,CAAC,QAAQ,CACnB,KAAa,EACb,MAAc;QAEf,MAAM,IAAI,GAAmB;YAC3B,MAAM,EAAE,MAAM;SACf,CAAC;QAEF,MAAM,MAAM,GAAY,MAAM,KAAK,CAAC,QAAQ,CAC1C,aAAa,KAAK,EAAE,EACpB,IAAI,EACJ,SAAS,EACT,SAAS,EACT;YACE,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC;SAClD,CACF,CAAC;QACF,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAE7E,OAAO,MAAM,CAAC;IACf,CAAC;CACD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commonsArrayRemoveUndefineds, commonsArrayUnique, commonsStringSplitString, commonsTypeHasPropertyEnum, commonsTypeHasPropertyEnumArray, commonsTypeIsString } from 'tscommons-esm-core';
|
|
1
|
+
import { commonsArrayRemoveUndefineds, commonsArrayUnique, CommonsFixedDuration, commonsStringSplitString, commonsTypeHasPropertyEnum, commonsTypeHasPropertyEnumArray, commonsTypeIsString } from 'tscommons-esm-core';
|
|
2
2
|
import { CommonsInternalHttpClientImplementation } from 'nodecommons-esm-http';
|
|
3
3
|
import { OllamaRestService } from './ollama-rest.service.mjs';
|
|
4
4
|
import { RigTicketedPromiseService } from './rig-ticketed-promise.service.mjs';
|
|
@@ -213,7 +213,7 @@ function parseToTLlamaGuardResult(s) {
|
|
|
213
213
|
export class RigLlamaGuardService extends RigTicketedPromiseService {
|
|
214
214
|
model;
|
|
215
215
|
constructor(url, key, model) {
|
|
216
|
-
super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url
|
|
216
|
+
super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url);
|
|
217
217
|
this.model = model;
|
|
218
218
|
}
|
|
219
219
|
async generate(text, keepAlive) {
|
|
@@ -260,7 +260,7 @@ Provide your safety assessment for ONLY THE LAST Agent in the above conversation
|
|
|
260
260
|
}
|
|
261
261
|
try {
|
|
262
262
|
const response = await this.submitAndWaitAsPromise(`/generate/${this.model}`, body, (data) => commonsTypeIsString(data), undefined, undefined, {
|
|
263
|
-
timeout:
|
|
263
|
+
timeout: CommonsFixedDuration.fromHis('00:00:30'),
|
|
264
264
|
maxReattempts: 0
|
|
265
265
|
});
|
|
266
266
|
if (!commonsTypeIsString(response))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rig-llama-guard.service.mjs","sourceRoot":"","sources":["../../src/services/rig-llama-guard.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,kBAAkB,
|
|
1
|
+
{"version":3,"file":"rig-llama-guard.service.mjs","sourceRoot":"","sources":["../../src/services/rig-llama-guard.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,+BAA+B,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAExN,OAAO,EAAE,uCAAuC,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,oCAAoC,CAAC;AAErG,MAAM,CAAN,IAAY,yBAGX;AAHD,WAAY,yBAAyB;IACnC,0CAAa,CAAA;IACb,8CAAiB,CAAA;AACnB,CAAC,EAHW,yBAAyB,KAAzB,yBAAyB,QAGpC;AAED,MAAM,UAAU,2BAA2B,CAAC,IAAY;IACvD,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE;YAC7C,OAAO,yBAAyB,CAAC,IAAI,CAAC;QACvC,KAAK,yBAAyB,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC/C,OAAO,yBAAyB,CAAC,MAAM,CAAC;IAC1C,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,IAA+B;IAC5E,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,yBAAyB,CAAC,IAAI;YAClC,OAAO,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClD,KAAK,yBAAyB,CAAC,MAAM;YACpC,OAAO,yBAAyB,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;IACrD,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,IAAa;IACxD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7C,OAAO,2BAA2B,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,GAAW;IACzD,QAAQ,GAAG,EAAE,CAAC;QACb,KAAK,MAAM;YACV,OAAO,yBAAyB,CAAC,IAAI,CAAC;QACvC,KAAK,QAAQ;YACZ,OAAO,yBAAyB,CAAC,MAAM,CAAC;IAC1C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,uDAAuD,GAAG,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,CAAC,MAAM,4BAA4B,GAAgC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC;KAC5G,GAAG,CAAC,CAAC,CAAS,EAA6B,EAAE,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;AAEpF,MAAM,CAAN,IAAY,mBAeX;AAfD,WAAY,mBAAmB;IAC7B,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,gCAAS,CAAA;IACT,kCAAW,CAAA;IACX,kCAAW,CAAA;IACX,kCAAW,CAAA;IACX,kCAAW,CAAA;IACX,kCAAW,CAAA;AACb,CAAC,EAfW,mBAAmB,KAAnB,mBAAmB,QAe9B;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IACjD,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE;YACrC,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE;YACtC,OAAO,mBAAmB,CAAC,GAAG,CAAC;IACjC,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAyB;IAChE,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,EAAE;YAC1B,OAAO,mBAAmB,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC1C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC3C,KAAK,mBAAmB,CAAC,GAAG;YAC3B,OAAO,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAa;IAClD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7C,OAAO,qBAAqB,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,GAAW;IACnD,QAAQ,GAAG,EAAE,CAAC;QACb,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,IAAI;YACR,OAAO,mBAAmB,CAAC,EAAE,CAAC;QAC/B,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;QAChC,KAAK,KAAK;YACT,OAAO,mBAAmB,CAAC,GAAG,CAAC;IACjC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,iDAAiD,GAAG,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAA6B;IACrE,QAAQ,QAAQ,EAAE,CAAC;QAClB,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACrD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACzD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,YAAY,CAAC;QACjD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACzD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,YAAY,CAAC;QACjD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,oBAAoB,CAAC;QACzD,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,SAAS,CAAC;QAC9C,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,uBAAuB,CAAC;QAC5D,KAAK,mBAAmB,CAAC,EAAE,CAAC,CAAC,OAAO,wBAAwB,CAAC;QAC7D,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,MAAM,CAAC;QAC5C,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,WAAW,CAAC;QACjD,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,gBAAgB,CAAC;QACtD,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,WAAW,CAAC;QACjD,KAAK,mBAAmB,CAAC,GAAG,CAAC,CAAC,OAAO,wBAAwB,CAAC;IAC/D,CAAC;AACF,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAA0B,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC;KAC1F,GAAG,CAAC,CAAC,CAAS,EAAuB,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,CAAC;AAaxE,SAAS,mBAAmB,CAAC,IAAa;IACzC,IAAI,CAAC,0BAA0B,CAA4B,IAAI,EAAE,gBAAgB,EAAE,2BAA2B,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9H,IAAI,IAAI,CAAC,cAAc,KAAK,yBAAyB,CAAC,MAAM,EAAE,CAAC;QAC9D,IAAI,CAAC,+BAA+B,CAAsB,IAAI,EAAE,YAAY,EAAE,qBAAqB,CAAC;YAAE,OAAO,KAAK,CAAC;IACpH,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,wBAAwB,CAAC,CAAS;IAC1C,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACb,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;QAClB,OAAO;YACL,cAAc,EAAE,yBAAyB,CAAC,IAAI;SAC/C,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAyB,0EAA0E,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvH,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IAEzB,MAAM,UAAU,GAA0B,kBAAkB,CAC1D,4BAA4B,CAC1B,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC;SAChD,GAAG,CAAC,CAAC,EAAU,EAAiC,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CACjF,EACD,SAAS,EACT,CAAC,CAAsB,EAAE,CAAsB,EAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CACrE,CAAC;IAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1C,OAAO;QACL,cAAc,EAAE,yBAAyB,CAAC,MAAM;QAChD,UAAU,EAAE,UAAU;KACvB,CAAC;AACH,CAAC;AAED,MAAM,OAAO,oBAAqB,SAAQ,yBAAyB;IAIxD;IAHV,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,CACJ,CAAC;QATO,UAAK,GAAL,KAAK,CAAQ;IAUvB,CAAC;IAEM,KAAK,CAAC,QAAQ,CACnB,IAAY,EACZ,SAAgC;QAEjC,MAAM,MAAM,GAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BtB;aACE,IAAI,EAAE;aACN,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QACtD,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;YACvC,IAAI,IAAI,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;gBACE,OAAO,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC;gBACjD,aAAa,EAAE,CAAC;aACjB,CACF,CAAC;YACF,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC;gBAAE,OAAO,SAAS,CAAC;YAErD,MAAM,MAAM,GAA4B,wBAAwB,CAAC,QAAQ,CAAC,CAAC;YAC3E,IAAI,CAAC,MAAM;gBAAE,OAAO,SAAS,CAAC;YAC9B,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;gBAAE,OAAO,SAAS,CAAC;YAEnD,OAAO,MAAM,CAAC;QACf,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { commonsTypeAttemptObject, commonsTypeHasPropertyNumber, commonsTypeHasPropertyStringOrUndefined, commonsTypeIsString } from 'tscommons-esm-core';
|
|
1
|
+
import { CommonsFixedDuration, commonsTypeAttemptObject, commonsTypeHasPropertyNumber, commonsTypeHasPropertyStringOrUndefined, commonsTypeIsString } from 'tscommons-esm-core';
|
|
2
2
|
import { CommonsInternalHttpClientImplementation } from 'nodecommons-esm-http';
|
|
3
3
|
import { OllamaRestService } from './ollama-rest.service.mjs';
|
|
4
4
|
import { RigTicketedPromiseService } from './rig-ticketed-promise.service.mjs';
|
|
@@ -43,7 +43,7 @@ const INTEREST_PROMPT = GENERIC_PROMPT
|
|
|
43
43
|
export class RigSubjectivityScaleService extends RigTicketedPromiseService {
|
|
44
44
|
model;
|
|
45
45
|
constructor(url, key, model) {
|
|
46
|
-
super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url
|
|
46
|
+
super(new OllamaRestService(new CommonsInternalHttpClientImplementation(), `${url}/ollama`, key), url);
|
|
47
47
|
this.model = model;
|
|
48
48
|
}
|
|
49
49
|
async generateSubjectivityScore(text, prompt, keepAlive) {
|
|
@@ -58,7 +58,7 @@ export class RigSubjectivityScaleService extends RigTicketedPromiseService {
|
|
|
58
58
|
}
|
|
59
59
|
try {
|
|
60
60
|
const response = await this.submitAndWaitAsPromise(`/generate/${this.model}`, body, (data) => commonsTypeIsString(data), undefined, undefined, {
|
|
61
|
-
timeout:
|
|
61
|
+
timeout: CommonsFixedDuration.fromHis('00:00:30'),
|
|
62
62
|
maxReattempts: 0
|
|
63
63
|
});
|
|
64
64
|
const attempt = commonsTypeAttemptObject(response);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rig-subjectivity-scale.service.mjs","sourceRoot":"","sources":["../../src/services/rig-subjectivity-scale.service.mts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"rig-subjectivity-scale.service.mjs","sourceRoot":"","sources":["../../src/services/rig-subjectivity-scale.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAmB,wBAAwB,EAAE,4BAA4B,EAAE,uCAAuC,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEjM,OAAO,EAAE,uCAAuC,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAwB,MAAM,oCAAoC,CAAC;AAMrG,SAAS,qBAAqB,CAAC,IAAa;IAC3C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3E,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,cAAc,GAAW;;;;;;;;;;;;;;CAc9B,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,cAAc,GAAW,cAAc;KAC1C,OAAO,CACN,cAAc,EACd,gDAAgD,CACjD;KACA,OAAO,CACN,aAAa,EACb;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,eAAe,GAAW,cAAc;KAC3C,OAAO,CACN,cAAc,EACd,iDAAiD,CAClD;KACA,OAAO,CACN,OAAO,EACP;;;;;KAKC,CAAC,IAAI,EAAE,CACT,CAAC;AAEJ,MAAM,OAAO,2BAA4B,SAAQ,yBAAyB;IAI/D;IAHV,YACE,GAAW,EACX,GAAW,EACH,KAAa;QAEtB,KAAK,CACH,IAAI,iBAAiB,CACnB,IAAI,uCAAuC,EAAE,EAC7C,GAAG,GAAG,SAAS,EACf,GAAG,CACJ,EACD,GAAG,CACJ,CAAC;QATO,UAAK,GAAL,KAAK,CAAQ;IAUvB,CAAC;IAEO,KAAK,CAAC,yBAAyB,CACrC,IAAY,EACZ,MAAc,EACd,SAAgC;QAEjC,MAAM,GAAG,MAAM;aACZ,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAE/B,MAAM,IAAI,GAAyB,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QACtD,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,GAAW,SAAS,CAAC,OAAO,CAAC;YACvC,IAAI,IAAI,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAExE,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,sBAAsB,CAIxD,aAAa,IAAI,CAAC,KAAK,EAAE,EACzB,IAAI,EACJ,CAAC,IAAa,EAAkB,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAC5D,SAAS,EACT,SAAS,EACT;gBACE,OAAO,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC;gBACjD,aAAa,EAAE,CAAC;aACjB,CACF,CAAC;YACF,MAAM,OAAO,GAAuC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;YACvF,IAAI,CAAC,OAAO;gBAAE,OAAO,SAAS,CAAC;YAE/B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC;gBAAE,OAAO,SAAS,CAAC;YAEtD,OAAO,OAAO,CAAC;QAChB,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAC/B,IAAY,EACZ,SAAgC;QAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,cAAc,EACd,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,qBAAqB,CAChC,IAAY,EACZ,SAAgC;QAEjC,OAAO,MAAM,IAAI,CAAC,yBAAyB,CACzC,IAAI,EACJ,eAAe,EACf,SAAS,CACV,CAAC;IACH,CAAC;CACD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TEncoded, TEncodedObject } from 'tscommons-esm-core';
|
|
1
|
+
import { CommonsFixedDuration, TEncoded, TEncodedObject } from 'tscommons-esm-core';
|
|
2
2
|
import { CommonsRestClientService } from 'tscommons-esm-rest';
|
|
3
3
|
import { TCommonsHttpRequestOptions } from 'tscommons-esm-http';
|
|
4
4
|
export type TPromptAndKeepLoaded = {
|
|
@@ -9,6 +9,6 @@ export declare abstract class RigTicketedPromiseService {
|
|
|
9
9
|
private restService;
|
|
10
10
|
private socketIoUrl;
|
|
11
11
|
private connectToSubmitDelay;
|
|
12
|
-
constructor(restService: CommonsRestClientService, socketIoUrl: string, connectToSubmitDelay?:
|
|
12
|
+
constructor(restService: CommonsRestClientService, socketIoUrl: string, connectToSubmitDelay?: CommonsFixedDuration);
|
|
13
13
|
submitAndWaitAsPromise<ResultT extends TEncoded = TEncoded, BodyT extends (TEncodedObject & TPromptAndKeepLoaded) = (TEncodedObject & TPromptAndKeepLoaded), ParamsT extends TEncodedObject = TEncodedObject, HeadersT extends TEncodedObject = TEncodedObject>(script: string, body: BodyT, checker: (data: unknown) => data is ResultT, params?: ParamsT, headers?: HeadersT, options?: TCommonsHttpRequestOptions): Promise<ResultT>;
|
|
14
14
|
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { CommonsFixedDuration } from 'tscommons-esm-core';
|
|
1
2
|
import { ECommonsHttpMethod } from 'tscommons-esm-http';
|
|
2
3
|
import { commonsTicketedNodeGenericSubmitAndWaitAsPromise } from 'nodecommons-esm-socket-io-ticketed';
|
|
3
4
|
export class RigTicketedPromiseService {
|
|
4
5
|
restService;
|
|
5
6
|
socketIoUrl;
|
|
6
7
|
connectToSubmitDelay;
|
|
7
|
-
constructor(restService, socketIoUrl, connectToSubmitDelay = 100) {
|
|
8
|
+
constructor(restService, socketIoUrl, connectToSubmitDelay = CommonsFixedDuration.fromMillis(100)) {
|
|
8
9
|
this.restService = restService;
|
|
9
10
|
this.socketIoUrl = socketIoUrl;
|
|
10
11
|
this.connectToSubmitDelay = connectToSubmitDelay;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rig-ticketed-promise.service.mjs","sourceRoot":"","sources":["../../src/services/rig-ticketed-promise.service.mts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rig-ticketed-promise.service.mjs","sourceRoot":"","sources":["../../src/services/rig-ticketed-promise.service.mts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAA4B,MAAM,oBAAoB,CAAC;AAEpF,OAAO,EAAE,kBAAkB,EAA8B,MAAM,oBAAoB,CAAC;AAEpF,OAAO,EAAE,gDAAgD,EAAE,MAAM,oCAAoC,CAAC;AAOtG,MAAM,OAAgB,yBAAyB;IAEpC;IACA;IACA;IAHV,YACU,WAAqC,EACrC,WAAmB,EACnB,uBAA6C,oBAAoB,CAAC,UAAU,CAAC,GAAG,CAAC;QAFjF,gBAAW,GAAX,WAAW,CAA0B;QACrC,gBAAW,GAAX,WAAW,CAAQ;QACnB,yBAAoB,GAApB,oBAAoB,CAA6D;IACxF,CAAC;IAEG,sBAAsB,CAM3B,MAAc,EACd,IAAW,EACX,OAA2C,EAC3C,MAAgB,EAChB,OAAkB,EAClB,UAAsC,EAAE;QAEzC,OAAO,gDAAgD,CAMrD,IAAI,CAAC,WAAW,EAChB,cAAc,EACd,IAAI,CAAC,WAAW,EAChB,kBAAkB,CAAC,IAAI,EACvB,MAAM,EACN,IAAI,EACJ,OAAO,EACP,MAAM,EACN,OAAO,EACP,OAAO,EACP,IAAI,CAAC,oBAAoB,CAC1B,CAAC;IACH,CAAC;CACD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hydra-crawler",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.0",
|
|
4
4
|
"description": "Node.js Hydra web crawler",
|
|
5
5
|
"author": "Pete Morris",
|
|
6
6
|
"license": "ISC",
|
|
@@ -24,26 +24,26 @@
|
|
|
24
24
|
"jpeg-js": "^0.4.4",
|
|
25
25
|
"memcached-promisify": "^2.0.0",
|
|
26
26
|
"mongodb": "^6.15.0",
|
|
27
|
-
"nodecommons-esm-app": "^0.0.
|
|
28
|
-
"nodecommons-esm-app-socket-io": "^
|
|
29
|
-
"nodecommons-esm-cli": "^0.0.
|
|
30
|
-
"nodecommons-esm-config": "^0.0.
|
|
31
|
-
"nodecommons-esm-database": "^0.
|
|
32
|
-
"nodecommons-esm-database-mongodb": "^0.0.
|
|
33
|
-
"nodecommons-esm-express": "^
|
|
34
|
-
"nodecommons-esm-file": "^0.1.
|
|
35
|
-
"nodecommons-esm-http": "^
|
|
36
|
-
"nodecommons-esm-log": "^0.
|
|
37
|
-
"nodecommons-esm-process": "^
|
|
38
|
-
"nodecommons-esm-rest": "^0.0.
|
|
39
|
-
"nodecommons-esm-security": "^0.0.
|
|
40
|
-
"nodecommons-esm-socket-io-ticketed": "^
|
|
41
|
-
"tscommons-esm-async": "^
|
|
42
|
-
"tscommons-esm-config": "^0.0.
|
|
43
|
-
"tscommons-esm-core": "^0.0
|
|
44
|
-
"tscommons-esm-format": "^0.0.
|
|
45
|
-
"tscommons-esm-http": "^
|
|
46
|
-
"tscommons-esm-rest": "^
|
|
27
|
+
"nodecommons-esm-app": "^0.0.15",
|
|
28
|
+
"nodecommons-esm-app-socket-io": "^2.0.2",
|
|
29
|
+
"nodecommons-esm-cli": "^0.0.14",
|
|
30
|
+
"nodecommons-esm-config": "^0.0.13",
|
|
31
|
+
"nodecommons-esm-database": "^0.2.6",
|
|
32
|
+
"nodecommons-esm-database-mongodb": "^0.0.16",
|
|
33
|
+
"nodecommons-esm-express": "^1.0.4",
|
|
34
|
+
"nodecommons-esm-file": "^0.1.6",
|
|
35
|
+
"nodecommons-esm-http": "^1.0.3",
|
|
36
|
+
"nodecommons-esm-log": "^1.0.3",
|
|
37
|
+
"nodecommons-esm-process": "^1.1.4",
|
|
38
|
+
"nodecommons-esm-rest": "^0.0.14",
|
|
39
|
+
"nodecommons-esm-security": "^0.0.10",
|
|
40
|
+
"nodecommons-esm-socket-io-ticketed": "^1.0.4",
|
|
41
|
+
"tscommons-esm-async": "^1.0.4",
|
|
42
|
+
"tscommons-esm-config": "^0.0.10",
|
|
43
|
+
"tscommons-esm-core": "^0.2.0",
|
|
44
|
+
"tscommons-esm-format": "^0.0.10",
|
|
45
|
+
"tscommons-esm-http": "^1.0.3",
|
|
46
|
+
"tscommons-esm-rest": "^1.0.3",
|
|
47
47
|
"unfluff": "^3.2.0"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|