hydra-crawler 3.3.4 → 3.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/cleanup.app.mjs +6 -6
- package/dist/apps/cleanup.app.mjs.map +1 -1
- package/dist/apps/cross-populate-export.app.mjs +2 -2
- package/dist/apps/cross-populate-export.app.mjs.map +1 -1
- package/dist/apps/cross-populate-import.app.mjs +2 -2
- package/dist/apps/cross-populate-import.app.mjs.map +1 -1
- package/dist/apps/denylist.app.mjs +8 -8
- package/dist/apps/denylist.app.mjs.map +1 -1
- package/dist/apps/export-domain-urls.mjs +2 -2
- package/dist/apps/export-domain-urls.mjs.map +1 -1
- package/dist/apps/import.app.mjs +2 -2
- package/dist/apps/import.app.mjs.map +1 -1
- package/dist/apps/invalid.app.mjs +4 -4
- package/dist/apps/invalid.app.mjs.map +1 -1
- package/dist/apps/reattempt.app.mjs +4 -4
- package/dist/apps/reattempt.app.mjs.map +1 -1
- package/dist/apps/unarchive-urls.app.mjs +2 -2
- package/dist/apps/unarchive-urls.app.mjs.map +1 -1
- package/dist/classes/cleaner.mjs +12 -9
- package/dist/classes/cleaner.mjs.map +1 -1
- package/dist/classes/expirer.mjs +2 -2
- package/dist/classes/expirer.mjs.map +1 -1
- package/dist/queries/complex-english.query.mjs +2 -2
- package/dist/queries/complex-english.query.mjs.map +1 -1
- package/dist/queries/readability-score.query.mjs +2 -2
- package/dist/queries/readability-score.query.mjs.map +1 -1
- package/dist/servers/maintenance.server.mjs +3 -2
- package/dist/servers/maintenance.server.mjs.map +1 -1
- package/package.json +31 -30
|
@@ -49,7 +49,7 @@ export class CleanupApp extends CommonsApp {
|
|
|
49
49
|
];
|
|
50
50
|
if (this.includeArchiveds)
|
|
51
51
|
statuses.push(EStatus.ARCHIVED);
|
|
52
|
-
const log = commonsLogDoing('cleanup', `Reseting urls for ${statuses.map((s) => s.toUpperCase()).join(', ')}
|
|
52
|
+
const log = commonsLogDoing('cleanup', `Reseting urls for ${statuses.map((s) => s.toUpperCase()).join(', ')}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
53
53
|
let tally = 0;
|
|
54
54
|
while (true) {
|
|
55
55
|
const clauses = { status: { $in: statuses } };
|
|
@@ -62,7 +62,7 @@ export class CleanupApp extends CommonsApp {
|
|
|
62
62
|
try {
|
|
63
63
|
while (true) {
|
|
64
64
|
tally++;
|
|
65
|
-
log.progress(tally
|
|
65
|
+
log.progress(tally);
|
|
66
66
|
const row = await result.next();
|
|
67
67
|
if (row === null)
|
|
68
68
|
break;
|
|
@@ -82,13 +82,13 @@ export class CleanupApp extends CommonsApp {
|
|
|
82
82
|
throw new Error('Database service has not been set');
|
|
83
83
|
const orphans = [];
|
|
84
84
|
{ // scope
|
|
85
|
-
const log = commonsLogDoing('cleanup', 'Searching for link orphans');
|
|
85
|
+
const log = commonsLogDoing('cleanup', 'Searching for link orphans', CommonsFixedDuration.fromHis('00:00:01'));
|
|
86
86
|
let tally = 0;
|
|
87
87
|
const result = this.databaseService.getLinks().find({}, {});
|
|
88
88
|
let batch = [];
|
|
89
89
|
while (true) {
|
|
90
90
|
tally++;
|
|
91
|
-
log.progress(`${tally}, ${orphans.length}
|
|
91
|
+
log.progress(`${tally}, ${orphans.length}`);
|
|
92
92
|
const link = await result.next();
|
|
93
93
|
if (link === null)
|
|
94
94
|
break;
|
|
@@ -110,11 +110,11 @@ export class CleanupApp extends CommonsApp {
|
|
|
110
110
|
log.result(orphans.length);
|
|
111
111
|
}
|
|
112
112
|
{ // scope
|
|
113
|
-
const log = commonsLogDoing('cleanup', 'Removing orphan links');
|
|
113
|
+
const log = commonsLogDoing('cleanup', 'Removing orphan links', CommonsFixedDuration.fromHis('00:00:01'));
|
|
114
114
|
let tally = 0;
|
|
115
115
|
for (const orphan of orphans) {
|
|
116
116
|
tally++;
|
|
117
|
-
log.progress(`${tally}
|
|
117
|
+
log.progress(`${tally}`);
|
|
118
118
|
await this.databaseService.getLinks().deleteMany({ url: orphan });
|
|
119
119
|
await this.databaseService.getLinks().deleteMany({ outgoing: orphan });
|
|
120
120
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup.app.mjs","sourceRoot":"","sources":["../../src/apps/cleanup.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAEzG,OAAO,EAAQ,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAExD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAUjD,MAAM,OAAO,UAAW,SAAQ,UAAU;IAM/B;IACA;IACA;IACA;IARF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK,EAC7B,gBAA0B,EAC1B,aAAsB;QAE/B,KAAK,CAAC,eAAe,CAAC,CAAC;QALd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,qBAAgB,GAAhB,gBAAgB,CAAU;QAC1B,kBAAa,GAAb,aAAa,CAAS;QAI/B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,KAAK;QAClB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,QAAQ,GAAc;YAC1B,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;YAClB,OAAO,CAAC,OAAO;SAChB,CAAC;QACF,IAAI,IAAI,CAAC,gBAAgB;YAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"cleanup.app.mjs","sourceRoot":"","sources":["../../src/apps/cleanup.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAEzG,OAAO,EAAQ,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAExD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAUjD,MAAM,OAAO,UAAW,SAAQ,UAAU;IAM/B;IACA;IACA;IACA;IARF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK,EAC7B,gBAA0B,EAC1B,aAAsB;QAE/B,KAAK,CAAC,eAAe,CAAC,CAAC;QALd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,qBAAgB,GAAhB,gBAAgB,CAAU;QAC1B,kBAAa,GAAb,aAAa,CAAS;QAI/B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,KAAK;QAClB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,QAAQ,GAAc;YAC1B,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;YAClB,OAAO,CAAC,OAAO;SAChB,CAAC;QACF,IAAI,IAAI,CAAC,gBAAgB;YAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAE9L,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;YAC5D,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CAAC,OAAO,CAAC;iBACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;YAEf,IAAI,CAAC;gBACJ,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAEpB,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC3C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;QACF,CAAC;QAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,KAAK;QAClB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4BAA4B,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YACnI,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,MAAM,GAA8B,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAE9F,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE5C,MAAM,IAAI,GAAe,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAEzB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;oBAAE,SAAS;gBAEnC,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3E,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CACxB,CAAC;gBAEF,MAAM,OAAO,GAAa,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;qBACrF,GAAG,CAAC,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEnC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBAED,KAAK,GAAG,EAAE,CAAC;YACZ,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,uBAAuB,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC9H,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;gBACzB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YACxE,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAE1C,MAAM,SAAS,GAAqB,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,YAAY,CAAC,CAAC;QAEtF,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,MAAM,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC7D,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;IACnC,CAAC;CACD"}
|
|
@@ -38,10 +38,10 @@ export class CrossPopulateExportApp extends CommonsApp {
|
|
|
38
38
|
// safer to do this directly rather than a call to listQueryResults
|
|
39
39
|
// ditto not using CommonsFile
|
|
40
40
|
const fd = fs.openSync(this.filename, 'w');
|
|
41
|
-
const log = commonsLogDoing('cross-populate-export', 'Exporting QUEUED, DONE and FAILED urls');
|
|
41
|
+
const log = commonsLogDoing('cross-populate-export', 'Exporting QUEUED, DONE and FAILED urls', CommonsFixedDuration.fromHis('00:00:01'));
|
|
42
42
|
let tally = 0;
|
|
43
43
|
while (true) {
|
|
44
|
-
log.progress(tally
|
|
44
|
+
log.progress(tally);
|
|
45
45
|
const row = await results.next();
|
|
46
46
|
if (row === null)
|
|
47
47
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cross-populate-export.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAIzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAO;YAClF,MAAM,EAAE,EAAE,GAAG,EAAE;oBACb,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;iBACf,EAAE;SACJ,EAAE,EAAE,CAAC,CAAC;QAEP,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,EAAE,GAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wCAAwC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"cross-populate-export.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAIzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAO;YAClF,MAAM,EAAE,EAAE,GAAG,EAAE;oBACb,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;iBACf,EAAE;SACJ,EAAE,EAAE,CAAC,CAAC;QAEP,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,EAAE,GAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wCAAwC,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAE7J,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,OAAO,IAAI,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAEpB,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,GAAG,KAAK,IAAI;gBAAE,MAAM;YAExB,EAAE,CAAC,SAAS,CACV,EAAE,EACF,IAAI,CAAC,SAAS,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;aACb,CAAC,GAAG,IAAI,CACV,CAAC;YAEF,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;gBAAE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAC7C,KAAK,EAAE,CAAC;QACT,CAAC;QAED,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACrB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEjB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;CACD"}
|
|
@@ -39,12 +39,12 @@ export class CrossPopulateImportApp extends CommonsApp {
|
|
|
39
39
|
input: fileStream,
|
|
40
40
|
crlfDelay: Infinity
|
|
41
41
|
});
|
|
42
|
-
const log = commonsLogDoing('cross-populate-import', 'Importing urls');
|
|
42
|
+
const log = commonsLogDoing('cross-populate-import', 'Importing urls', CommonsFixedDuration.fromHis('00:00:01'));
|
|
43
43
|
let tally = 0;
|
|
44
44
|
for await (const line of rl) {
|
|
45
45
|
if (line.trim() === '')
|
|
46
46
|
continue;
|
|
47
|
-
log.progress(tally
|
|
47
|
+
log.progress(tally);
|
|
48
48
|
const json = JSON.parse(line);
|
|
49
49
|
if (!isTRow(json)) {
|
|
50
50
|
commonsLogError('Invalid row. Skipping');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cross-populate-import.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAExF,OAAO,EAAsB,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAajD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;YACtD,KAAK,EAAE,UAAU;YACjB,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"cross-populate-import.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAExF,OAAO,EAAsB,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAajD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;YACtD,KAAK,EAAE,UAAU;YACjB,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,gBAAgB,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAErI,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YAEjC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAEpB,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,eAAe,CAAC,uBAAuB,CAAC,CAAC;gBACzC,SAAS;YACV,CAAC;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClD,KAAK,EAAE,CAAC;QACT,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAElB,UAAU,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACD"}
|
|
@@ -36,7 +36,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
36
36
|
let found = 0;
|
|
37
37
|
const urls = [];
|
|
38
38
|
{ // scope
|
|
39
|
-
const log = commonsLogDoing('denylist', 'Searching for non-DENY urls on the denylist');
|
|
39
|
+
const log = commonsLogDoing('denylist', 'Searching for non-DENY urls on the denylist', CommonsFixedDuration.fromHis('00:00:01'));
|
|
40
40
|
const clauses = {
|
|
41
41
|
status: { $nin: [EStatus.DENY, EStatus.ARCHIVED, EStatus.INVALID] }
|
|
42
42
|
};
|
|
@@ -46,7 +46,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
46
46
|
.find(clauses, {});
|
|
47
47
|
while (true) {
|
|
48
48
|
tally++;
|
|
49
|
-
log.progress(`${tally}, ${found}
|
|
49
|
+
log.progress(`${tally}, ${found}`);
|
|
50
50
|
if (limit !== undefined && tally > limit)
|
|
51
51
|
break;
|
|
52
52
|
const row = await result.next();
|
|
@@ -66,7 +66,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
66
66
|
return;
|
|
67
67
|
}
|
|
68
68
|
{ // scope
|
|
69
|
-
const log = commonsLogDoing('denylist', 'Marking detected as DENY');
|
|
69
|
+
const log = commonsLogDoing('denylist', 'Marking detected as DENY', CommonsFixedDuration.fromHis('00:00:01'));
|
|
70
70
|
const batches = commonsArrayChunk(urls, 100);
|
|
71
71
|
tally = 0;
|
|
72
72
|
for (const batch of batches) {
|
|
@@ -76,7 +76,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
76
76
|
ttl: true
|
|
77
77
|
} });
|
|
78
78
|
tally += 100;
|
|
79
|
-
log.progress(tally
|
|
79
|
+
log.progress(tally);
|
|
80
80
|
}
|
|
81
81
|
log.success();
|
|
82
82
|
}
|
|
@@ -88,7 +88,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
88
88
|
let found = 0;
|
|
89
89
|
const urls = [];
|
|
90
90
|
{ // scope
|
|
91
|
-
const log = commonsLogDoing('denylist', 'Searching for DENY urls not on the denylist');
|
|
91
|
+
const log = commonsLogDoing('denylist', 'Searching for DENY urls not on the denylist', CommonsFixedDuration.fromHis('00:00:01'));
|
|
92
92
|
const clauses = {
|
|
93
93
|
status: EStatus.DENY
|
|
94
94
|
};
|
|
@@ -98,7 +98,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
98
98
|
.find(clauses, {});
|
|
99
99
|
while (true) {
|
|
100
100
|
tally++;
|
|
101
|
-
log.progress(`${tally}, ${found}
|
|
101
|
+
log.progress(`${tally}, ${found}`);
|
|
102
102
|
if (limit !== undefined && tally > limit)
|
|
103
103
|
break;
|
|
104
104
|
const row = await result.next();
|
|
@@ -118,7 +118,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
118
118
|
return;
|
|
119
119
|
}
|
|
120
120
|
{ // scope
|
|
121
|
-
const log = commonsLogDoing('denylist', 'Marking detected as QUEUED');
|
|
121
|
+
const log = commonsLogDoing('denylist', 'Marking detected as QUEUED', CommonsFixedDuration.fromHis('00:00:01'));
|
|
122
122
|
const batches = commonsArrayChunk(urls, 100);
|
|
123
123
|
tally = 0;
|
|
124
124
|
for (const batch of batches) {
|
|
@@ -128,7 +128,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
128
128
|
ttl: true
|
|
129
129
|
} });
|
|
130
130
|
tally += 100;
|
|
131
|
-
log.progress(tally
|
|
131
|
+
log.progress(tally);
|
|
132
132
|
}
|
|
133
133
|
log.success();
|
|
134
134
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"denylist.app.mjs","sourceRoot":"","sources":["../../src/apps/denylist.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,WAAY,SAAQ,UAAU;IAClC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc,EAAE,MAAe;QACpD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"denylist.app.mjs","sourceRoot":"","sources":["../../src/apps/denylist.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,WAAY,SAAQ,UAAU;IAClC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc,EAAE,MAAe;QACpD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAErJ,MAAM,OAAO,GAGT;gBACF,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;aACtE,CAAC;YACF,IAAI,MAAM;gBAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;YAEpC,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,OAAO,EACP,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACnC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAErD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,OAAO;QACR,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,0BAA0B,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAElI,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,IAAI;qBACrB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc,EAAE,MAAe;QACpD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAErJ,MAAM,OAAO,GAGT;gBACF,MAAM,EAAE,OAAO,CAAC,IAAI;aACrB,CAAC;YACF,IAAI,MAAM;gBAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;YAEpC,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,OAAO,EACP,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACnC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,OAAO;QACR,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,4BAA4B,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAEpI,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;QAE5C,MAAM,KAAK,GAAqB,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QACxE,MAAM,MAAM,GAAqB,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;QAEzE,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;CACD"}
|
|
@@ -56,10 +56,10 @@ export class ExportDomainUrlsApp extends CommonsApp {
|
|
|
56
56
|
]);
|
|
57
57
|
// safer to do this directly rather than a call to listQueryResults
|
|
58
58
|
// ditto not using CommonsFile
|
|
59
|
-
const log = commonsLogDoing('export-domain-urls', `Exporting URLs for domain ${this.domain}
|
|
59
|
+
const log = commonsLogDoing('export-domain-urls', `Exporting URLs for domain ${this.domain}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
60
60
|
const urls = [];
|
|
61
61
|
while (true) {
|
|
62
|
-
log.progress(urls.length
|
|
62
|
+
log.progress(urls.length);
|
|
63
63
|
const row = await cursor.next();
|
|
64
64
|
if (row === null)
|
|
65
65
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export-domain-urls.mjs","sourceRoot":"","sources":["../../src/apps/export-domain-urls.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAIxC;IAHF,eAAe,CAA4B;IAEnD,YACU,MAAc;QAEvB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,WAAM,GAAN,MAAM,CAAQ;IAGxB,CAAC;IAEM,UAAU;QAChB,OAAO,4BAA4B,CAAC;IACrC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,wBAAwB,CAAC,CAAC;QAChG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEhE,MAAM,aAAa,GAA4B;YAC7C,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;SACpC,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAChB,EAAE,MAAM,EAAE;oBACR,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE;oBACjE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAE,GAAG,EAAE,GAAG,CAAE,EAAE;iBAClC,EAAE,CACJ,CAAC;QACH,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,GAAG;gBACpB,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;qBACrB,EAAE;gBACH,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE;qBAChD,EAAE;aACJ,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAA+C,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;aACtF,SAAS,CAAC;YACT,GAAG,aAAa;YAChB,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;SACxC,CAAC,CAAC;QAEL,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"export-domain-urls.mjs","sourceRoot":"","sources":["../../src/apps/export-domain-urls.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAIxC;IAHF,eAAe,CAA4B;IAEnD,YACU,MAAc;QAEvB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,WAAM,GAAN,MAAM,CAAQ;IAGxB,CAAC;IAEM,UAAU;QAChB,OAAO,4BAA4B,CAAC;IACrC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,wBAAwB,CAAC,CAAC;QAChG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEhE,MAAM,aAAa,GAA4B;YAC7C,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;SACpC,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAChB,EAAE,MAAM,EAAE;oBACR,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE;oBACjE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAE,GAAG,EAAE,GAAG,CAAE,EAAE;iBAClC,EAAE,CACJ,CAAC;QACH,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,GAAG;gBACpB,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;qBACrB,EAAE;gBACH,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE;qBAChD,EAAE;aACJ,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAA+C,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;aACtF,SAAS,CAAC;YACT,GAAG,aAAa;YAChB,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;SACxC,CAAC,CAAC;QAEL,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QAE5J,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,OAAO,IAAI,EAAE,CAAC;YACb,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE1B,MAAM,GAAG,GAAyB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACtD,IAAI,GAAG,KAAK,IAAI;gBAAE,MAAM;YAExB,IAAI,GAAG,GAAW,GAAG,CAAC,GAAG;iBACtB,IAAI,EAAE;iBACN,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;iBACrB,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC;iBACrC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;iBAClB,IAAI,EAAE,CAAC;YAEV,IAAI,SAAS,EAAE,CAAC;gBACf,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACrC,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExB,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}
|
package/dist/apps/import.app.mjs
CHANGED
|
@@ -31,12 +31,12 @@ export class ImportApp extends CommonsApp {
|
|
|
31
31
|
log.result(urls.length);
|
|
32
32
|
}
|
|
33
33
|
{ // scope
|
|
34
|
-
const log = commonsLogDoing('import', 'Seeding into queue');
|
|
34
|
+
const log = commonsLogDoing('import', 'Seeding into queue', CommonsFixedDuration.fromHis('00:00:01'));
|
|
35
35
|
let tally = 0;
|
|
36
36
|
let done = 0;
|
|
37
37
|
for (const url of urls) {
|
|
38
38
|
tally++;
|
|
39
|
-
log.progress(`${tally}, imported ${done}
|
|
39
|
+
log.progress(`${tally}, imported ${done}`);
|
|
40
40
|
const outcome = await this.databaseService.queue(url);
|
|
41
41
|
if (outcome) {
|
|
42
42
|
done++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import.app.mjs","sourceRoot":"","sources":["../../src/apps/import.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,MAAM,OAAO,SAAU,SAAQ,UAAU;IAChC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;QACpF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;YACrF,IAAI,GAAG,MAAM,wBAAwB,EAAE,CAAC;YACxC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"import.app.mjs","sourceRoot":"","sources":["../../src/apps/import.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,MAAM,OAAO,SAAU,SAAQ,UAAU;IAChC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;QACpF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;YACrF,IAAI,GAAG,MAAM,wBAAwB,EAAE,CAAC;YACxC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,oBAAoB,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC1H,IAAI,KAAK,GAAW,CAAC,CAAC;YAAC,IAAI,IAAI,GAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,cAAc,IAAI,EAAE,CAAC,CAAC;gBAE3C,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE,CAAC;oBACb,IAAI,EAAE,CAAC;gBACR,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}
|
|
@@ -29,12 +29,12 @@ export class InvalidApp extends CommonsApp {
|
|
|
29
29
|
let found = 0;
|
|
30
30
|
const urls = [];
|
|
31
31
|
{ // scope
|
|
32
|
-
const log = commonsLogDoing('invalid', 'Searching for invalid urls not marked INVALID');
|
|
32
|
+
const log = commonsLogDoing('invalid', 'Searching for invalid urls not marked INVALID', CommonsFixedDuration.fromHis('00:00:01'));
|
|
33
33
|
const result = this.databaseService.getUrls()
|
|
34
34
|
.find({ status: { $ne: EStatus.INVALID } }, {});
|
|
35
35
|
while (true) {
|
|
36
36
|
tally++;
|
|
37
|
-
log.progress(`${tally}, ${found}
|
|
37
|
+
log.progress(`${tally}, ${found}`);
|
|
38
38
|
const row = await result.next();
|
|
39
39
|
if (row === null)
|
|
40
40
|
break;
|
|
@@ -47,7 +47,7 @@ export class InvalidApp extends CommonsApp {
|
|
|
47
47
|
log.result(found);
|
|
48
48
|
}
|
|
49
49
|
{ // scope
|
|
50
|
-
const log = commonsLogDoing('invalid', 'Marking detected as INVALID');
|
|
50
|
+
const log = commonsLogDoing('invalid', 'Marking detected as INVALID', CommonsFixedDuration.fromHis('00:00:01'));
|
|
51
51
|
const batches = commonsArrayChunk(urls, 100);
|
|
52
52
|
tally = 0;
|
|
53
53
|
for (const batch of batches) {
|
|
@@ -57,7 +57,7 @@ export class InvalidApp extends CommonsApp {
|
|
|
57
57
|
ttl: true
|
|
58
58
|
} });
|
|
59
59
|
tally += 100;
|
|
60
|
-
log.progress(tally
|
|
60
|
+
log.progress(tally);
|
|
61
61
|
}
|
|
62
62
|
log.success();
|
|
63
63
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"invalid.app.mjs","sourceRoot":"","sources":["../../src/apps/invalid.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAIrD,MAAM,OAAO,UAAW,SAAQ,UAAU;IACjC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,+CAA+C,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"invalid.app.mjs","sourceRoot":"","sources":["../../src/apps/invalid.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAIrD,MAAM,OAAO,UAAW,SAAQ,UAAU;IACjC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,+CAA+C,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAEtJ,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,EACpC,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBAEnC,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,MAAM,OAAO,GAAc,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,OAAO,KAAK,KAAK;oBAAE,SAAS;gBAEhC,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,6BAA6B,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAEpI,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,OAAO;qBACxB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}
|
|
@@ -36,14 +36,14 @@ export class ReattemptApp extends CommonsApp {
|
|
|
36
36
|
const urls = [];
|
|
37
37
|
{ // scope
|
|
38
38
|
let tally = 0;
|
|
39
|
-
const log = commonsLogDoing('reattempt', `Searching for allowlisted ${this.include.join(' and ')}
|
|
39
|
+
const log = commonsLogDoing('reattempt', `Searching for allowlisted ${this.include.join(' and ')}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
40
40
|
const results = this.databaseService.getUrls()
|
|
41
41
|
.find({
|
|
42
42
|
status: { $in: this.include }
|
|
43
43
|
}, {});
|
|
44
44
|
tally = 0;
|
|
45
45
|
while (true) {
|
|
46
|
-
log.progress(tally
|
|
46
|
+
log.progress(tally);
|
|
47
47
|
const row = await results.next();
|
|
48
48
|
if (row === null)
|
|
49
49
|
break;
|
|
@@ -57,11 +57,11 @@ export class ReattemptApp extends CommonsApp {
|
|
|
57
57
|
log.result(tally);
|
|
58
58
|
}
|
|
59
59
|
{ // scope
|
|
60
|
-
const log = commonsLogDoing('reattempt', 'Requeuing matches');
|
|
60
|
+
const log = commonsLogDoing('reattempt', 'Requeuing matches', CommonsFixedDuration.fromHis('00:00:01'));
|
|
61
61
|
let tally = 0;
|
|
62
62
|
for (const url of urls) {
|
|
63
63
|
tally++;
|
|
64
|
-
log.progress(tally
|
|
64
|
+
log.progress(tally);
|
|
65
65
|
await this.databaseService.getUrls()
|
|
66
66
|
.updateOne({ url: url }, {
|
|
67
67
|
$set: { status: EStatus.QUEUED },
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reattempt.app.mjs","sourceRoot":"","sources":["../../src/apps/reattempt.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,YAAa,SAAQ,UAAU;IAMjC;IALF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;QACvF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"reattempt.app.mjs","sourceRoot":"","sources":["../../src/apps/reattempt.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,YAAa,SAAQ,UAAU;IAMjC;IALF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;QACvF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAElK,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;aAC9B,EACD,EAAE,CACH,CAAC;YAEJ,KAAK,GAAG,CAAC,CAAC;YAEV,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAEpB,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,mBAAmB,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5H,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAEpB,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACjC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;oBACvB,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;iBACnB,CAAC,CAAC;YACN,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACF,CAAC;CACD"}
|
|
@@ -34,13 +34,13 @@ export class UnarchiveUrlsApp extends CommonsApp {
|
|
|
34
34
|
throw new Error('Database service has not been set');
|
|
35
35
|
const urls = [];
|
|
36
36
|
{ // scope
|
|
37
|
-
const log = commonsLogDoing('unarchive-urls', 'Enumerating ARCHIVED URLS');
|
|
37
|
+
const log = commonsLogDoing('unarchive-urls', 'Enumerating ARCHIVED URLS', CommonsFixedDuration.fromHis('00:00:01'));
|
|
38
38
|
const results = this.databaseService.getUrls()
|
|
39
39
|
.find({ status: EStatus.ARCHIVED }, {});
|
|
40
40
|
let tally = 0;
|
|
41
41
|
let found = 0;
|
|
42
42
|
while (true) {
|
|
43
|
-
log.progress(`${tally} ${found}
|
|
43
|
+
log.progress(`${tally} ${found}`);
|
|
44
44
|
tally++;
|
|
45
45
|
const row = await results.next();
|
|
46
46
|
if (row === null)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unarchive-urls.app.mjs","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"unarchive-urls.app.mjs","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,2BAA2B,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAEzI,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;gBAClC,KAAK,EAAE,CAAC;gBAER,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,0BAA0B,CAAC,CAAC;YAC9F,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACpB,KAAK,IAAI,GAAG,CAAC;gBAEb,MAAM,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;iBACjC,CACF,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}
|
package/dist/classes/cleaner.mjs
CHANGED
|
@@ -15,7 +15,7 @@ export class Cleaner {
|
|
|
15
15
|
let found = 0;
|
|
16
16
|
const orphanUrls = [];
|
|
17
17
|
{ // scope
|
|
18
|
-
const log = commonsLogDoing('cleaner', detectingCaption);
|
|
18
|
+
const log = commonsLogDoing('cleaner', detectingCaption, CommonsFixedDuration.fromHis('00:00:01'));
|
|
19
19
|
const urls = this.databaseService.getUrls()
|
|
20
20
|
.aggregate([
|
|
21
21
|
{ $match: { $or: [
|
|
@@ -37,7 +37,7 @@ export class Cleaner {
|
|
|
37
37
|
{ $project: { links: false } }
|
|
38
38
|
]);
|
|
39
39
|
while (true) {
|
|
40
|
-
log.progress(`${found} orphans
|
|
40
|
+
log.progress(`${found} orphans`);
|
|
41
41
|
if (hardLimit && found >= hardLimit)
|
|
42
42
|
break;
|
|
43
43
|
const row = await urls.next();
|
|
@@ -101,7 +101,10 @@ export class Cleaner {
|
|
|
101
101
|
return this.detectAndFlagOrphans(`Detecting ${statuses.join(', ')} status code between ${gte}<${lt} orphan urls`, clauses, matcher, hardLimit);
|
|
102
102
|
}
|
|
103
103
|
detectFakeVersionedOrphans(domain, hardLimit) {
|
|
104
|
-
const clauses = {
|
|
104
|
+
const clauses = {
|
|
105
|
+
status: { $ne: EStatus.ARCHIVED },
|
|
106
|
+
url: /\?ver=[0-9]+$/
|
|
107
|
+
};
|
|
105
108
|
if (domain)
|
|
106
109
|
clauses.domain = domain;
|
|
107
110
|
return this.detectAndFlagOrphans('Detecting artificial ?ver=12345678 style orphan urls', clauses, (_url) => true, hardLimit);
|
|
@@ -164,7 +167,7 @@ export class Cleaner {
|
|
|
164
167
|
await this.detectFakeVersionedOrphans(domain, hardLimit);
|
|
165
168
|
let tally = 0;
|
|
166
169
|
{ // scope
|
|
167
|
-
const log = commonsLogDoing('cleaner', 'Removing orphan outgoing links');
|
|
170
|
+
const log = commonsLogDoing('cleaner', 'Removing orphan outgoing links', CommonsFixedDuration.fromHis('00:00:01'));
|
|
168
171
|
while (true) {
|
|
169
172
|
const urls2 = this.databaseService.getUrls()
|
|
170
173
|
.find({ orphan: true }, {})
|
|
@@ -173,7 +176,7 @@ export class Cleaner {
|
|
|
173
176
|
try {
|
|
174
177
|
while (true) {
|
|
175
178
|
tally++;
|
|
176
|
-
log.progress(tally
|
|
179
|
+
log.progress(tally);
|
|
177
180
|
const row = await urls2.next();
|
|
178
181
|
if (row === null)
|
|
179
182
|
break;
|
|
@@ -235,13 +238,13 @@ export class Cleaner {
|
|
|
235
238
|
let found = 0;
|
|
236
239
|
const domains = [];
|
|
237
240
|
{ // scope
|
|
238
|
-
const log = commonsLogDoing('cleaner', 'Enumerating domains');
|
|
241
|
+
const log = commonsLogDoing('cleaner', 'Enumerating domains', CommonsFixedDuration.fromHis('00:00:01'));
|
|
239
242
|
const results = this.databaseService.getDomains().find({}, {});
|
|
240
243
|
try {
|
|
241
244
|
tally = 0;
|
|
242
245
|
while (true) {
|
|
243
246
|
tally++;
|
|
244
|
-
log.progress(tally
|
|
247
|
+
log.progress(tally);
|
|
245
248
|
const row = await results.next();
|
|
246
249
|
if (row === null)
|
|
247
250
|
break;
|
|
@@ -257,10 +260,10 @@ export class Cleaner {
|
|
|
257
260
|
{ // scope
|
|
258
261
|
tally = 0;
|
|
259
262
|
found = 0;
|
|
260
|
-
const log = commonsLogDoing('cleaner', 'Detecting and removing empty domains');
|
|
263
|
+
const log = commonsLogDoing('cleaner', 'Detecting and removing empty domains', CommonsFixedDuration.fromHis('00:00:01'));
|
|
261
264
|
for (const domain of domains) {
|
|
262
265
|
tally++;
|
|
263
|
-
log.progress(`${tally} domains, ${found} removed
|
|
266
|
+
log.progress(`${tally} domains, ${found} removed`);
|
|
264
267
|
const urls = await this.databaseService.getUrls()
|
|
265
268
|
.findOne({
|
|
266
269
|
domain: domain,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGzG,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGzG,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAEvH,MAAM,IAAI,GAAoC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzE,SAAS,CAAO;gBACf,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACf,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBACjB,EAAE,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,EAAE;gBACnB,gHAAgH;gBAChH,EAAE,OAAO,EAAE;wBACT,IAAI,EAAE,OAAO;wBACb,UAAU,EAAE,KAAK;wBACjB,YAAY,EAAE,UAAU;wBACxB,QAAQ,EAAE;4BACR,EAAE,MAAM,EAAE,CAAC,EAAE;yBACd;wBACD,EAAE,EAAE,OAAO;qBACZ,EAAE;gBACH,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC7C,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;aAC/B,CAAC,CAAC;YAEL,OAAO,IAAI,EAAE,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;gBAEjC,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;oBAAE,MAAM;gBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAE5B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,KAAK,EAAE,CAAC;YACT,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QAChC,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC9B,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,sBAAsB,CAC5B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,sBAAsB,EACtB,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACvB,CACF,CAAC;QACH,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,4BAA4B,CAClC,QAAmB,EACnB,OAA+B,EAC/B,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,sBAAsB,CAChC,uBAAuB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EACxD,OAAO,EACP,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEO,0BAA0B,CAChC,QAAmB,EACnB,OAA+B,EAC/B,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,oBAAoB,CAC9B,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EACnD,OAAO,EACP,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEO,yBAAyB,CAC/B,QAAmB,EACnB,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAGT,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,oBAAoB,CAC9B,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAC5D,OAAO,EACP,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;IACH,CAAC;IAEO,uBAAuB,CAC7B,QAAmB,EACnB,GAAW,EACX,EAAU,EACV,OAA+B,EAC/B,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAET,EAAE,IAAI,EAAE;gBACV,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;gBAC7B,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;gBAC7B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;aAC5B,EAAE,CAAC;QACJ,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,oBAAoB,CAC9B,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,GAAG,IAAI,EAAE,cAAc,EAC/E,OAAO,EACP,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEO,0BAA0B,CAChC,MAAe,EACf,SAAkB;QAEnB,MAAM,OAAO,GAET;YACF,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;YACjC,GAAG,EAAE,eAAe;SACrB,CAAC;QACF,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAEpC,OAAO,IAAI,CAAC,oBAAoB,CAC9B,sDAAsD,EACtD,OAAO,EACP,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,eAAe,CAC1B,MAAe,EACf,SAAkB;QAEnB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,MAAM,IAAI,CAAC,4BAA4B,CACrC,CAAE,OAAO,CAAC,OAAO,CAAE,EACnB,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,MAAM,CACP,CAAC;QAEF,MAAM,IAAI,CAAC,0BAA0B,CACnC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EACD,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,MAAM,EACN,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,0BAA0B,CACnC;YACE,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;SACnB,EACD,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,MAAM,EACN,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,yBAAyB,CAClC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EACD,MAAM,EACN,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,uBAAuB,CAChC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;YAClB,OAAO,CAAC,OAAO;SAChB,EACD,GAAG,EACH,GAAG,EACH,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,MAAM,EACN,SAAS,CACV,CAAC;QACF,MAAM,IAAI,CAAC,uBAAuB,CAChC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;YAClB,OAAO,CAAC,OAAO;SAChB,EACD,GAAG,EACH,GAAG,EACH,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,MAAM,EACN,SAAS,CACV,CAAC;QACF,MAAM,IAAI,CAAC,uBAAuB,CAChC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;YAClB,OAAO,CAAC,OAAO;SAChB,EACD,GAAG,EACH,GAAG,EACH,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAAE,iCAAiC;QAChE,MAAM,EACN,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,0BAA0B,CACnC,MAAM,EACN,SAAS,CACV,CAAC;QAEF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gCAAgC,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YACvI,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAEpB,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBACpE,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAChF,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;oBACZ,oBAAoB,EAAE,IAAI;oBAC1B,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,QAAQ,EAAE,IAAI;oBACd,cAAc,EAAE,IAAI;oBACpB,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAE5H,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,IAAI,CAAC;gBACJ,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAEpB,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC7I,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAEnD,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;iBACxD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;CAED"}
|
package/dist/classes/expirer.mjs
CHANGED
|
@@ -27,7 +27,7 @@ export class Expirer {
|
|
|
27
27
|
const queued = await this.listQueuedAndActive();
|
|
28
28
|
const expired = [];
|
|
29
29
|
{ // scope
|
|
30
|
-
const log = commonsLogDoing('expirer', 'Searching for expired URLs');
|
|
30
|
+
const log = commonsLogDoing('expirer', 'Searching for expired URLs', CommonsFixedDuration.fromHis('00:00:01'));
|
|
31
31
|
const results = this.database.getUrls().find({ $and: [
|
|
32
32
|
{ status: { $nin: [
|
|
33
33
|
EStatus.ARCHIVED,
|
|
@@ -43,7 +43,7 @@ export class Expirer {
|
|
|
43
43
|
let found = 0;
|
|
44
44
|
while (true) {
|
|
45
45
|
tally++;
|
|
46
|
-
log.progress(`${tally}, ${found}
|
|
46
|
+
log.progress(`${tally}, ${found}`);
|
|
47
47
|
if (limit !== undefined && tally > limit)
|
|
48
48
|
break;
|
|
49
49
|
const row = await results.next();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expirer.mjs","sourceRoot":"","sources":["../../src/classes/expirer.mts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,0BAA0B,EAAmB,MAAM,oBAAoB,CAAC;AAEnJ,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAI3F,OAAO,EAAgC,aAAa,EAAE,MAAM,kCAAkC,CAAC;AAE/F,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,MAAc,EACd,QAAyB;QADzB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;IAChC,CAAC;IAEI,KAAK,CAAC,mBAAmB;QAChC,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;QAClG,MAAM,OAAO,GAA2C,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,SAAS,CAAc;YACrG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE;YACnE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;SAC/B,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3B,MAAM,MAAM,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAE3F,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE3B,OAAO,MAAM;YACX,gDAAgD;aAC/C,GAAG,CAAC,CAAC,CAAc,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,KAAc;QACjC,eAAe,CAAC,uBAAuB,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE1D,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"expirer.mjs","sourceRoot":"","sources":["../../src/classes/expirer.mts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,0BAA0B,EAAmB,MAAM,oBAAoB,CAAC;AAEnJ,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAI3F,OAAO,EAAgC,aAAa,EAAE,MAAM,kCAAkC,CAAC;AAE/F,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,MAAc,EACd,QAAyB;QADzB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;IAChC,CAAC;IAEI,KAAK,CAAC,mBAAmB;QAChC,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;QAClG,MAAM,OAAO,GAA2C,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,SAAS,CAAc;YACrG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE;YACnE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;SAC/B,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3B,MAAM,MAAM,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAE3F,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE3B,OAAO,MAAM;YACX,gDAAgD;aAC/C,GAAG,CAAC,CAAC,CAAc,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,KAAc;QACjC,eAAe,CAAC,uBAAuB,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE1D,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4BAA4B,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAEnI,MAAM,OAAO,GAA6B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACpE,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE;gCAChB,OAAO,CAAC,QAAQ;gCAChB,OAAO,CAAC,OAAO;gCACf,OAAO,CAAC,MAAM;gCACd,OAAO,CAAC,MAAM;6BACf,EAAE,EAAE;oBACL,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;iBACjC,EAAC,EACF,EAAE,CACH,CAAC;YAEF,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YAEhD,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACnC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,0BAA0B,CAAC,GAAG,EAAE,WAAW,CAAC;oBAAE,SAAS;gBAE5D,IAAI,CAAC;oBACJ,MAAM,QAAQ,GAAW,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAE5D,MAAM,SAAS,GAAa,GAAuB,CAAC,WAAW,CAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;oBAC3F,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,EAAE,CAAC;wBAClC,iEAAiE;wBACjE,OAAO,CAAC,IAAI,CAAE,GAAuB,CAAC,KAAK,CAAC,CAAC,CAAC;wBAC9C,KAAK,EAAE,CAAC;oBACT,CAAC;gBACF,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,SAAS;gBACV,CAAC;YACF,CAAC;YACD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAEnB,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC/B,CAAC;QAED,MAAM,OAAO,GAAe,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAE5D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACjF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAuB,KAAK;qBACvC,GAAG,CAAC,CAAC,EAAU,EAAoB,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEnE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC1B;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;iBACtB,CACF,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,SAAe;QACvC,eAAe,CAAC,kCAAkC,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE1D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mCAAmC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpI,MAAM,KAAK,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACtD,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE,EAAE;oBAC3D,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF,EAAE,CACH,CAAC,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAEnB,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC/B,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE,EAAE;oBAC3D,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;gBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;aACtB,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,IAAI;QACJ,4FAA4F;QAE5F,OAAO,IAAI,CAAC;IACb,CAAC;CACD"}
|
|
@@ -16,7 +16,7 @@ export const QUERY = async (args, databaseService, lists, _expiry, parsersConfig
|
|
|
16
16
|
const keyDictionary = parser.getDictionary();
|
|
17
17
|
if (!keyDictionary)
|
|
18
18
|
throw new Error('No dictionary available');
|
|
19
|
-
const log = commonsLogDoing('complex-english-parser', `Searching for complex english detections for domain ${domain}
|
|
19
|
+
const log = commonsLogDoing('complex-english-parser', `Searching for complex english detections for domain ${domain}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
20
20
|
const result = databaseService.getUrls().find({
|
|
21
21
|
domain: domain,
|
|
22
22
|
status: EStatus.DONE,
|
|
@@ -27,7 +27,7 @@ export const QUERY = async (args, databaseService, lists, _expiry, parsersConfig
|
|
|
27
27
|
let tally = 0;
|
|
28
28
|
while (true) {
|
|
29
29
|
tally++;
|
|
30
|
-
log.progress(`${tally}
|
|
30
|
+
log.progress(`${tally}`);
|
|
31
31
|
const row = await result.next();
|
|
32
32
|
if (row === null)
|
|
33
33
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complex-english.query.mjs","sourceRoot":"","sources":["../../src/queries/complex-english.query.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE/F,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAInD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,OAAO,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAM7E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAU3C,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAA6C,EAC9B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAwC,MAAM,CAAC,aAAa,EAAE,CAAC;IAClF,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,MAAM,GAAG,GAAuB,eAAe,CAAC,wBAAwB,EAAE,uDAAuD,MAAM,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"complex-english.query.mjs","sourceRoot":"","sources":["../../src/queries/complex-english.query.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE/F,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAInD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,OAAO,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAM7E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAU3C,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAA6C,EAC9B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAwC,MAAM,CAAC,aAAa,EAAE,CAAC;IAClF,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,MAAM,GAAG,GAAuB,eAAe,CAAC,wBAAwB,EAAE,uDAAuD,MAAM,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAErL,MAAM,MAAM,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACrE;QACE,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO,CAAC,IAAI;QACpB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAClC,EACD,EAAE,CACH,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE,CAAC;QACb,KAAK,EAAE,CAAC;QACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAEzB,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,MAAM,QAAQ,GAAkB,GAA+B,CAAC;QAEhE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,SAAS,EAAE,CAAC;YACtF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAClB,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAChC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAClC;gBAAE,SAAS;QACb,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,cAAc,EAAE,QAAQ,CAAC,cAAc;SACxC,CAAC,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAqB,MAAM,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,eAAe,CAAC,8BAA8B,CAAC,CAAC;YAChD,SAAS;QACV,CAAC;QAED,MAAM,GAAG,GAAiC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,GAAG,EAAE,CAAC;YACV,eAAe,CAAC,gCAAgC,CAAC,CAAC;YAClD,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAyB,GAAG,CAAC,GAAG,CAAC;aACzC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAgB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAE1F,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwC,EAAQ,EAAE;YAClF,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAa,WAAW;aAChC,IAAI,CAAC,IAAI,CAAC;aACV,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAa,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,OAAO,GAAW,IAAI,CAAC;YAE3B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAqB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAEhC,MAAM,WAAW,GAAa,aAAa,CAAC,OAAO,CAAC,CAAC;gBAErD,MAAM,KAAK,GAAa,CAAE,KAAK,OAAO,EAAE,CAAE,CAAC;gBAC3C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAErE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9D,CAAC;YAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC,CAAC"}
|
|
@@ -7,7 +7,7 @@ export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConf
|
|
|
7
7
|
commonsLogError('No domain specified');
|
|
8
8
|
return;
|
|
9
9
|
}
|
|
10
|
-
const log = commonsLogDoing('readability-core', `Scoring and ordering readability of DONE URLs for domain ${domain}
|
|
10
|
+
const log = commonsLogDoing('readability-core', `Scoring and ordering readability of DONE URLs for domain ${domain}`, CommonsFixedDuration.fromHis('00:00:01'));
|
|
11
11
|
const results = databaseService.getUrls().aggregate([
|
|
12
12
|
{ $match: {
|
|
13
13
|
status: { $nin: [EStatus.ARCHIVED, EStatus.INVALID] },
|
|
@@ -55,7 +55,7 @@ export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConf
|
|
|
55
55
|
let tally = 0;
|
|
56
56
|
while (true) {
|
|
57
57
|
tally++;
|
|
58
|
-
log.progress(`${tally}
|
|
58
|
+
log.progress(`${tally}`);
|
|
59
59
|
const row = await results.next();
|
|
60
60
|
if (row === null)
|
|
61
61
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"readability-score.query.mjs","sourceRoot":"","sources":["../../src/queries/readability-score.query.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAuB3F,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAA8C,EAC/B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,GAAG,GAAuB,eAAe,CAAC,kBAAkB,EAAE,4DAA4D,MAAM,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"readability-score.query.mjs","sourceRoot":"","sources":["../../src/queries/readability-score.query.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAmB,MAAM,oBAAoB,CAAC;AAE3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAuB3F,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAA8C,EAC/B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,GAAG,GAAuB,eAAe,CAAC,kBAAkB,EAAE,4DAA4D,MAAM,EAAE,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAEpL,MAAM,OAAO,GAA+B,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAU;QACvF,EAAE,MAAM,EAAE;gBACR,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;gBACvD,MAAM,EAAE,MAAM;gBACd,sBAAsB,EAAE,WAAW;aACpC,EAAE;QACH,EAAE,MAAM,EAAE;gBACR,2BAA2B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC3C,0BAA0B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;aAC3C,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE;oBACL,UAAU,EAAE,kCAAkC;oBAC9C,SAAS,EAAE,iCAAiC;oBAC5C,KAAK,EAAE,6BAA6B;oBACpC,IAAI,EAAE,4BAA4B;oBAClC,GAAG,EAAE,2BAA2B;iBACjC;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,CAAE,EAAE;oBACtD,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,EAAE,EAAE,YAAY,CAAE,EAAE,CAAE,EAAE;iBACpD;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE;oBACR,IAAI,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,EAAE,GAAG,CAAE,EAAE;oBACjE,GAAG,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,YAAY,EAAE,CAAC,CAAE,EAAE,EAAE,EAAE,CAAE,EAAE;iBAC7D;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,IAAI,EAAE,CAAE,gBAAgB,EAAE,eAAe,CAAE,EAAE,EAAE,CAAC,CAAE,EAAE;aAC3E,EAAE;QACH,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAc,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE,CAAC;QACb,KAAK,EAAE,CAAC;QACR,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAEzB,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElB,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACjK,CAAC;AACF,CAAC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { commonsAsyncTimeout, CommonsSchedule } from 'tscommons-esm-async';
|
|
2
|
-
import { commonsLogAlert, commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
|
|
2
|
+
import { commonsLogAlert, commonsLogDebug, commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
|
|
3
3
|
import { commonsGracefulAbortAddCallback } from 'nodecommons-esm-process';
|
|
4
4
|
import { Expirer } from '../classes/expirer.mjs';
|
|
5
5
|
import { Cleaner } from '../classes/cleaner.mjs';
|
|
@@ -45,8 +45,9 @@ export class MaintenanceServer {
|
|
|
45
45
|
this.isPaused = true;
|
|
46
46
|
claimedPause = true;
|
|
47
47
|
this.crawl.pause();
|
|
48
|
+
const log = commonsLogDoing('maintenance', 'Going down for maintenance');
|
|
48
49
|
for (let i = 30; i-- > 0;) {
|
|
49
|
-
|
|
50
|
+
log.progress(i);
|
|
50
51
|
try {
|
|
51
52
|
await commonsAsyncTimeout(1000);
|
|
52
53
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"maintenance.server.mjs","sourceRoot":"","sources":["../../src/servers/maintenance.server.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,qBAAqB,CAAC;AAEjG,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"maintenance.server.mjs","sourceRoot":"","sources":["../../src/servers/maintenance.server.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,qBAAqB,CAAC;AAEjG,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC7H,OAAO,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAC;AAI1E,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAMjD,IAAK,OAIJ;AAJD,WAAK,OAAO;IACX,4BAAiB,CAAA;IACjB,sDAA2C,CAAA;IAC3C,kDAAuC,CAAA;AACxC,CAAC,EAJI,OAAO,KAAP,OAAO,QAIX;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACtC,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC7B,OAAO,OAAO,CAAC,MAAM,CAAC;QACvB,KAAK,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YAC1C,OAAO,OAAO,CAAC,mBAAmB,CAAC;QACpC,KAAK,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE;YACxC,OAAO,OAAO,CAAC,iBAAiB,CAAC;IACnC,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAanB;IACA;IAbF,QAAQ,CAAkB;IAC1B,OAAO,CAAU;IACjB,OAAO,CAAU;IAEjB,QAAQ,GAAY,KAAK,CAAC;IAC1B,SAAS,GAAY,KAAK,CAAC;IAEnC,YACE,KAA6B,EAC7B,MAAc,EACd,KAAY,EACZ,QAAyB,EACjB,KAAkB,EAClB,SAAkB;QADlB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAS;QAE3B,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CACjB,KAAK,EACL,CAAC,MAAc,EAAqB,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EACxD,KAAK,EAAE,MAAe,EAAiB,EAAE;YACxC,yBAAyB;YACzB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,eAAe,CAAC,gDAAgD,CAAC,CAAC;gBAClE,OAAO;YACR,CAAC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YAEtB,IAAI,CAAC;gBACJ,IAAI,YAAY,GAAY,KAAK,CAAC;gBAElC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACrB,YAAY,GAAG,IAAI,CAAC;oBAEpB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;oBAEnB,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;oBAC7F,KAAK,IAAI,CAAC,GAAW,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;wBACnC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAEhB,IAAI,CAAC;4BACJ,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBACjC,CAAC;wBAAC,OAAO,GAAG,EAAE,CAAC;4BACd,gBAAgB;wBACjB,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAE3B,IAAI,YAAY,EAAE,CAAC;oBAClB,eAAe,CAAC,2BAA2B,CAAC,CAAC;oBAE7C,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;oBAEtB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,CAAC;YACF,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,eAAe,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;YACvC,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACxB,CAAC;QACF,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE5C,+BAA+B,CAAC,GAAS,EAAE;YAC1C,eAAe,CAAC,wDAAwD,CAAC,CAAC;YAC1E,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,MAAe;QACpC,QAAQ,MAAM,EAAE,CAAC;YAChB,KAAK,OAAO,CAAC,MAAM;gBAClB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM;YACP,KAAK,OAAO,CAAC,mBAAmB;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBACvC,MAAM;YACP,KAAK,OAAO,CAAC,iBAAiB;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC9D,MAAM;QACR,CAAC;IACF,CAAC;IAEM,KAAK;QACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hydra-crawler",
|
|
3
|
-
"version": "3.3.
|
|
3
|
+
"version": "3.3.6",
|
|
4
4
|
"description": "Node.js Hydra web crawler",
|
|
5
5
|
"author": "Pete Morris",
|
|
6
6
|
"license": "ISC",
|
|
@@ -18,41 +18,42 @@
|
|
|
18
18
|
"main": "dist/cli.mjs",
|
|
19
19
|
"dependencies": {
|
|
20
20
|
"chardet": "^2.1.0",
|
|
21
|
-
"cheerio": "^1.
|
|
22
|
-
"express": "^
|
|
23
|
-
"hydra-crawler-esm-assets": "^3.1.
|
|
21
|
+
"cheerio": "^1.2.0",
|
|
22
|
+
"express": "^4.21.2",
|
|
23
|
+
"hydra-crawler-esm-assets": "^3.1.1",
|
|
24
24
|
"jpeg-js": "^0.4.4",
|
|
25
25
|
"memcached-promisify": "^2.0.0",
|
|
26
|
-
"mongodb": "^6.
|
|
27
|
-
"nodecommons-esm-app": "^
|
|
28
|
-
"nodecommons-esm-app-socket-io": "^
|
|
29
|
-
"nodecommons-esm-cli": "^
|
|
30
|
-
"nodecommons-esm-config": "^
|
|
31
|
-
"nodecommons-esm-database": "^0.
|
|
32
|
-
"nodecommons-esm-database-mongodb": "^
|
|
33
|
-
"nodecommons-esm-express": "^
|
|
34
|
-
"nodecommons-esm-file": "^0.1
|
|
35
|
-
"nodecommons-esm-http": "^
|
|
36
|
-
"nodecommons-esm-log": "^1.
|
|
37
|
-
"nodecommons-esm-process": "^
|
|
38
|
-
"nodecommons-esm-rest": "^
|
|
39
|
-
"nodecommons-esm-security": "^
|
|
40
|
-
"nodecommons-esm-socket-io-ticketed": "^
|
|
41
|
-
"tscommons-esm-async": "^
|
|
42
|
-
"tscommons-esm-config": "^
|
|
43
|
-
"tscommons-esm-core": "^0.2
|
|
44
|
-
"tscommons-esm-format": "^
|
|
45
|
-
"tscommons-esm-http": "^
|
|
46
|
-
"tscommons-esm-rest": "^
|
|
26
|
+
"mongodb": "^6.12.0",
|
|
27
|
+
"nodecommons-esm-app": "^1.0.3",
|
|
28
|
+
"nodecommons-esm-app-socket-io": "^3.0.7",
|
|
29
|
+
"nodecommons-esm-cli": "^1.0.1",
|
|
30
|
+
"nodecommons-esm-config": "^1.0.1",
|
|
31
|
+
"nodecommons-esm-database": "^1.0.1",
|
|
32
|
+
"nodecommons-esm-database-mongodb": "^1.0.1",
|
|
33
|
+
"nodecommons-esm-express": "^2.0.3",
|
|
34
|
+
"nodecommons-esm-file": "^1.0.1",
|
|
35
|
+
"nodecommons-esm-http": "^3.0.1",
|
|
36
|
+
"nodecommons-esm-log": "^3.1.2",
|
|
37
|
+
"nodecommons-esm-process": "^2.0.3",
|
|
38
|
+
"nodecommons-esm-rest": "^1.1.4",
|
|
39
|
+
"nodecommons-esm-security": "^2.1.1",
|
|
40
|
+
"nodecommons-esm-socket-io-ticketed": "^2.0.7",
|
|
41
|
+
"tscommons-esm-async": "^3.0.1",
|
|
42
|
+
"tscommons-esm-config": "^2.0.1",
|
|
43
|
+
"tscommons-esm-core": "^1.0.2",
|
|
44
|
+
"tscommons-esm-format": "^1.0.1",
|
|
45
|
+
"tscommons-esm-http": "^3.0.1",
|
|
46
|
+
"tscommons-esm-rest": "^2.0.2",
|
|
47
47
|
"unfluff": "^3.2.0"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@stylistic/eslint-plugin-ts": "^2.11.0",
|
|
51
50
|
"@types/node": "^22.10.0",
|
|
52
|
-
"
|
|
53
|
-
"eslint-plugin
|
|
54
|
-
"
|
|
55
|
-
"
|
|
51
|
+
"@types/express": "^4.17.25",
|
|
52
|
+
"@stylistic/eslint-plugin": "^5.10.0",
|
|
53
|
+
"eslint-plugin-import": "^2.32.0",
|
|
54
|
+
"eslint-plugin-prefer-arrow-functions": "^3.9.1",
|
|
55
|
+
"typescript": "~5.8.0",
|
|
56
|
+
"typescript-eslint": "^8.53.0"
|
|
56
57
|
},
|
|
57
58
|
"files": [
|
|
58
59
|
"dist/**/*"
|