hydra-crawler 3.2.8 → 3.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/classes/cleaner.mjs
CHANGED
|
@@ -18,18 +18,17 @@ export class Cleaner {
|
|
|
18
18
|
const log = commonsLogDoing('cleaner', detectingCaption);
|
|
19
19
|
const urls = this.databaseService.getUrls()
|
|
20
20
|
.aggregate([
|
|
21
|
-
{ $match: { $
|
|
22
|
-
|
|
23
|
-
{
|
|
24
|
-
{ orphan: null },
|
|
25
|
-
{ orphan: false }
|
|
26
|
-
] }
|
|
21
|
+
{ $match: { $or: [
|
|
22
|
+
{ orphan: null },
|
|
23
|
+
{ orphan: false }
|
|
27
24
|
] } },
|
|
25
|
+
{ $match: clauses },
|
|
26
|
+
// NB, using $expr $eq in the pipeline doesn't use indexes, so more efficient to use localField and foreignField
|
|
28
27
|
{ $lookup: {
|
|
29
28
|
from: 'links',
|
|
30
|
-
|
|
29
|
+
localField: 'url',
|
|
30
|
+
foreignField: 'outgoing',
|
|
31
31
|
pipeline: [
|
|
32
|
-
{ $match: { $expr: { $eq: ['$outgoing', '$$url'] } } },
|
|
33
32
|
{ $limit: 1 }
|
|
34
33
|
],
|
|
35
34
|
as: 'links'
|
|
@@ -93,6 +92,17 @@ export class Cleaner {
|
|
|
93
92
|
] }, (_url) => true, hardLimit);
|
|
94
93
|
}
|
|
95
94
|
async purgeOrphanUrls(hardLimit) {
|
|
95
|
+
{ // scope
|
|
96
|
+
const log = commonsLogDoing('cleaner', 'Unsetting any existing orphan flags');
|
|
97
|
+
await this.databaseService.getUrls()
|
|
98
|
+
.updateMany({ orphan: true }, {
|
|
99
|
+
$unset: {
|
|
100
|
+
orphan: true
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
log.success();
|
|
104
|
+
}
|
|
105
|
+
await this.detectAndDeleteStatusOrphans([EStatus.INVALID], hardLimit);
|
|
96
106
|
await this.detectAndDeleteStatusOrphans([EStatus.INVALID], hardLimit);
|
|
97
107
|
await this.detectAndFlagStatusOrphans([
|
|
98
108
|
EStatus.DENY,
|
|
@@ -164,6 +174,16 @@ export class Cleaner {
|
|
|
164
174
|
});
|
|
165
175
|
log.success();
|
|
166
176
|
}
|
|
177
|
+
{ // scope
|
|
178
|
+
const log = commonsLogDoing('cleaner', 'Unsetting any leftover orphan flags');
|
|
179
|
+
await this.databaseService.getUrls()
|
|
180
|
+
.updateMany({ orphan: true }, {
|
|
181
|
+
$unset: {
|
|
182
|
+
orphan: true
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
log.success();
|
|
186
|
+
}
|
|
167
187
|
}
|
|
168
188
|
async purgeEmptyDomains() {
|
|
169
189
|
let tally = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGnF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAoC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzE,SAAS,CAAO;gBACf,EAAE,MAAM,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGnF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAoC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzE,SAAS,CAAO;gBACf,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACf,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBACjB,EAAE,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,EAAE;gBACnB,gHAAgH;gBAChH,EAAE,OAAO,EAAE;wBACT,IAAI,EAAE,OAAO;wBACb,UAAU,EAAE,KAAK;wBACjB,YAAY,EAAE,UAAU;wBACxB,QAAQ,EAAE;4BACR,EAAE,MAAM,EAAE,CAAC,EAAE;yBACd;wBACD,EAAE,EAAE,OAAO;qBACZ,EAAE;gBACH,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC7C,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;aAC/B,CAAC,CAAC;YAEL,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;gBAEzD,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;oBAAE,MAAM;gBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAE5B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,KAAK,EAAE,CAAC;YACT,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QAChC,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC9B,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBAEvD,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,sBAAsB,CAC5B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,sBAAsB,EACtB,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACvB,CACF,CAAC;QACH,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,4BAA4B,CAClC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,sBAAsB,CAChC,uBAAuB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EACxD,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEO,0BAA0B,CAChC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EACnD,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEO,yBAAyB,CAC/B,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAC5D,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;IACH,CAAC;IAEO,uBAAuB,CAC7B,EAAU,EACV,EAAU,EACV,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,sCAAsC,EAAE,IAAI,EAAE,cAAc,EAC5D,EAAE,IAAI,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;gBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;gBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;aAC5B,EAAE,EACH,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,SAAkB;QAC9C,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,MAAM,IAAI,CAAC,4BAA4B,CAAC,CAAE,OAAO,CAAC,OAAO,CAAE,EAAE,SAAS,CAAC,CAAC;QAExE,MAAM,IAAI,CAAC,4BAA4B,CAAC,CAAE,OAAO,CAAC,OAAO,CAAE,EAAE,SAAS,CAAC,CAAC;QAExE,MAAM,IAAI,CAAC,0BAA0B,CAAC;YACpC,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;SACnB,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,yBAAyB,CAAC;YACnC,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QAExD,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAC;YAC7F,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAE7C,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBACpE,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAChF,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;oBACZ,oBAAoB,EAAE,IAAI;oBAC1B,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,QAAQ,EAAE,IAAI;oBACd,cAAc,EAAE,IAAI;oBACpB,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAElF,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,IAAI,CAAC;gBACJ,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAE7C,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,CAAC,CAAC;YACnG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAE3E,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;iBACxD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;CAED"}
|