hydra-crawler 3.2.10 → 3.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -29,7 +29,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
29
29
|
log.success();
|
|
30
30
|
await super.init();
|
|
31
31
|
}
|
|
32
|
-
async forward(limit) {
|
|
32
|
+
async forward(limit, domain) {
|
|
33
33
|
if (!this.databaseService)
|
|
34
34
|
throw new Error('Database service has not been set');
|
|
35
35
|
let tally = 0;
|
|
@@ -37,8 +37,13 @@ export class DenylistApp extends CommonsApp {
|
|
|
37
37
|
const urls = [];
|
|
38
38
|
{ // scope
|
|
39
39
|
const log = commonsLogDoing('denylist', 'Searching for non-DENY urls on the denylist');
|
|
40
|
+
const clauses = {
|
|
41
|
+
status: { $nin: [EStatus.DENY, EStatus.ARCHIVED, EStatus.INVALID] }
|
|
42
|
+
};
|
|
43
|
+
if (domain)
|
|
44
|
+
clauses.domain = domain;
|
|
40
45
|
const result = this.databaseService.getUrls()
|
|
41
|
-
.find(
|
|
46
|
+
.find(clauses, {});
|
|
42
47
|
while (true) {
|
|
43
48
|
tally++;
|
|
44
49
|
if ((tally % 100) === 0)
|
|
@@ -71,7 +76,7 @@ export class DenylistApp extends CommonsApp {
|
|
|
71
76
|
log.success();
|
|
72
77
|
}
|
|
73
78
|
}
|
|
74
|
-
async reverse(limit) {
|
|
79
|
+
async reverse(limit, domain) {
|
|
75
80
|
if (!this.databaseService)
|
|
76
81
|
throw new Error('Database service has not been set');
|
|
77
82
|
let tally = 0;
|
|
@@ -79,8 +84,13 @@ export class DenylistApp extends CommonsApp {
|
|
|
79
84
|
const urls = [];
|
|
80
85
|
{ // scope
|
|
81
86
|
const log = commonsLogDoing('denylist', 'Searching for DENY urls not on the denylist');
|
|
87
|
+
const clauses = {
|
|
88
|
+
status: EStatus.DENY
|
|
89
|
+
};
|
|
90
|
+
if (domain)
|
|
91
|
+
clauses.domain = domain;
|
|
82
92
|
const result = this.databaseService.getUrls()
|
|
83
|
-
.find(
|
|
93
|
+
.find(clauses, {});
|
|
84
94
|
while (true) {
|
|
85
95
|
tally++;
|
|
86
96
|
if ((tally % 100) === 0)
|
|
@@ -115,8 +125,10 @@ export class DenylistApp extends CommonsApp {
|
|
|
115
125
|
}
|
|
116
126
|
async run() {
|
|
117
127
|
const args = new CommonsArgs();
|
|
118
|
-
|
|
119
|
-
|
|
128
|
+
const limit = args.getFirstNumberOrUndefined('limit');
|
|
129
|
+
const domain = args.getFirstValueOrUndefined('domain');
|
|
130
|
+
await this.forward(limit, domain);
|
|
131
|
+
await this.reverse(limit, domain);
|
|
120
132
|
}
|
|
121
133
|
}
|
|
122
134
|
//# sourceMappingURL=denylist.app.mjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"denylist.app.mjs","sourceRoot":"","sources":["../../src/apps/denylist.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,
|
|
1
|
+
{"version":3,"file":"denylist.app.mjs","sourceRoot":"","sources":["../../src/apps/denylist.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAmB,MAAM,oBAAoB,CAAC;AAExE,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,WAAY,SAAQ,UAAU;IAClC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc,EAAE,MAAe;QACpD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,CAAC,CAAC;YAE3G,MAAM,OAAO,GAGT;gBACF,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;aACtE,CAAC;YACF,IAAI,MAAM;gBAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;YAEpC,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,OAAO,EACP,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBAC5D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAErD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAC;YAExF,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,IAAI;qBACrB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc,EAAE,MAAe;QACpD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,CAAC,CAAC;YAE3G,MAAM,OAAO,GAGT;gBACF,MAAM,EAAE,OAAO,CAAC,IAAI;aACrB,CAAC;YACF,IAAI,MAAM;gBAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;YAEpC,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,OAAO,EACP,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBAC5D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,4BAA4B,CAAC,CAAC;YAE1F,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;QAE5C,MAAM,KAAK,GAAqB,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QACxE,MAAM,MAAM,GAAqB,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;QAEzE,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unarchive-urls.app.mjs","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,CAAC;YAE/F,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;gBAC3D,KAAK,EAAE,CAAC;gBAER,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;
|
|
1
|
+
{"version":3,"file":"unarchive-urls.app.mjs","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,CAAC;YAE/F,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;gBAC3D,KAAK,EAAE,CAAC;gBAER,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,0BAA0B,CAAC,CAAC;YAC9F,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACpB,KAAK,IAAI,GAAG,CAAC;gBAEb,MAAM,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;iBACjC,CACF,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}
|
package/dist/classes/cleaner.mjs
CHANGED
|
@@ -75,21 +75,21 @@ export class Cleaner {
|
|
|
75
75
|
});
|
|
76
76
|
}, hardLimit);
|
|
77
77
|
}
|
|
78
|
-
detectAndDeleteStatusOrphans(statuses, hardLimit) {
|
|
79
|
-
return this.detectAndDeleteOrphans(`Detecting deletable ${statuses.join(', ')} orphan urls`, { status: { $in: statuses } },
|
|
78
|
+
detectAndDeleteStatusOrphans(statuses, matcher, hardLimit) {
|
|
79
|
+
return this.detectAndDeleteOrphans(`Detecting deletable ${statuses.join(', ')} orphan urls`, { status: { $in: statuses } }, matcher, hardLimit);
|
|
80
80
|
}
|
|
81
|
-
detectAndFlagStatusOrphans(statuses, hardLimit) {
|
|
82
|
-
return this.detectAndFlagOrphans(`Detecting orphans for ${statuses.join(', ')} urls`, { status: { $in: statuses } },
|
|
81
|
+
detectAndFlagStatusOrphans(statuses, matcher, hardLimit) {
|
|
82
|
+
return this.detectAndFlagOrphans(`Detecting orphans for ${statuses.join(', ')} urls`, { status: { $in: statuses } }, matcher, hardLimit);
|
|
83
83
|
}
|
|
84
84
|
detectNonAllowlistOrphans(statuses, hardLimit) {
|
|
85
85
|
return this.detectAndFlagOrphans(`Detecting non-allowlist ${statuses.join(', ')} orphan urls`, { status: { $in: statuses } }, (url) => !this.lists.match(EList.ALLOW, url.url), hardLimit);
|
|
86
86
|
}
|
|
87
|
-
detectStatusCodeOrphans(gt, lt, hardLimit) {
|
|
87
|
+
detectStatusCodeOrphans(gt, lt, matcher, hardLimit) {
|
|
88
88
|
return this.detectAndFlagOrphans(`Detecting DONE status code between ${gt}<${lt} orphan urls`, { $and: [
|
|
89
89
|
{ status: EStatus.DONE },
|
|
90
90
|
{ statusCode: { $gt: gt } },
|
|
91
91
|
{ statusCode: { $lt: lt } }
|
|
92
|
-
] },
|
|
92
|
+
] }, matcher, hardLimit);
|
|
93
93
|
}
|
|
94
94
|
async purgeOrphanUrls(hardLimit) {
|
|
95
95
|
{ // scope
|
|
@@ -102,20 +102,21 @@ export class Cleaner {
|
|
|
102
102
|
});
|
|
103
103
|
log.success();
|
|
104
104
|
}
|
|
105
|
-
await this.detectAndDeleteStatusOrphans([EStatus.INVALID],
|
|
106
|
-
await this.detectAndDeleteStatusOrphans([EStatus.INVALID], hardLimit);
|
|
105
|
+
await this.detectAndDeleteStatusOrphans([EStatus.INVALID], (_url) => true);
|
|
107
106
|
await this.detectAndFlagStatusOrphans([
|
|
108
|
-
EStatus.DENY,
|
|
109
107
|
EStatus.FAILED,
|
|
110
|
-
EStatus.DEAD
|
|
108
|
+
EStatus.DEAD
|
|
109
|
+
], (_url) => true, hardLimit);
|
|
110
|
+
await this.detectAndFlagStatusOrphans([
|
|
111
|
+
EStatus.DENY,
|
|
111
112
|
EStatus.DISALLOWED
|
|
112
|
-
], hardLimit);
|
|
113
|
+
], (url) => !this.lists.match(EList.ALLOW, url.url), hardLimit);
|
|
113
114
|
await this.detectNonAllowlistOrphans([
|
|
114
115
|
EStatus.QUEUED,
|
|
115
116
|
EStatus.DONE
|
|
116
117
|
], hardLimit);
|
|
117
|
-
await this.detectStatusCodeOrphans(300, 310, hardLimit);
|
|
118
|
-
await this.detectStatusCodeOrphans(400, 500, hardLimit);
|
|
118
|
+
await this.detectStatusCodeOrphans(300, 310, (url) => !this.lists.match(EList.ALLOW, url.url), hardLimit);
|
|
119
|
+
await this.detectStatusCodeOrphans(400, 500, (url) => !this.lists.match(EList.ALLOW, url.url), hardLimit);
|
|
119
120
|
let tally = 0;
|
|
120
121
|
{ // scope
|
|
121
122
|
const log = commonsLogDoing('cleaner', 'Removing orphan outgoing links');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGnF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAoC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzE,SAAS,CAAO;gBACf,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACf,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBACjB,EAAE,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,EAAE;gBACnB,gHAAgH;gBAChH,EAAE,OAAO,EAAE;wBACT,IAAI,EAAE,OAAO;wBACb,UAAU,EAAE,KAAK;wBACjB,YAAY,EAAE,UAAU;wBACxB,QAAQ,EAAE;4BACR,EAAE,MAAM,EAAE,CAAC,EAAE;yBACd;wBACD,EAAE,EAAE,OAAO;qBACZ,EAAE;gBACH,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC7C,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;aAC/B,CAAC,CAAC;YAEL,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;gBAEzD,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;oBAAE,MAAM;gBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAE5B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,KAAK,EAAE,CAAC;YACT,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QAChC,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC9B,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBAEvD,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,sBAAsB,CAC5B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,sBAAsB,EACtB,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACvB,CACF,CAAC;QACH,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,4BAA4B,CAClC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,sBAAsB,CAChC,uBAAuB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EACxD,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,
|
|
1
|
+
{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGnF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAoC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzE,SAAS,CAAO;gBACf,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACf,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBACjB,EAAE,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,EAAE;gBACnB,gHAAgH;gBAChH,EAAE,OAAO,EAAE;wBACT,IAAI,EAAE,OAAO;wBACb,UAAU,EAAE,KAAK;wBACjB,YAAY,EAAE,UAAU;wBACxB,QAAQ,EAAE;4BACR,EAAE,MAAM,EAAE,CAAC,EAAE;yBACd;wBACD,EAAE,EAAE,OAAO;qBACZ,EAAE;gBACH,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC7C,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;aAC/B,CAAC,CAAC;YAEL,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;gBAEzD,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;oBAAE,MAAM;gBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAE5B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,KAAK,EAAE,CAAC;YACT,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QAChC,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC9B,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBAEvD,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,sBAAsB,CAC5B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,sBAAsB,EACtB,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACvB,CACF,CAAC;QACH,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,4BAA4B,CAClC,QAAmB,EACnB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,sBAAsB,CAChC,uBAAuB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EACxD,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEO,0BAA0B,CAChC,QAAmB,EACnB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EACnD,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEO,yBAAyB,CAC/B,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAC5D,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;IACH,CAAC;IAEO,uBAAuB,CAC7B,EAAU,EACV,EAAU,EACV,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,sCAAsC,EAAE,IAAI,EAAE,cAAc,EAC5D,EAAE,IAAI,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;gBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;gBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;aAC5B,EAAE,EACH,OAAO,EACP,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,SAAkB;QAC9C,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,MAAM,IAAI,CAAC,4BAA4B,CACrC,CAAE,OAAO,CAAC,OAAO,CAAE,EACnB,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,CAC9B,CAAC;QAEF,MAAM,IAAI,CAAC,0BAA0B,CACnC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EACD,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,0BAA0B,CACnC;YACE,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;SACnB,EACD,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,yBAAyB,CAClC;YACE,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EACD,SAAS,CACV,CAAC;QAEF,MAAM,IAAI,CAAC,uBAAuB,CAChC,GAAG,EACH,GAAG,EACH,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;QACF,MAAM,IAAI,CAAC,uBAAuB,CAChC,GAAG,EACH,GAAG,EACH,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;QAEF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAC;YAC7F,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAE7C,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBACpE,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAChF,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;oBACZ,oBAAoB,EAAE,IAAI;oBAC1B,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,QAAQ,EAAE,IAAI;oBACd,cAAc,EAAE,IAAI;oBACpB,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;YAClG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACJ,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAElF,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,IAAI,CAAC;gBACJ,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAE7C,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,CAAC,CAAC;YACnG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAE3E,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;iBACxD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;CAED"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hydra-crawler",
|
|
3
|
-
"version": "3.2.
|
|
3
|
+
"version": "3.2.12",
|
|
4
4
|
"description": "Node.js Hydra web crawler",
|
|
5
5
|
"author": "Pete Morris",
|
|
6
6
|
"license": "ISC",
|
|
@@ -31,9 +31,9 @@
|
|
|
31
31
|
"nodecommons-esm-database": "^0.0.12",
|
|
32
32
|
"nodecommons-esm-database-mongodb": "^0.0.5",
|
|
33
33
|
"nodecommons-esm-express": "^0.0.10",
|
|
34
|
-
"nodecommons-esm-file": "^0.
|
|
34
|
+
"nodecommons-esm-file": "^0.1.2",
|
|
35
35
|
"nodecommons-esm-http": "^0.0.8",
|
|
36
|
-
"nodecommons-esm-log": "^0.0
|
|
36
|
+
"nodecommons-esm-log": "^0.1.0",
|
|
37
37
|
"nodecommons-esm-process": "^0.1.0",
|
|
38
38
|
"nodecommons-esm-rest": "^0.0.9",
|
|
39
39
|
"nodecommons-esm-security": "^0.0.5",
|