hydra-crawler 2.8.3 → 2.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/apps/cleanup.app.js
CHANGED
|
@@ -10,7 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
import { commonsTypeHasPropertyNumber } from 'tscommons-es-core';
|
|
11
11
|
import { isIUrl } from 'hydra-crawler-ts-assets';
|
|
12
12
|
import { EStatus } from 'hydra-crawler-ts-assets';
|
|
13
|
-
import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
|
|
13
|
+
import { commonsOutputDoing, commonsOutputPercent, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
|
|
14
14
|
import { CommonsApp } from 'nodecommons-es-app';
|
|
15
15
|
import { Lists } from '../classes/lists';
|
|
16
16
|
import { Cleaner } from '../classes/cleaner';
|
|
@@ -47,7 +47,8 @@ export class CleanupApp extends CommonsApp {
|
|
|
47
47
|
return __awaiter(this, void 0, void 0, function* () {
|
|
48
48
|
if (!this.databaseService)
|
|
49
49
|
throw new Error('Database service has not been set');
|
|
50
|
-
commonsOutputDoing('
|
|
50
|
+
commonsOutputDoing('Enumerating urls for DENY, FAILED, DEAD and DISALLOWED');
|
|
51
|
+
const urls = [];
|
|
51
52
|
let tally = 0;
|
|
52
53
|
while (true) {
|
|
53
54
|
const result = this.databaseService.getUrls()
|
|
@@ -67,7 +68,7 @@ export class CleanupApp extends CommonsApp {
|
|
|
67
68
|
const row = yield result.next();
|
|
68
69
|
if (row === null)
|
|
69
70
|
break;
|
|
70
|
-
|
|
71
|
+
urls.push(row.url);
|
|
71
72
|
}
|
|
72
73
|
break;
|
|
73
74
|
}
|
|
@@ -77,6 +78,15 @@ export class CleanupApp extends CommonsApp {
|
|
|
77
78
|
}
|
|
78
79
|
}
|
|
79
80
|
commonsOutputResult(tally);
|
|
81
|
+
commonsOutputDoing('Resetting links for DENY, FAILED, DEAD and DISALLOWED');
|
|
82
|
+
let i = 0;
|
|
83
|
+
for (const url of urls) {
|
|
84
|
+
i++;
|
|
85
|
+
if ((i % 100) === 0)
|
|
86
|
+
commonsOutputPercent(i, urls.length);
|
|
87
|
+
yield this.databaseService.getLinks().deleteMany({ url: url });
|
|
88
|
+
}
|
|
89
|
+
commonsOutputSuccess();
|
|
80
90
|
});
|
|
81
91
|
}
|
|
82
92
|
links() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup.app.js","sourceRoot":"","sources":["../../src/apps/cleanup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"cleanup.app.js","sourceRoot":"","sources":["../../src/apps/cleanup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAChJ,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAU7C,MAAM,OAAO,UAAW,SAAQ,UAAU;IAKzC,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK;QAEtC,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAItC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,wDAAwD,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACpE,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACrB,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,MAAM;4BACd,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,UAAU;yBACnB,EAAE,EAAE,CAAC;qBACL,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC3C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;qBACnB;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,uDAAuD,CAAC,CAAC;YAC5E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,oBAAoB,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE1D,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;aAC/D;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YACjD,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,MAAM,GAA8B,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAE9F,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE/E,MAAM,IAAI,GAAe,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAEzB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;oBAAE,SAAS;gBAEnC,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3E,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CACxB,CAAC;gBAEF,MAAM,OAAO,GAAa,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;qBACrF,GAAG,CAAC,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEnC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;oBACtB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAC1C;gBAED,KAAK,GAAG,EAAE,CAAC;aACX;YACD,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEpC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;YAC5C,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;aACvE;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1C,IAAI,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAE1C,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAEtF,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACvE,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACnC,CAAC;KAAA;CACD"}
|