hydra-crawler 2.8.4 → 3.0.1

package/dist/apis/{autocomplete.api.d.ts → autocomplete.api.d.mts}

@@ -1,8 +1,8 @@
 import * as express from 'express';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { RestServer } from '../servers/rest.server';
-import { DatabaseService } from '../services/database.service';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { RestServer } from '../servers/rest.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class AutocompleteApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     private databaseService;
     constructor(restServer: RestServer, databaseService: DatabaseService, path: string);

package/dist/apis/autocomplete.api.mjs

@@ -0,0 +1,14 @@
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+export class AutocompleteApi extends CommonsRestApi {
+    databaseService;
+    constructor(restServer, databaseService, path) {
+        super(restServer);
+        this.databaseService = databaseService;
+        super.getHandler(`${path}rest/autocomplete/domain/:term[idname]`, async (req, _res) => {
+            const domains = await this.databaseService.listDomainsByLike(req.strictParams.term);
+            return domains
+                .map((domain) => domain.domain);
+        });
+    }
+}
+//# sourceMappingURL=autocomplete.api.mjs.map

package/dist/apis/autocomplete.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"autocomplete.api.mjs","sourceRoot":"","sources":["../../src/apis/autocomplete.api.mts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAMtD,MAAM,OAAO,eAAgB,SAAQ,cAAiE;IAG3F;IAFV,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,wCAAwC,EAC/C,KAAK,EAAE,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,MAAM,OAAO,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAc,CAAC,CAAC;YAEzG,OAAO,OAAO;iBACX,GAAG,CAAC,CAAC,MAAe,EAAU,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/{bugs.api.d.ts → bugs.api.d.mts}

@@ -1,8 +1,8 @@
 import * as express from 'express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { RestServer } from '../servers/rest.server';
-import { DatabaseService } from '../services/database.service';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { RestServer } from '../servers/rest.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class BugsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     private databaseService;
     constructor(restServer: RestServer, databaseService: DatabaseService, path: string);

package/dist/apis/bugs.api.mjs

@@ -0,0 +1,18 @@
+import { commonsTypeEncode } from 'tscommons-esm-core';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+export class BugsApi extends CommonsRestApi {
+    databaseService;
+    constructor(restServer, databaseService, path) {
+        super(restServer);
+        this.databaseService = databaseService;
+        super.getHandler(`${path}rest/bugs/php`, async (_req, _res) => {
+            const result = await this.databaseService.listPhpErrors();
+            return commonsTypeEncode(result);
+        });
+        super.getHandler(`${path}rest/bugs/asp`, async (_req, _res) => {
+            const result = await this.databaseService.listAspErrors();
+            return commonsTypeEncode(result);
+        });
+    }
+}
+//# sourceMappingURL=bugs.api.mjs.map

package/dist/apis/bugs.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"bugs.api.mjs","sourceRoot":"","sources":["../../src/apis/bugs.api.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,oBAAoB,CAAC;AAMjE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAOtD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAGnF;IAFV,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,eAAe,EACtB,KAAK,EAAE,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,MAAM,GAA4B,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;YAEnF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CACF,CAAC;QAEF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,eAAe,EACtB,KAAK,EAAE,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,MAAM,GAA4B,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;YAEnF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/{crawl.api.d.ts → crawl.api.d.mts}

@@ -1,8 +1,8 @@
 import * as express from 'express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { RestServer } from '../servers/rest.server';
-import { DatabaseService } from '../services/database.service';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { RestServer } from '../servers/rest.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class CrawlApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     private databaseService;
     constructor(restServer: RestServer, databaseService: DatabaseService, path: string);

package/dist/apis/crawl.api.mjs

@@ -0,0 +1,17 @@
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+export class CrawlApi extends CommonsRestApi {
+    databaseService;
+    constructor(restServer, databaseService, path) {
+        super(restServer);
+        this.databaseService = databaseService;
+        super.getHandler(`${path}rest/crawl/queue`, async (_req, _res) => {
+            const tallies = await this.databaseService.listDomainQueuedTallies();
+            return Array.from(tallies.keys())
+                .map((domain) => ({
+                domain: domain,
+                queue: tallies.get(domain) || 0
+            }));
+        });
+    }
+}
+//# sourceMappingURL=crawl.api.mjs.map

package/dist/apis/crawl.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.api.mjs","sourceRoot":"","sources":["../../src/apis/crawl.api.mts"],"names":[],"mappings":"AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAOtD,MAAM,OAAO,QAAS,SAAQ,cAAiE;IAGpF;IAFV,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,kBAAkB,EACzB,KAAK,EAAE,IAAqB,EAAE,IAAsB,EAA2B,EAAE;YAChF,MAAM,OAAO,GAAwB,MAAM,IAAI,CAAC,eAAe,CAAC,uBAAuB,EAAE,CAAC;YAE1F,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;iBAC9B,GAAG,CAAC,CAAC,MAAc,EAAgB,EAAE,CAAC,CAAC;gBACtC,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;aAChC,CAAC,CAAC,CAAC;QACP,CAAC,CACF,CAAC;IAEH,CAAC;CACD"}

package/dist/apis/{domains.api.d.ts → domains.api.d.mts}

@@ -1,8 +1,8 @@
 import * as express from 'express';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { RestServer } from '../servers/rest.server';
-import { DatabaseService } from '../services/database.service';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { RestServer } from '../servers/rest.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class DomainsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     private databaseService;
     constructor(restServer: RestServer, databaseService: DatabaseService, path: string);

package/dist/apis/{domains.api.js → domains.api.mjs}

@@ -1,26 +1,18 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTreeStringSegmentArrayToTree, commonsTypeEncode } from 'tscommons-es-core';
-import { CommonsRestApi, commonsRestApiBadRequest, commonsRestApiError } from 'nodecommons-es-rest';
+import { commonsTreeStringSegmentArrayToTree, commonsTypeEncode } from 'tscommons-esm-core';
+import { CommonsRestApi, commonsRestApiBadRequest, commonsRestApiError } from 'nodecommons-esm-rest';
 export class DomainsApi extends CommonsRestApi {
+    databaseService;
     constructor(restServer, databaseService, path) {
         super(restServer);
         this.databaseService = databaseService;
-        super.getHandler(`${path}rest/domains`, (_req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const domains = yield this.databaseService.listDomains();
+        super.getHandler(`${path}rest/domains`, async (_req, _res) => {
+            const domains = await this.databaseService.listDomains();
             return commonsTypeEncode(domains);
-        }));
-        super.getHandler(`${path}rest/domains/:domain[idname]/tree`, (req, _res) => __awaiter(this, void 0, void 0, function* () {
+        });
+        super.getHandler(`${path}rest/domains/:domain[idname]/tree`, async (req, _res) => {
             if (!/^[-a-z0-9]+(\.[-a-z0-9]+)*$/i.test(req.strictParams.domain))
                 return commonsRestApiBadRequest('Invalid domain');
-            const urls = yield this.databaseService.listDone200DomainUrls(req.params.domain);
+            const urls = await this.databaseService.listDone200DomainUrls(req.params.domain);
             const splits = [];
             for (const url of urls) {
                 const regex = new RegExp(`^http(?:s?)://${req.params.domain}(?::[0-9]+)?(/.*)$`);
@@ -36,7 +28,7 @@ export class DomainsApi extends CommonsRestApi {
             }
             const tree = commonsTreeStringSegmentArrayToTree(splits);
             return commonsTypeEncode(tree);
-        }));
+        });
     }
 }
-//# sourceMappingURL=domains.api.js.map
+//# sourceMappingURL=domains.api.mjs.map

package/dist/apis/domains.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"domains.api.mjs","sourceRoot":"","sources":["../../src/apis/domains.api.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,mCAAmC,EAAE,iBAAiB,EAA0B,MAAM,oBAAoB,CAAC;AAKpH,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAMrG,MAAM,OAAO,UAAW,SAAQ,cAAiE;IAGtF;IAFV,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,cAAc,EACrB,KAAK,EAAE,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,OAAO,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEpE,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CACF,CAAC;QAEF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,mCAAmC,EAC1C,KAAK,EAAE,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAgB,CAAC;gBAAE,OAAO,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;YAE/H,MAAM,IAAI,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE3F,MAAM,MAAM,GAAe,EAAE,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,iBAAiB,GAAG,CAAC,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC;gBACzF,MAAM,KAAK,GAAyB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAEpD,IAAI,KAAK,KAAK,IAAI;oBAAE,OAAO,mBAAmB,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;gBAE7E,IAAI,OAAO,GAAW,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAAE,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEjE,MAAM,QAAQ,GAAa,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9C,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,kBAAkB;gBAElC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,IAAI,GAAiB,mCAAmC,CAAC,MAAM,CAAC,CAAC;YACvE,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/{images.api.d.ts → images.api.d.mts}

@@ -1,8 +1,8 @@
 import * as express from 'express';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { RestServer } from '../servers/rest.server';
-import { DatabaseService } from '../services/database.service';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { RestServer } from '../servers/rest.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class ImagesApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     private databaseService;
     constructor(restServer: RestServer, databaseService: DatabaseService, path: string);

package/dist/apis/images.api.mjs

@@ -0,0 +1,19 @@
+import { commonsTypeEncode } from 'tscommons-esm-core';
+import { toEComparator, ECOMPARATORS } from 'hydra-crawler-esm-assets';
+import { CommonsRestApi, commonsRestApiBadRequest } from 'nodecommons-esm-rest';
+export class ImagesApi extends CommonsRestApi {
+    databaseService;
+    constructor(restServer, databaseService, path) {
+        super(restServer);
+        this.databaseService = databaseService;
+        super.getHandler(`${path}rest/images/:comparator(${ECOMPARATORS.join('|')})/:size[int]`, async (req, _res) => {
+            const comparator = toEComparator(req.strictParams.comparator);
+            if (!comparator)
+                return commonsRestApiBadRequest('Invalid comparator');
+            const size = req.strictParams.size;
+            const matches = await this.databaseService.listImagesBySizeThreshold(size, comparator);
+            return commonsTypeEncode(matches);
+        });
+    }
+}
+//# sourceMappingURL=images.api.mjs.map

package/dist/apis/images.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"images.api.mjs","sourceRoot":"","sources":["../../src/apis/images.api.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,oBAAoB,CAAC;AAGjE,OAAO,EAAe,aAAa,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAGpF,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAMhF,MAAM,OAAO,SAAU,SAAQ,cAAiE;IAGrF;IAFV,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,2BAA2B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,EACtE,KAAK,EAAE,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,MAAM,UAAU,GAA0B,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,UAAoB,CAAC,CAAC;YAC/F,IAAI,CAAC,UAAU;gBAAE,OAAO,wBAAwB,CAAC,oBAAoB,CAAC,CAAC;YAEvE,MAAM,IAAI,GAAW,GAAG,CAAC,YAAY,CAAC,IAAc,CAAC;YAErD,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,yBAAyB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAEjG,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/{statistics.api.d.ts → statistics.api.d.mts}

@@ -1,8 +1,8 @@
 import * as express from 'express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { RestServer } from '../servers/rest.server';
-import { DatabaseService } from '../services/database.service';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { RestServer } from '../servers/rest.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class StatisticsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     private databaseService;
     private memcached;

package/dist/apis/statistics.api.mjs

@@ -0,0 +1,33 @@
+import Cache from 'memcached-promisify';
+import { commonsTypeHasPropertyEnum, commonsTypeHasPropertyNumber, commonsTypeIsTArray } from 'tscommons-esm-core';
+import { isEStatus } from 'hydra-crawler-esm-assets';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+function isTStatusTally(test) {
+    if (!commonsTypeHasPropertyEnum(test, 'status', isEStatus))
+        return false;
+    if (!commonsTypeHasPropertyNumber(test, 'tally'))
+        return false;
+    return true;
+}
+export class StatisticsApi extends CommonsRestApi {
+    databaseService;
+    memcached;
+    constructor(restServer, databaseService, path) {
+        super(restServer);
+        this.databaseService = databaseService;
+        this.memcached = new Cache({ keyPrefix: 'hydra', cacheHost: 'localhost:11211' });
+        super.getHandler(`${path}rest/statistics/status`, async (_req, _res) => {
+            const index = 'statistics/status';
+            const cached = await this.memcached.get(index);
+            if (cached && commonsTypeIsTArray(cached, isTStatusTally))
+                return cached;
+            const result = await this.databaseService.listStatusTallies();
+            const tallies = [];
+            for (const key of result.keys())
+                tallies.push({ status: key, tally: result.get(key) || 0 });
+            await this.memcached.set(index, tallies, 10);
+            return tallies;
+        });
+    }
+}
+//# sourceMappingURL=statistics.api.mjs.map

package/dist/apis/statistics.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.api.mjs","sourceRoot":"","sources":["../../src/apis/statistics.api.mts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,qBAAqB,CAAC;AAExC,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnH,OAAO,EAAW,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAWtD,SAAS,cAAc,CAAC,IAAa;IACpC,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IACzE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,cAAiE;IAKzF;IAJF,SAAS,CAAQ;IAEzB,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,IAAI,CAAC,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAEjF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,wBAAwB,EAC/B,KAAK,EAAE,IAAqB,EAAE,IAAsB,EAA2B,EAAE;YAChF,MAAM,KAAK,GAAW,mBAAmB,CAAC;YAE1C,MAAM,MAAM,GAAY,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxD,IAAI,MAAM,IAAI,mBAAmB,CAAe,MAAM,EAAE,cAAc,CAAC;gBAAE,OAAO,MAAM,CAAC;YAEvF,MAAM,MAAM,GAAyB,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,EAAE,CAAC;YAEpF,MAAM,OAAO,GAAmB,EAAE,CAAC;YACnC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE5F,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;YAE7C,OAAO,OAAO,CAAC;QAChB,CAAC,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/{test.api.d.ts → test.api.d.mts}

@@ -1,7 +1,7 @@
 import * as express from 'express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { RestServer } from '../servers/rest.server';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { RestServer } from '../servers/rest.server.mjs';
 export declare class TestApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     constructor(restServer: RestServer, path: string);
 }

package/dist/apis/test.api.mjs

@@ -0,0 +1,10 @@
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+export class TestApi extends CommonsRestApi {
+    constructor(restServer, path) {
+        super(restServer);
+        super.getHandler(`${path}test/hello`, 
+        // eslint-disable-next-line @typescript-eslint/require-await
+        async (_) => 'Hello world');
+    }
+}
+//# sourceMappingURL=test.api.mjs.map

package/dist/apis/test.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.api.mjs","sourceRoot":"","sources":["../../src/apis/test.api.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAKtD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAC7F,YACE,UAAsB,EACtB,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAElB,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,YAAY;QACnB,4DAA4D;QAC5D,KAAK,EAAE,CAAU,EAAmB,EAAE,CAAC,aAAa,CACrD,CAAC;IACH,CAAC;CACD"}

package/dist/apis/{urls.api.d.ts → urls.api.d.mts}

@@ -1,8 +1,8 @@
 import * as express from 'express';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
-import { RestServer } from '../servers/rest.server';
-import { DatabaseService } from '../services/database.service';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+import { ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
+import { RestServer } from '../servers/rest.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class UrlsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
     private databaseService;
     constructor(restServer: RestServer, databaseService: DatabaseService, path: string);

package/dist/apis/urls.api.mjs

@@ -0,0 +1,21 @@
+import { commonsTypeEncode } from 'tscommons-esm-core';
+import { EDirection } from 'hydra-crawler-esm-assets';
+import { CommonsRestApi } from 'nodecommons-esm-rest';
+export class UrlsApi extends CommonsRestApi {
+    databaseService;
+    constructor(restServer, databaseService, path) {
+        super(restServer);
+        this.databaseService = databaseService;
+        // Can't use strict and ICommonsRequestWithStrictParams as the url could be anything, and idname won't accept that
+        super.getHandler(`${path}rest/urls/:url/links`, async (req, _res) => {
+            const outbound = await this.databaseService.listOutboundLinks(req.params.url);
+            const inbound = await this.databaseService.listInboundLinks(req.params.url);
+            const urlLinks = {
+                [EDirection.INBOUND]: inbound,
+                [EDirection.OUTBOUND]: outbound
+            };
+            return commonsTypeEncode(urlLinks);
+        });
+    }
+}
+//# sourceMappingURL=urls.api.mjs.map

package/dist/apis/urls.api.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"urls.api.mjs","sourceRoot":"","sources":["../../src/apis/urls.api.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,oBAAoB,CAAC;AAGjE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAEtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAOtD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAGnF;IAFV,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,kHAAkH;QAElH,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,sBAAsB,EAC7B,KAAK,EAAE,GAAoB,EAAE,IAAsB,EAAqB,EAAE;YACzE,MAAM,QAAQ,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACxF,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAEtF,MAAM,QAAQ,GAAc;gBAC1B,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO;gBAC7B,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,QAAQ;aAChC,CAAC;YAEF,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apps/{cleanup.app.d.ts → cleanup.app.d.mts}

@@ -1,8 +1,8 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app.mjs';
 export declare class CleanupApp extends CommonsApp implements IInternalHydraCommonListApp {
     private includeReset;
     private includeLinks;

package/dist/apps/cleanup.app.mjs

@@ -0,0 +1,129 @@
+import { commonsTypeHasPropertyNumber } from 'tscommons-esm-core';
+import { isIUrl } from 'hydra-crawler-esm-assets';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Lists } from '../classes/lists.mjs';
+import { Cleaner } from '../classes/cleaner.mjs';
+export class CleanupApp extends CommonsApp {
+    includeReset;
+    includeLinks;
+    databaseService;
+    lists;
+    constructor(includeReset = false, includeLinks = false) {
+        super('hydra-crawler');
+        this.includeReset = includeReset;
+        this.includeLinks = includeLinks;
+        this.lists = new Lists();
+    }
+    getAppName() {
+        return 'Hydra - Cleanup';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('cleanup', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async reset() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const log = commonsLogDoing('cleanup', 'Reseting urls for DENY, FAILED, DEAD and DISALLOWED');
+        let tally = 0;
+        while (true) {
+            const result = this.databaseService.getUrls()
+                .find({ status: { $in: [
+                        EStatus.DENY,
+                        EStatus.FAILED,
+                        EStatus.DEAD,
+                        EStatus.DISALLOWED
+                    ] } })
+                .sort({ _id: 1 })
+                .skip(tally);
+            try {
+                while (true) {
+                    tally++;
+                    if ((tally % 100) === 0)
+                        log.progress(tally);
+                    const row = await result.next();
+                    if (row === null)
+                        break;
+                    await this.databaseService.getLinks().deleteMany({ url: row.url });
+                }
+                break;
+            }
+            catch (err) {
+                if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                    throw err;
+            }
+        }
+        log.result(tally);
+    }
+    async links() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const orphans = [];
+        { // scope
+            const log = commonsLogDoing('cleanup', 'Searching for link orphans');
+            let tally = 0;
+            const result = this.databaseService.getLinks().find({}, {});
+            let batch = [];
+            while (true) {
+                tally++;
+                if ((tally % 1000) === 0)
+                    log.progress(`${tally}, ${orphans.length}`);
+                const link = await result.next();
+                if (link === null)
+                    break;
+                if (!batch.includes(link.url))
+                    batch.push(link.url);
+                if (!batch.includes(link.outgoing))
+                    batch.push(link.outgoing);
+                if (batch.length < 10000)
+                    continue;
+                const result2 = this.databaseService.getUrls().find({ url: { $in: batch } });
+                const matches = (await this.databaseService.listQueryResults(result2, isIUrl))
+                    .map((u) => u.url);
+                for (const l of batch) {
+                    if (!matches.includes(l))
+                        orphans.push(l);
+                }
+                batch = [];
+            }
+            log.result(orphans.length);
+        }
+        { // scope
+            const log = commonsLogDoing('cleanup', 'Removing orphan links');
+            let tally = 0;
+            for (const orphan of orphans) {
+                tally++;
+                if ((tally % 100) === 0)
+                    log.progress(`${tally}`);
+                await this.databaseService.getLinks().deleteMany({ url: orphan });
+                await this.databaseService.getLinks().deleteMany({ outgoing: orphan });
+            }
+            log.success();
+        }
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        if (this.includeReset)
+            await this.reset();
+        if (this.includeLinks)
+            await this.links();
+        const hardLimit = this.args.getFirstNumberOrUndefined('hard-limit');
+        const cleaner = new Cleaner(this.lists, this.databaseService);
+        await cleaner.purgeOrphanUrls(hardLimit);
+        await cleaner.purgeEmptyDomains();
+    }
+}
+//# sourceMappingURL=cleanup.app.mjs.map

package/dist/apps/cleanup.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanup.app.mjs","sourceRoot":"","sources":["../../src/apps/cleanup.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAElE,OAAO,EAAQ,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAExD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAUjD,MAAM,OAAO,UAAW,SAAQ,UAAU;IAM/B;IACA;IANF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK;QAEtC,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAItC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,KAAK;QAClB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qDAAqD,CAAC,CAAC;QAElH,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;wBACrB,OAAO,CAAC,IAAI;wBACZ,OAAO,CAAC,MAAM;wBACd,OAAO,CAAC,IAAI;wBACZ,OAAO,CAAC,UAAU;qBACnB,EAAE,EAAE,CAAC;iBACL,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;YAEf,IAAI,CAAC;gBACJ,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAE7C,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC3C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAED,MAAM;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;QACF,CAAC;QAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,KAAK;QAClB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YACzF,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,MAAM,GAA8B,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAE9F,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAEtE,MAAM,IAAI,GAAe,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAEzB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;oBAAE,SAAS;gBAEnC,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3E,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CACxB,CAAC;gBAEF,MAAM,OAAO,GAAa,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;qBACrF,GAAG,CAAC,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEnC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBAED,KAAK,GAAG,EAAE,CAAC;YACZ,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;YACpF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;gBAClD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YACxE,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAE1C,MAAM,SAAS,GAAqB,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,YAAY,CAAC,CAAC;QAEtF,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;IACnC,CAAC;CACD"}

package/dist/apps/{cross-populate-export.app.d.ts → cross-populate-export.app.d.mts}

@@ -1,6 +1,6 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app.mjs';
 export declare class CrossPopulateExportApp extends CommonsApp implements IInternalHydraCommonDbApp {
     private filename;
     private databaseService;

package/dist/apps/cross-populate-export.app.mjs

@@ -0,0 +1,61 @@
+import * as fs from 'fs';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+// export the QUEUED, DONE and FAILED urls for importing into another instance
+export class CrossPopulateExportApp extends CommonsApp {
+    filename;
+    databaseService;
+    constructor(filename) {
+        super('hydra-crawler');
+        this.filename = filename;
+    }
+    getAppName() {
+        return 'Hydra - Cross Populate Export';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('cross-populate-export', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const results = this.databaseService.getUrls().find({
+            status: { $in: [
+                    EStatus.QUEUED,
+                    EStatus.DONE,
+                    EStatus.FAILED
+                ] }
+        }, {});
+        // safer to do this directly rather than a call to listQueryResults
+        // ditto not using CommonsFile
+        const fd = fs.openSync(this.filename, 'w');
+        const log = commonsLogDoing('cross-populate-export', 'Exporting QUEUED, DONE and FAILED urls');
+        let tally = 0;
+        while (true) {
+            if (tally % 1000 === 0)
+                log.progress(tally);
+            const row = await results.next();
+            if (row === null)
+                break;
+            fs.writeSync(fd, JSON.stringify({
+                domain: row.domain,
+                url: row.url
+            }) + '\n');
+            if (tally % 1000 === 0)
+                fs.fdatasyncSync(fd);
+            tally++;
+        }
+        fs.fdatasyncSync(fd);
+        fs.closeSync(fd);
+        log.result(tally);
+    }
+}
+//# sourceMappingURL=cross-populate-export.app.mjs.map

package/dist/apps/cross-populate-export.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cross-populate-export.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAKzB,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAO;YAClF,MAAM,EAAE,EAAE,GAAG,EAAE;oBACb,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;iBACf,EAAE;SACJ,EAAE,EAAE,CAAC,CAAC;QAEP,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,EAAE,GAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wCAAwC,CAAC,CAAC;QAEnH,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;gBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAE5C,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,GAAG,KAAK,IAAI;gBAAE,MAAM;YAExB,EAAE,CAAC,SAAS,CACV,EAAE,EACF,IAAI,CAAC,SAAS,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;aACb,CAAC,GAAG,IAAI,CACV,CAAC;YAEF,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;gBAAE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAC7C,KAAK,EAAE,CAAC;QACT,CAAC;QAED,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACrB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEjB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;CACD"}

package/dist/apps/{cross-populate-import.app.d.ts → cross-populate-import.app.d.mts}

@@ -1,6 +1,6 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app.mjs';
 export declare class CrossPopulateImportApp extends CommonsApp implements IInternalHydraCommonDbApp {
     private filename;
     private databaseService;