hydra-crawler 2.2.5 → 2.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/autocomplete.api.d.ts +3 -1
- package/dist/apis/autocomplete.api.js.map +1 -1
- package/dist/apis/bugs.api.d.ts +3 -1
- package/dist/apis/bugs.api.js.map +1 -1
- package/dist/apis/crawl.api.d.ts +3 -1
- package/dist/apis/crawl.api.js.map +1 -1
- package/dist/apis/domains.api.d.ts +3 -1
- package/dist/apis/domains.api.js.map +1 -1
- package/dist/apis/images.api.d.ts +3 -1
- package/dist/apis/images.api.js.map +1 -1
- package/dist/apis/statistics.api.d.ts +3 -1
- package/dist/apis/statistics.api.js.map +1 -1
- package/dist/apis/test.api.d.ts +3 -1
- package/dist/apis/test.api.js.map +1 -1
- package/dist/apis/urls.api.d.ts +3 -1
- package/dist/apis/urls.api.js.map +1 -1
- package/dist/apps/cleanup.app.js +2 -1
- package/dist/apps/cleanup.app.js.map +1 -1
- package/dist/apps/cross-populate-export.app.js.map +1 -1
- package/dist/apps/cross-populate-import.app.js +21 -14
- package/dist/apps/cross-populate-import.app.js.map +1 -1
- package/dist/apps/denylist.app.js.map +1 -1
- package/dist/apps/hydra.app.d.ts +1 -0
- package/dist/apps/hydra.app.js +13 -2
- package/dist/apps/hydra.app.js.map +1 -1
- package/dist/apps/reattempt.app.js.map +1 -1
- package/dist/apps/startup.app.js.map +1 -1
- package/dist/apps/unarchive-urls.app.js.map +1 -1
- package/dist/classes/cleaner.d.ts +1 -1
- package/dist/classes/cleaner.js +24 -11
- package/dist/classes/cleaner.js.map +1 -1
- package/dist/classes/crawler.js +12 -12
- package/dist/classes/crawler.js.map +1 -1
- package/dist/classes/expirer.js +4 -1
- package/dist/classes/expirer.js.map +1 -1
- package/dist/classes/robot.js +7 -7
- package/dist/classes/robot.js.map +1 -1
- package/dist/parsers/complex-english.parser.js +12 -12
- package/dist/parsers/complex-english.parser.js.map +1 -1
- package/dist/parsers/html.parser.js +7 -7
- package/dist/parsers/html.parser.js.map +1 -1
- package/dist/parsers/hyperlinks.parser.js +5 -5
- package/dist/parsers/hyperlinks.parser.js.map +1 -1
- package/dist/parsers/image-tags.parser.d.ts +1 -1
- package/dist/parsers/image-tags.parser.js +4 -4
- package/dist/parsers/image-tags.parser.js.map +1 -1
- package/dist/queries/complex-english.query.js.map +1 -1
- package/dist/queries/flash-content.query.js.map +1 -1
- package/dist/queries/linking-to-domains.query.js.map +1 -1
- package/dist/servers/maintenance.server.d.ts +2 -1
- package/dist/servers/maintenance.server.js +3 -2
- package/dist/servers/maintenance.server.js.map +1 -1
- package/dist/servers/rest.server.d.ts +3 -2
- package/dist/servers/rest.server.js +2 -2
- package/dist/servers/rest.server.js.map +1 -1
- package/dist/servers/socket-io.server.d.ts +2 -2
- package/dist/servers/socket-io.server.js +2 -2
- package/dist/servers/socket-io.server.js.map +1 -1
- package/dist/services/database.service.d.ts +8 -8
- package/dist/services/database.service.js +103 -34
- package/dist/services/database.service.js.map +1 -1
- package/dist/types/tcrawl-config.d.ts +1 -1
- package/dist/types/thydra-config.d.ts +1 -1
- package/dist/types/tparser-ctor.d.ts +1 -1
- package/dist/types/tquery.d.ts +1 -1
- package/dist/types/trobots-config.d.ts +1 -1
- package/package.json +32 -32
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
2
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
1
3
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
5
|
import { DatabaseService } from '../services/database.service';
|
|
4
|
-
export declare class AutocompleteApi extends CommonsRestApi {
|
|
6
|
+
export declare class AutocompleteApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
5
7
|
private databaseService;
|
|
6
8
|
constructor(restServer: RestServer, databaseService: DatabaseService, path: string);
|
|
7
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autocomplete.api.js","sourceRoot":"","sources":["../../src/apis/autocomplete.api.ts"],"names":[],"mappings":";;;;;;;;;AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAMrD,MAAM,OAAO,eAAgB,SAAQ,
|
|
1
|
+
{"version":3,"file":"autocomplete.api.js","sourceRoot":"","sources":["../../src/apis/autocomplete.api.ts"],"names":[],"mappings":";;;;;;;;;AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAMrD,MAAM,OAAO,eAAgB,SAAQ,cAAiE;IACrG,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,wCAAwC,EAC/C,CAAO,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,MAAM,OAAO,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAc,CAAC,CAAC;YAEzG,OAAO,OAAO;iBACX,GAAG,CAAC,CAAC,MAAe,EAAU,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}
|
package/dist/apis/bugs.api.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
1
2
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
3
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
5
|
import { DatabaseService } from '../services/database.service';
|
|
4
|
-
export declare class BugsApi extends CommonsRestApi {
|
|
6
|
+
export declare class BugsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
5
7
|
private databaseService;
|
|
6
8
|
constructor(restServer: RestServer, databaseService: DatabaseService, path: string);
|
|
7
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bugs.api.js","sourceRoot":"","sources":["../../src/apis/bugs.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"bugs.api.js","sourceRoot":"","sources":["../../src/apis/bugs.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOrD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAC7F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,eAAe,EACtB,CAAO,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,MAAM,GAA4B,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;YAEnF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAA,CACF,CAAC;QAEF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,eAAe,EACtB,CAAO,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,MAAM,GAA4B,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;YAEnF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}
|
package/dist/apis/crawl.api.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
1
2
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
3
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
5
|
import { DatabaseService } from '../services/database.service';
|
|
4
|
-
export declare class CrawlApi extends CommonsRestApi {
|
|
6
|
+
export declare class CrawlApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
5
7
|
private databaseService;
|
|
6
8
|
constructor(restServer: RestServer, databaseService: DatabaseService, path: string);
|
|
7
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl.api.js","sourceRoot":"","sources":["../../src/apis/crawl.api.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"crawl.api.js","sourceRoot":"","sources":["../../src/apis/crawl.api.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOrD,MAAM,OAAO,QAAS,SAAQ,cAAiE;IAC9F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,kBAAkB,EACzB,CAAO,IAAqB,EAAE,IAAsB,EAA2B,EAAE;YAChF,MAAM,OAAO,GAAwB,MAAM,IAAI,CAAC,eAAe,CAAC,uBAAuB,EAAE,CAAC;YAE1F,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;iBAC9B,GAAG,CAAC,CAAC,MAAc,EAAgB,EAAE,CAAC,CAAC;gBACtC,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;aAChC,CAAC,CAAC,CAAC;QACP,CAAC,CAAA,CACF,CAAC;IAEH,CAAC;CACD"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
2
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
1
3
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
5
|
import { DatabaseService } from '../services/database.service';
|
|
4
|
-
export declare class DomainsApi extends CommonsRestApi {
|
|
6
|
+
export declare class DomainsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
5
7
|
private databaseService;
|
|
6
8
|
constructor(restServer: RestServer, databaseService: DatabaseService, path: string);
|
|
7
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"domains.api.js","sourceRoot":"","sources":["../../src/apis/domains.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,mCAAmC,EAAE,iBAAiB,EAA0B,MAAM,mBAAmB,CAAC;AAKnH,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAMpG,MAAM,OAAO,UAAW,SAAQ,
|
|
1
|
+
{"version":3,"file":"domains.api.js","sourceRoot":"","sources":["../../src/apis/domains.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,mCAAmC,EAAE,iBAAiB,EAA0B,MAAM,mBAAmB,CAAC;AAKnH,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAMpG,MAAM,OAAO,UAAW,SAAQ,cAAiE;IAChG,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,cAAc,EACrB,CAAO,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,OAAO,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEpE,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CAAA,CACF,CAAC;QAEF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,mCAAmC,EAC1C,CAAO,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAgB,CAAC;gBAAE,OAAO,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;YAE/H,MAAM,IAAI,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE3F,MAAM,MAAM,GAAe,EAAE,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,iBAAiB,GAAG,CAAC,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC;gBACzF,MAAM,KAAK,GAAyB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAEpD,IAAI,KAAK,KAAK,IAAI;oBAAE,OAAO,mBAAmB,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;gBAE7E,IAAI,OAAO,GAAW,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAAE,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEjE,MAAM,QAAQ,GAAa,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9C,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,kBAAkB;gBAElC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;aACtB;YAED,MAAM,IAAI,GAAiB,mCAAmC,CAAC,MAAM,CAAC,CAAC;YACvE,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
2
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
1
3
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
5
|
import { DatabaseService } from '../services/database.service';
|
|
4
|
-
export declare class ImagesApi extends CommonsRestApi {
|
|
6
|
+
export declare class ImagesApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
5
7
|
private databaseService;
|
|
6
8
|
constructor(restServer: RestServer, databaseService: DatabaseService, path: string);
|
|
7
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"images.api.js","sourceRoot":"","sources":["../../src/apis/images.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAe,aAAa,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGnF,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAM/E,MAAM,OAAO,SAAU,SAAQ,
|
|
1
|
+
{"version":3,"file":"images.api.js","sourceRoot":"","sources":["../../src/apis/images.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAe,aAAa,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGnF,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAM/E,MAAM,OAAO,SAAU,SAAQ,cAAiE;IAC/F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,2BAA2B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,EACtE,CAAO,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,MAAM,UAAU,GAA0B,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,UAAoB,CAAC,CAAC;YAC/F,IAAI,CAAC,UAAU;gBAAE,OAAO,wBAAwB,CAAC,oBAAoB,CAAC,CAAC;YAEvE,MAAM,IAAI,GAAW,GAAG,CAAC,YAAY,CAAC,IAAc,CAAC;YAErD,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,yBAAyB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAEjG,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
1
2
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
3
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
5
|
import { DatabaseService } from '../services/database.service';
|
|
4
|
-
export declare class StatisticsApi extends CommonsRestApi {
|
|
6
|
+
export declare class StatisticsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
5
7
|
private databaseService;
|
|
6
8
|
private memcached;
|
|
7
9
|
constructor(restServer: RestServer, databaseService: DatabaseService, path: string);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.api.js","sourceRoot":"","sources":["../../src/apis/statistics.api.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,KAAK,MAAM,qBAAqB,CAAC;AAExC,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAElH,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"statistics.api.js","sourceRoot":"","sources":["../../src/apis/statistics.api.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,KAAK,MAAM,qBAAqB,CAAC;AAExC,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAElH,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAarD,SAAS,cAAc,CAAC,IAAa;IACpC,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IACzE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,cAAiE;IAGnG,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,sGAAsG;QACtG,IAAI,CAAC,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAEjF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,wBAAwB,EAC/B,CAAO,IAAqB,EAAE,IAAsB,EAA2B,EAAE;YAChF,MAAM,KAAK,GAAW,mBAAmB,CAAC;YAE1C,MAAM,MAAM,GAAsB,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClE,IAAI,MAAM,IAAI,mBAAmB,CAAe,MAAM,EAAE,cAAc,CAAC;gBAAE,OAAO,MAAM,CAAC;YAEvF,MAAM,MAAM,GAAyB,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,EAAE,CAAC;YAEpF,MAAM,OAAO,GAAmB,EAAE,CAAC;YACnC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE5F,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;YAE7C,OAAO,OAAO,CAAC;QAChB,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}
|
package/dist/apis/test.api.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
1
2
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
3
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
|
-
export declare class TestApi extends CommonsRestApi {
|
|
5
|
+
export declare class TestApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
4
6
|
constructor(restServer: RestServer, path: string);
|
|
5
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test.api.js","sourceRoot":"","sources":["../../src/apis/test.api.ts"],"names":[],"mappings":";;;;;;;;;
|
|
1
|
+
{"version":3,"file":"test.api.js","sourceRoot":"","sources":["../../src/apis/test.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAKrD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAC7F,YACE,UAAsB,EACtB,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAElB,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,YAAY;QACnB,4DAA4D;QAC5D,CAAO,CAAU,EAAmB,EAAE,gDAAC,OAAA,aAAa,CAAA,GAAA,CACrD,CAAC;IACH,CAAC;CACD"}
|
package/dist/apis/urls.api.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
1
2
|
import { CommonsRestApi } from 'nodecommons-es-rest';
|
|
3
|
+
import { ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
2
4
|
import { RestServer } from '../servers/rest.server';
|
|
3
5
|
import { DatabaseService } from '../services/database.service';
|
|
4
|
-
export declare class UrlsApi extends CommonsRestApi {
|
|
6
|
+
export declare class UrlsApi extends CommonsRestApi<ICommonsRequestWithStrictParams, express.Response> {
|
|
5
7
|
private databaseService;
|
|
6
8
|
constructor(restServer: RestServer, databaseService: DatabaseService, path: string);
|
|
7
9
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"urls.api.js","sourceRoot":"","sources":["../../src/apis/urls.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"urls.api.js","sourceRoot":"","sources":["../../src/apis/urls.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOrD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAC7F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,kHAAkH;QAElH,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,sBAAsB,EAC7B,CAAO,GAAoB,EAAE,IAAsB,EAAqB,EAAE;YACzE,MAAM,QAAQ,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACxF,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAEtF,MAAM,QAAQ,GAAc;gBAC1B,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO;gBAC7B,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,QAAQ;aAChC,CAAC;YAEF,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}
|
package/dist/apps/cleanup.app.js
CHANGED
|
@@ -131,8 +131,9 @@ export class CleanupApp extends CommonsApp {
|
|
|
131
131
|
yield this.reset();
|
|
132
132
|
if (this.includeLinks)
|
|
133
133
|
yield this.links();
|
|
134
|
+
const hardLimit = this.getArgs().getNumberOrUndefined('hard-limit');
|
|
134
135
|
const cleaner = new Cleaner(this.lists, this.databaseService);
|
|
135
|
-
yield cleaner.purgeOrphanUrls();
|
|
136
|
+
yield cleaner.purgeOrphanUrls(hardLimit);
|
|
136
137
|
yield cleaner.purgeEmptyDomains();
|
|
137
138
|
});
|
|
138
139
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup.app.js","sourceRoot":"","sources":["../../src/apps/cleanup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAU7C,MAAM,OAAO,UAAW,SAAQ,UAAU;IAKzC,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK;QAEtC,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAItC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,qDAAqD,CAAC,CAAC;YAE1E,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"cleanup.app.js","sourceRoot":"","sources":["../../src/apps/cleanup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAU7C,MAAM,OAAO,UAAW,SAAQ,UAAU;IAKzC,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK;QAEtC,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAItC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,qDAAqD,CAAC,CAAC;YAE1E,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACpE,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACrB,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,MAAM;4BACd,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,UAAU;yBACnB,EAAE,EAAE,CAAC;qBACL,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC3C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;qBACnE;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YACjD,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,MAAM,GAA8B,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAE9F,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE/E,MAAM,IAAI,GAAe,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAEzB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;oBAAE,SAAS;gBAEnC,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3E,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CACxB,CAAC;gBAEF,MAAM,OAAO,GAAa,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;qBACrF,GAAG,CAAC,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEnC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;oBACtB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAC1C;gBAED,KAAK,GAAG,EAAE,CAAC;aACX;YACD,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEpC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;YAC5C,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;aACvE;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1C,IAAI,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAE1C,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAEtF,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACvE,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACnC,CAAC;KAAA;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cross-populate-export.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAKzB,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"cross-populate-export.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAKzB,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAO;gBAClF,MAAM,EAAE,EAAE,GAAG,EAAE;wBACb,OAAO,CAAC,MAAM;wBACd,OAAO,CAAC,IAAI;wBACZ,OAAO,CAAC,MAAM;qBACf,EAAE;aACJ,EAAE,EAAE,CAAC,CAAC;YAEP,mEAAmE;YACnE,8BAA8B;YAE9B,MAAM,EAAE,GAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAEnD,kBAAkB,CAAC,wCAAwC,CAAC,CAAC;YAE7D,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAErD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,EAAE,CAAC,SAAS,CACV,EAAE,EACF,IAAI,CAAC,SAAS,CAAC;oBACb,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;iBACb,CAAC,GAAG,IAAI,CACV,CAAC;gBAEF,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;oBAAE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBAC7C,KAAK,EAAE,CAAC;aACR;YAED,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACrB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAEjB,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}
|
|
@@ -51,7 +51,7 @@ export class CrossPopulateImportApp extends CommonsApp {
|
|
|
51
51
|
});
|
|
52
52
|
}
|
|
53
53
|
run() {
|
|
54
|
-
var e_1,
|
|
54
|
+
var _a, e_1, _b, _c;
|
|
55
55
|
return __awaiter(this, void 0, void 0, function* () {
|
|
56
56
|
if (!this.databaseService)
|
|
57
57
|
throw new Error('Database service has not been set');
|
|
@@ -63,25 +63,32 @@ export class CrossPopulateImportApp extends CommonsApp {
|
|
|
63
63
|
commonsOutputDoing('Importing urls');
|
|
64
64
|
let tally = 0;
|
|
65
65
|
try {
|
|
66
|
-
for (var rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(),
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
66
|
+
for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a;) {
|
|
67
|
+
_c = rl_1_1.value;
|
|
68
|
+
_d = false;
|
|
69
|
+
try {
|
|
70
|
+
const line = _c;
|
|
71
|
+
if (line.trim() === '')
|
|
72
|
+
continue;
|
|
73
|
+
if (tally % 1000 === 0)
|
|
74
|
+
commonsOutputProgress(tally);
|
|
75
|
+
const json = JSON.parse(line);
|
|
76
|
+
if (!isTRow(json)) {
|
|
77
|
+
commonsOutputError('Invalid row. Skipping');
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
yield this.databaseService.queue(json.url, false);
|
|
81
|
+
tally++;
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
_d = true;
|
|
76
85
|
}
|
|
77
|
-
yield this.databaseService.queue(json.url, false);
|
|
78
|
-
tally++;
|
|
79
86
|
}
|
|
80
87
|
}
|
|
81
88
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
82
89
|
finally {
|
|
83
90
|
try {
|
|
84
|
-
if (
|
|
91
|
+
if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
|
|
85
92
|
}
|
|
86
93
|
finally { if (e_1) throw e_1.error; }
|
|
87
94
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cross-populate-import.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAahD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;gBACtD,KAAK,EAAE,UAAU;gBACjB,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;YAEH,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;YAErC,IAAI,KAAK,GAAW,CAAC,CAAC;;gBACtB,KAAyB,
|
|
1
|
+
{"version":3,"file":"cross-populate-import.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAahD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;gBACtD,KAAK,EAAE,UAAU;gBACjB,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;YAEH,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;YAErC,IAAI,KAAK,GAAW,CAAC,CAAC;;gBACtB,KAAyB,eAAA,OAAA,cAAA,EAAE,CAAA,QAAA;oBAAF,kBAAE;oBAAF,WAAE;;wBAAhB,MAAM,IAAI,KAAA,CAAA;wBACpB,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;4BAAE,SAAS;wBAEjC,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAErD,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;4BAClB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;4BAC5C,SAAS;yBACT;wBAED,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBAClD,KAAK,EAAE,CAAC;;;;;iBACR;;;;;;;;;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,UAAU,CAAC,KAAK,EAAE,CAAC;;KACnB;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"denylist.app.js","sourceRoot":"","sources":["../../src/apps/denylist.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,WAAY,SAAQ,UAAU;IAK1C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,OAAO,CAAC,KAAc;;YACnC,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6CAA6C,CAAC,CAAC;YAElE,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"denylist.app.js","sourceRoot":"","sources":["../../src/apps/denylist.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,WAAY,SAAQ,UAAU;IAK1C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,OAAO,CAAC,KAAc;;YACnC,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6CAA6C,CAAC,CAAC;YAElE,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAE,EAAE,EAAE,EACxD,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAErD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACnB;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YAE/C,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,IAAI;qBACrB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEa,OAAO,CAAC,KAAc;;YACnC,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6CAA6C,CAAC,CAAC;YAElE,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,EACxB,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACnB;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,GAAG;;YACf,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;YAE5C,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACxD,CAAC;KAAA;CACD"}
|
package/dist/apps/hydra.app.d.ts
CHANGED
|
@@ -21,6 +21,7 @@ export declare class HydraApp extends CommonsSocketIoApp<SocketIoServer> impleme
|
|
|
21
21
|
private maintenanceServer;
|
|
22
22
|
private tracker;
|
|
23
23
|
constructor();
|
|
24
|
+
protected buildHttpServer(): CommonsStrictExpressServer;
|
|
24
25
|
protected buildSocketIoServer(expressServer: CommonsStrictExpressServer, expressConfig: ICommonsExpressConfig): SocketIoServer;
|
|
25
26
|
getAppName(): string;
|
|
26
27
|
setDatabaseService(databaseService: DatabaseService): void;
|
package/dist/apps/hydra.app.js
CHANGED
|
@@ -9,6 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
9
9
|
};
|
|
10
10
|
import { commonsTypeIsTKeyObject } from 'tscommons-es-core';
|
|
11
11
|
import { commonsOutputDebug, commonsOutputDoing, commonsOutputInfo, commonsOutputSuccess } from 'nodecommons-es-cli';
|
|
12
|
+
import { commonsExpressBuildDefaultServer, isICommonsExpressConfig } from 'nodecommons-es-express';
|
|
12
13
|
import { CommonsSocketIoApp } from 'nodecommons-es-app-socket-io';
|
|
13
14
|
import { AutocompleteApi } from '../apis/autocomplete.api';
|
|
14
15
|
import { BugsApi } from '../apis/bugs.api';
|
|
@@ -62,8 +63,14 @@ export class HydraApp extends CommonsSocketIoApp {
|
|
|
62
63
|
throw new Error('Robots config is not valid');
|
|
63
64
|
this.robotsConfig = robotsConfig;
|
|
64
65
|
}
|
|
66
|
+
buildHttpServer() {
|
|
67
|
+
const expressConfig = this.getConfigArea('express');
|
|
68
|
+
if (!isICommonsExpressConfig(expressConfig))
|
|
69
|
+
throw new Error('Invalid express config');
|
|
70
|
+
return commonsExpressBuildDefaultServer(this.httpConfig.port, expressConfig.bodyParserLimit);
|
|
71
|
+
}
|
|
65
72
|
buildSocketIoServer(expressServer, expressConfig) {
|
|
66
|
-
return new SocketIoServer(expressServer,
|
|
73
|
+
return new SocketIoServer(expressServer, expressConfig);
|
|
67
74
|
}
|
|
68
75
|
getAppName() {
|
|
69
76
|
return 'Hydra';
|
|
@@ -140,7 +147,11 @@ export class HydraApp extends CommonsSocketIoApp {
|
|
|
140
147
|
this.crawlServer.addParser(parser);
|
|
141
148
|
yield this.databaseService.initParser(parser);
|
|
142
149
|
}
|
|
143
|
-
|
|
150
|
+
const maintenanceHardLimit = this.getArgs().getNumberOrUndefined('maintenance-hard-limit');
|
|
151
|
+
if (maintenanceHardLimit)
|
|
152
|
+
commonsOutputDebug(`Setting maintenance hard limit to ${maintenanceHardLimit}`);
|
|
153
|
+
this.maintenanceServer = new MaintenanceServer(this.maintenanceTimes, this.expiry, this.lists, this.databaseService, this.crawlServer, maintenanceHardLimit // to prevent running out of heap on very large collections
|
|
154
|
+
);
|
|
144
155
|
});
|
|
145
156
|
}
|
|
146
157
|
listening() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;
|
|
1
|
+
{"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAGxE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACrH,OAAO,EAAE,gCAAgC,EAA0D,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAG3J,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yCAAyC,CAAC;AAErF,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI9E,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAUzE,MAAM,OAAO,QAAS,SAAQ,kBAAkC;IAkB/D;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QANhB,qBAAgB,GAA2B,EAAE,CAAC;QAQrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QAEnC,MAAM,YAAY,GAAY,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IAEkB,eAAe;QACjC,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAEvF,OAAO,gCAAgC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAC9F,CAAC;IAES,mBAAmB,CAC3B,aAAyC,EACzC,aAAoC;QAErC,OAAO,IAAI,cAAc,CACvB,aAAa,EACb,aAAa,CACd,CAAC;IACH,CAAC;IAEM,UAAU;QAChB,OAAO,OAAO,CAAC;IAChB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEM,sBAAsB,CAC3B,KAA6B;QAE9B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,CAAC,OAAO,CACV,CAAC,UAAgF,EAAE,IAAY,EAAQ,EAAE;gBACxG,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAExF,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;gBACnD,IAAI,eAAe,CACjB,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;gBAC3C,IAAI,QAAQ,CACV,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;gBAC7C,IAAI,UAAU,CACZ,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;gBAC5C,IAAI,SAAS,CACX,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAChD,IAAI,aAAa,CACf,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACxB,CAAC,CACF,CAAC;YAEF,MAAM,OAAM,IAAI,WAAE,CAAC;YAEnB,MAAM,cAAc,GAA6B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAE7E,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,IAAI,CAAC,eAAe,EACpB,cAAc,CACf,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;YAEF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI;gBACnB,YAAY;gBACZ,gBAAgB;gBAChB,eAAe;gBACf,UAAU;gBACV,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,oBAAoB;gBACpB,0BAA0B;aAC3B,EAAE;gBACF,kBAAkB,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC9C;YAED,MAAM,oBAAoB,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,wBAAwB,CAAC,CAAC;YAC7G,IAAI,oBAAoB;gBAAE,kBAAkB,CAAC,qCAAqC,oBAAoB,EAAE,CAAC,CAAC;YAE1G,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,oBAAoB,CAAC,2DAA2D;aACjF,CAAC;QACH,CAAC;KAAA;IAES,SAAS;QAClB,KAAK,CAAC,GAAwB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAA,CAAC,EAAE,CAAC;IACN,CAAC;IAEY,GAAG;;;;;YACf,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,SAAS,EAAE;gBACd,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAE5E,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;gBAEvC,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,CAAC,QAAQ,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAC9B,IAAI,CAAC,KAAK,CACX,CAAC;gBAEF,iBAAiB,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAW,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;gBAE7D,OAAO;aACP;YAED,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAChG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;YAErH,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAE1B,MAAM,OAAM,GAAG,WAAE,CAAC;QACnB,CAAC;KAAA;IAEe,QAAQ;;;;;YACvB,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAElC,MAAM,OAAM,QAAQ,WAAE,CAAC;QACxB,CAAC;KAAA;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reattempt.app.js","sourceRoot":"","sources":["../../src/apps/reattempt.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,YAAa,SAAQ,UAAU;IAK3C,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9E,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"reattempt.app.js","sourceRoot":"","sources":["../../src/apps/reattempt.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,YAAa,SAAQ,UAAU;IAK3C,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9E,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;aAC9B,EACD,EAAE,CACH,CAAC;YAEJ,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,KAAK,GAAG,CAAC,CAAC;YAEV,OAAO,IAAI,EAAE;gBACZ,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAEtD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEtD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YACxC,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAErD,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACjC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;oBACvB,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;iBACnB,CAAC,CAAC;aACL;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"startup.app.js","sourceRoot":"","sources":["../../src/apps/startup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,sCAAsC;AAEtC,MAAM,OAAO,UAAW,SAAQ,UAAU;IAGzC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,GAAG,
|
|
1
|
+
{"version":3,"file":"startup.app.js","sourceRoot":"","sources":["../../src/apps/startup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,sCAAsC;AAEtC,MAAM,OAAO,UAAW,SAAQ,UAAU;IAGzC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,GAAG,GAAe,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,CAAC;YAE9D,KAAK,MAAM,UAAU,IAAI,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;gBACzD,kBAAkB,CAAC,WAAW,UAAU,8CAA8C,CAAC,CAAC;gBACxF,MAAM,KAAK,GAAW,MAAM,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACxE,mBAAmB,CAAC,KAAK,CAAC,CAAC;aAC3B;YAED,kBAAkB,CAAC,4DAA4D,CAAC,CAAC;YACjF,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC;gBAC3C,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,yDAAyD,CAAC,CAAC;YAC9E,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC;gBACxC,GAAG,EAAE,gBAAgB;aACtB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,0DAA0D,CAAC,CAAC;YAC/E,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC;gBACzC,QAAQ,EAAE,gBAAgB;aAC3B,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unarchive-urls.app.js","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IAK/C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;YAEhD,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"unarchive-urls.app.js","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IAK/C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;YAEhD,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YAErB,OAAO,IAAI,EAAE;gBACZ,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC1D,KAAK,EAAE,CAAC;gBAER,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEtD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YAC/C,KAAK,GAAG,CAAC,CAAC;YACV,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAC7B,KAAK,IAAI,GAAG,CAAC;gBAEb,MAAM,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;iBACjC,CACF,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aACpB;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}
|
|
@@ -7,6 +7,6 @@ export declare class Cleaner {
|
|
|
7
7
|
private detectStatusOrphans;
|
|
8
8
|
private detectNonAllowlistOrphans;
|
|
9
9
|
private detectStatusCodeOrphans;
|
|
10
|
-
purgeOrphanUrls(): Promise<void>;
|
|
10
|
+
purgeOrphanUrls(hardLimit?: number): Promise<void>;
|
|
11
11
|
purgeEmptyDomains(): Promise<void>;
|
|
12
12
|
}
|
package/dist/classes/cleaner.js
CHANGED
|
@@ -16,7 +16,7 @@ export class Cleaner {
|
|
|
16
16
|
this.lists = lists;
|
|
17
17
|
this.databaseService = databaseService;
|
|
18
18
|
}
|
|
19
|
-
detectStatusOrphans(statuses) {
|
|
19
|
+
detectStatusOrphans(statuses, hardLimit) {
|
|
20
20
|
return __awaiter(this, void 0, void 0, function* () {
|
|
21
21
|
if (!this.databaseService)
|
|
22
22
|
throw new Error('Database service has not been set yet');
|
|
@@ -41,10 +41,14 @@ export class Cleaner {
|
|
|
41
41
|
tally++;
|
|
42
42
|
if ((tally % 100) === 0)
|
|
43
43
|
commonsOutputProgress(`${tally} urls, ${found} orphans`);
|
|
44
|
+
if (hardLimit && tally >= hardLimit)
|
|
45
|
+
break;
|
|
44
46
|
const row = yield urls.next();
|
|
45
47
|
if (row === null)
|
|
46
48
|
break;
|
|
47
|
-
const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
|
|
49
|
+
const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
|
|
50
|
+
.limit(1)
|
|
51
|
+
.count();
|
|
48
52
|
if (incoming > 0)
|
|
49
53
|
continue;
|
|
50
54
|
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
|
@@ -61,7 +65,7 @@ export class Cleaner {
|
|
|
61
65
|
commonsOutputResult(`${tally} urls, ${found} orphans`);
|
|
62
66
|
});
|
|
63
67
|
}
|
|
64
|
-
detectNonAllowlistOrphans(statuses) {
|
|
68
|
+
detectNonAllowlistOrphans(statuses, hardLimit) {
|
|
65
69
|
return __awaiter(this, void 0, void 0, function* () {
|
|
66
70
|
let tally = 0;
|
|
67
71
|
let found = 0;
|
|
@@ -84,12 +88,16 @@ export class Cleaner {
|
|
|
84
88
|
tally++;
|
|
85
89
|
if ((tally % 100) === 0)
|
|
86
90
|
commonsOutputProgress(`${tally} urls, ${found} orphans`);
|
|
91
|
+
if (hardLimit && tally >= hardLimit)
|
|
92
|
+
break;
|
|
87
93
|
const row = yield urls.next();
|
|
88
94
|
if (row === null)
|
|
89
95
|
break;
|
|
90
96
|
if (this.lists.match(EList.ALLOW, row.url))
|
|
91
97
|
continue;
|
|
92
|
-
const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
|
|
98
|
+
const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
|
|
99
|
+
.limit(1)
|
|
100
|
+
.count();
|
|
93
101
|
if (incoming > 0)
|
|
94
102
|
continue;
|
|
95
103
|
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
|
@@ -106,7 +114,7 @@ export class Cleaner {
|
|
|
106
114
|
commonsOutputResult(`${tally} urls, ${found} orphans`);
|
|
107
115
|
});
|
|
108
116
|
}
|
|
109
|
-
detectStatusCodeOrphans(gt, lt) {
|
|
117
|
+
detectStatusCodeOrphans(gt, lt, hardLimit) {
|
|
110
118
|
return __awaiter(this, void 0, void 0, function* () {
|
|
111
119
|
let tally = 0;
|
|
112
120
|
let found = 0;
|
|
@@ -131,10 +139,14 @@ export class Cleaner {
|
|
|
131
139
|
tally++;
|
|
132
140
|
if ((tally % 100) === 0)
|
|
133
141
|
commonsOutputProgress(`${tally} urls, ${found} orphans`);
|
|
142
|
+
if (hardLimit && tally >= hardLimit)
|
|
143
|
+
break;
|
|
134
144
|
const row = yield urls.next();
|
|
135
145
|
if (row === null)
|
|
136
146
|
break;
|
|
137
|
-
const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
|
|
147
|
+
const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
|
|
148
|
+
.limit(1)
|
|
149
|
+
.count();
|
|
138
150
|
if (incoming > 0)
|
|
139
151
|
continue;
|
|
140
152
|
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
|
@@ -151,20 +163,20 @@ export class Cleaner {
|
|
|
151
163
|
commonsOutputResult(`${tally} urls, ${found} orphans`);
|
|
152
164
|
});
|
|
153
165
|
}
|
|
154
|
-
purgeOrphanUrls() {
|
|
166
|
+
purgeOrphanUrls(hardLimit) {
|
|
155
167
|
return __awaiter(this, void 0, void 0, function* () {
|
|
156
168
|
yield this.detectStatusOrphans([
|
|
157
169
|
EStatus.DENY,
|
|
158
170
|
EStatus.FAILED,
|
|
159
171
|
EStatus.DEAD,
|
|
160
172
|
EStatus.DISALLOWED
|
|
161
|
-
]);
|
|
173
|
+
], hardLimit);
|
|
162
174
|
yield this.detectNonAllowlistOrphans([
|
|
163
175
|
EStatus.QUEUED,
|
|
164
176
|
EStatus.DONE
|
|
165
|
-
]);
|
|
166
|
-
yield this.detectStatusCodeOrphans(300, 310);
|
|
167
|
-
yield this.detectStatusCodeOrphans(400, 500);
|
|
177
|
+
], hardLimit);
|
|
178
|
+
yield this.detectStatusCodeOrphans(300, 310, hardLimit);
|
|
179
|
+
yield this.detectStatusCodeOrphans(400, 500, hardLimit);
|
|
168
180
|
let tally = 0;
|
|
169
181
|
commonsOutputDoing('Removing orphan outgoing links');
|
|
170
182
|
while (true) {
|
|
@@ -244,6 +256,7 @@ export class Cleaner {
|
|
|
244
256
|
domain: domain,
|
|
245
257
|
status: { $ne: EStatus.ARCHIVED }
|
|
246
258
|
}, {})
|
|
259
|
+
.limit(1)
|
|
247
260
|
.count();
|
|
248
261
|
if (urls > 0)
|
|
249
262
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../src/classes/cleaner.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAGjE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAM1H,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,MAAM,OAAO,OAAO;IACnB,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEU,mBAAmB,
|
|
1
|
+
{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../src/classes/cleaner.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAGjE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAM1H,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,MAAM,OAAO,OAAO;IACnB,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEU,mBAAmB,CAC/B,QAAmB,EACnB,SAAkB;;YAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEnE,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACjE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CACtB;6BACE,KAAK,CAAC,CAAC,CAAC;6BACR,KAAK,EAAE,CAAC;wBACX,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,yBAAyB,CACrC,QAAmB,EACnB,SAAkB;;YAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;4BAAE,SAAS;wBAErD,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACjE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CACtB;6BACE,KAAK,CAAC,CAAC,CAAC;6BACR,KAAK,EAAE,CAAC;wBACX,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,uBAAuB,CACnC,EAAU,EACV,EAAU,EACV,SAAkB;;YAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,sCAAsC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;wBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACjE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CACtB;6BACE,KAAK,CAAC,CAAC,CAAC;6BACR,KAAK,EAAE,CAAC;wBACX,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YAED,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEY,eAAe,CAAC,SAAkB;;YAC9C,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAC7B,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;aACnB,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,IAAI,CAAC,yBAAyB,CAAC;gBACnC,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;aACb,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;YACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;YAExD,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;YACrD,OAAO,IAAI,EAAE;gBACZ,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;qBACnE;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;iBACZ;aACF,CACF,CAAC;YACF,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,iBAAiB;;YAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;YAE1C,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI;gBACH,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE;oBACZ,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;oBAEtD,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;iBACzB;aACD;YAAC,OAAO,GAAG,EAAE;gBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;aAC7E;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;YAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAEpF,MAAM,IAAI,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACtD,IAAI,CAAO;oBACV,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;iBAClC,EAAE,EAAE,CAAC;qBACL,KAAK,CAAC,CAAC,CAAC;qBACR,KAAK,EAAE,CAAC;gBAEX,IAAI,IAAI,GAAG,CAAC;oBAAE,SAAS;gBAEvB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAC3D,CAAC;KAAA;CAED"}
|