hydra-crawler 2.2.5 → 2.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/autocomplete.api.d.ts +3 -1
- package/dist/apis/autocomplete.api.js.map +1 -1
- package/dist/apis/bugs.api.d.ts +3 -1
- package/dist/apis/bugs.api.js.map +1 -1
- package/dist/apis/crawl.api.d.ts +3 -1
- package/dist/apis/crawl.api.js.map +1 -1
- package/dist/apis/domains.api.d.ts +3 -1
- package/dist/apis/domains.api.js.map +1 -1
- package/dist/apis/images.api.d.ts +3 -1
- package/dist/apis/images.api.js.map +1 -1
- package/dist/apis/statistics.api.d.ts +3 -1
- package/dist/apis/statistics.api.js.map +1 -1
- package/dist/apis/test.api.d.ts +3 -1
- package/dist/apis/test.api.js.map +1 -1
- package/dist/apis/urls.api.d.ts +3 -1
- package/dist/apis/urls.api.js.map +1 -1
- package/dist/apps/cleanup.app.js +2 -1
- package/dist/apps/cleanup.app.js.map +1 -1
- package/dist/apps/cross-populate-export.app.js.map +1 -1
- package/dist/apps/cross-populate-import.app.js +21 -14
- package/dist/apps/cross-populate-import.app.js.map +1 -1
- package/dist/apps/denylist.app.js.map +1 -1
- package/dist/apps/hydra.app.d.ts +1 -0
- package/dist/apps/hydra.app.js +13 -2
- package/dist/apps/hydra.app.js.map +1 -1
- package/dist/apps/reattempt.app.js.map +1 -1
- package/dist/apps/startup.app.js.map +1 -1
- package/dist/apps/unarchive-urls.app.js.map +1 -1
- package/dist/classes/cleaner.d.ts +1 -1
- package/dist/classes/cleaner.js +24 -11
- package/dist/classes/cleaner.js.map +1 -1
- package/dist/classes/crawler.js +12 -12
- package/dist/classes/crawler.js.map +1 -1
- package/dist/classes/expirer.js +4 -1
- package/dist/classes/expirer.js.map +1 -1
- package/dist/classes/robot.js +7 -7
- package/dist/classes/robot.js.map +1 -1
- package/dist/parsers/complex-english.parser.js +12 -12
- package/dist/parsers/complex-english.parser.js.map +1 -1
- package/dist/parsers/html.parser.js +7 -7
- package/dist/parsers/html.parser.js.map +1 -1
- package/dist/parsers/hyperlinks.parser.js +5 -5
- package/dist/parsers/hyperlinks.parser.js.map +1 -1
- package/dist/parsers/image-tags.parser.d.ts +1 -1
- package/dist/parsers/image-tags.parser.js +4 -4
- package/dist/parsers/image-tags.parser.js.map +1 -1
- package/dist/queries/complex-english.query.js.map +1 -1
- package/dist/queries/flash-content.query.js.map +1 -1
- package/dist/queries/linking-to-domains.query.js.map +1 -1
- package/dist/servers/maintenance.server.d.ts +2 -1
- package/dist/servers/maintenance.server.js +3 -2
- package/dist/servers/maintenance.server.js.map +1 -1
- package/dist/servers/rest.server.d.ts +3 -2
- package/dist/servers/rest.server.js +2 -2
- package/dist/servers/rest.server.js.map +1 -1
- package/dist/servers/socket-io.server.d.ts +2 -2
- package/dist/servers/socket-io.server.js +2 -2
- package/dist/servers/socket-io.server.js.map +1 -1
- package/dist/services/database.service.d.ts +8 -8
- package/dist/services/database.service.js +103 -34
- package/dist/services/database.service.js.map +1 -1
- package/dist/types/tcrawl-config.d.ts +1 -1
- package/dist/types/thydra-config.d.ts +1 -1
- package/dist/types/tparser-ctor.d.ts +1 -1
- package/dist/types/tquery.d.ts +1 -1
- package/dist/types/trobots-config.d.ts +1 -1
- package/package.json +32 -32
package/dist/classes/crawler.js
CHANGED
|
@@ -21,18 +21,6 @@ import { Random } from '../helpers/random';
|
|
|
21
21
|
import { EList } from '../enums/elist';
|
|
22
22
|
import { Dns } from './dns';
|
|
23
23
|
export class Crawler {
|
|
24
|
-
constructor(domain, database, crawlConfig, parsersConfig, robotsConfig, parsers, lists, tracker) {
|
|
25
|
-
this.domain = domain;
|
|
26
|
-
this.database = database;
|
|
27
|
-
this.crawlConfig = crawlConfig;
|
|
28
|
-
this.parsersConfig = parsersConfig;
|
|
29
|
-
this.robotsConfig = robotsConfig;
|
|
30
|
-
this.parsers = parsers;
|
|
31
|
-
this.lists = lists;
|
|
32
|
-
this.tracker = tracker;
|
|
33
|
-
this.isPaused = false;
|
|
34
|
-
this.isAborted = false;
|
|
35
|
-
}
|
|
36
24
|
static applyMasqueradeHeaders(request) {
|
|
37
25
|
request.setHeader('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8');
|
|
38
26
|
request.setHeader('Accept-Encoding', 'identity');
|
|
@@ -170,6 +158,18 @@ export class Crawler {
|
|
|
170
158
|
}
|
|
171
159
|
return map;
|
|
172
160
|
}
|
|
161
|
+
constructor(domain, database, crawlConfig, parsersConfig, robotsConfig, parsers, lists, tracker) {
|
|
162
|
+
this.domain = domain;
|
|
163
|
+
this.database = database;
|
|
164
|
+
this.crawlConfig = crawlConfig;
|
|
165
|
+
this.parsersConfig = parsersConfig;
|
|
166
|
+
this.robotsConfig = robotsConfig;
|
|
167
|
+
this.parsers = parsers;
|
|
168
|
+
this.lists = lists;
|
|
169
|
+
this.tracker = tracker;
|
|
170
|
+
this.isPaused = false;
|
|
171
|
+
this.isAborted = false;
|
|
172
|
+
}
|
|
173
173
|
abort() {
|
|
174
174
|
this.isAborted = true;
|
|
175
175
|
commonsAsyncAbortTimeout(`crawl_${this.domain}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/classes/crawler.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAG/B,OAAO,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAErF,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnF,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3I,OAAO,EAAE,+BAA+B,EAAE,kCAAkC,EAAE,MAAM,wBAAwB,CAAC;AAC7G,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAGzD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAKzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAU3C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAE5B,MAAM,OAAO,OAAO;IAqMnB,YACU,MAAc,EACd,QAAyB,EACzB,WAAyB,EACzB,aAAwC,EACxC,YAA2B,EAC3B,OAAsB,EACtB,KAAY,EACZ,OAAiB;QAPjB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAA2B;QACxC,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAe;QACtB,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAU;QAXnB,aAAQ,GAAY,KAAK,CAAC;QAC1B,cAAS,GAAY,KAAK,CAAC;IAWhC,CAAC;IA7MG,MAAM,CAAC,sBAAsB,CAAC,OAA2B;QAC/D,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,uFAAuF,CAAC,CAAC;QACrH,OAAO,CAAC,SAAS,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,qJAAqJ,CAAC,CAAC;IACxL,CAAC;IAEM,MAAM,CAAC,OAAO,CACnB,OAAiC,EACjC,GAAW,EACX,cAAsB,EACtB,WAAmB,EACnB,OAAiB;QAElB,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAqC,EAAE,MAA0B,EAAQ,EAAE;YAC/G,MAAM,KAAK,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,GAAY,KAAK,CAAC;YAE9B,IAAI,SAAS,GAAY,KAAK,CAAC;YAC/B,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,eAAe,GAAW,uBAAuB,GAAG,EAAE,CAAC;YAE7D,MAAM,kBAAkB,GAAW,WAAW,GAAG,oBAAoB,CAAC;YACtE,mBAAmB,CACjB,cAAc,GAAG,KAAK,EACtB,kBAAkB,CACnB;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,kBAAkB,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;gBAEnD,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACnC,OAAO;YACR,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC;YACT,CAAC,CAAC,CAAC;YAEL,MAAM,GAAG,GAAuB,OAAO,CAAC,OAAO,CAC7C,GAAG,EACH,CAAC,GAAyB,EAAQ,EAAE;gBACnC,4EAA4E;gBAC5E,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAW,GAAG,GAAG,KAAK,CAAC;gBAEpC,gDAAgD;gBAEhD,IAAI,IAAI,GAAW,CAAC,CAAC;gBACrB,MAAM,IAAI,GAAa,EAAE,CAAC;gBAC1B,GAAG,CAAC,EAAE,CACJ,MAAM,EACN,CAAC,KAAa,EAAQ,EAAE;oBACvB,IAAI,IAAI,GAAG,WAAW,EAAE;wBACvB,IAAI,CAAC,QAAQ,EAAE;4BACd,QAAQ,GAAG,IAAI,CAAC;4BAChB,GAAG,CAAC,OAAO,EAAE,CAAC;4BAEd,kBAAkB,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;yBAC3D;wBAED,OAAO;qBACP;oBACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACjB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;oBACrB,IAAI,OAAO;wBAAE,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC,CACF,CAAC;gBACF,GAAG,CAAC,EAAE,CACJ,KAAK,EACL,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,SAAS,GAAG,IAAI,CAAC;oBAEjB,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,OAAO,CAAC;wBACN,OAAO,EAAE,OAAO;wBAChB,mEAAmE;wBACnE,EAAE,EAAE,GAAG,CAAC,UAAU,CAAC;wBACnB,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,GAAG;wBACjC,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;wBACzB,QAAQ,EAAE,QAAQ;qBACnB,CAAC,CAAC;gBACJ,CAAC,CACF,CAAC;gBAEF,wEAAwE;gBACxE,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;oBACpB,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;gBACb,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAE/B,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAEpC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAS,EAAE;gBAC5B,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,QAAQ,GAAG,IAAI,CAAC;gBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;gBAEd,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CACJ,UAAU,EACV,CAAC,GAAyB,EAAQ,EAAE;gBACnC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YAChD,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;gBACpB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;YACb,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,QAAQ,EACR,CAAC,MAAc,EAAQ,EAAE;gBACxB,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAClC,MAAM,CAAC,EAAE,CACP,SAAS,EACT,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,QAAQ,GAAG,IAAI,CAAC;oBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;oBAEd,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACpC,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,GAAG,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,OAA4B,EAAE,WAAqB;QAC9E,MAAM,GAAG,GAAoC,IAAI,GAAG,EAA8B,CAAC;QAEnF,MAAM,IAAI,GAAoC,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE;YAC5B,MAAM,EAAE,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACvC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACtD,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAE9D,MAAM,KAAK,GAAW,OAAO,CAAC,GAAG,CAAC,GAAG,CAAW,CAAC;YACjD,QAAQ,GAAG,CAAC,IAAI,EAAE;gBACjB,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAAC,MAAM;gBACzD,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;oBAAC,MAAM;gBAC5D,KAAK,MAAM;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAC,MAAM;gBACtD,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;aACpD;SACD;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;IAgBO,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAEM,KAAK;QACX,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtB,CAAC;IAEM,MAAM;QACZ,sBAAsB,CAAC,wBAAwB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACvB,CAAC;IAEY,KAAK,CAAC,GAAW,EAAE,WAAoB;;YACnD,iBAAiB,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YAErC,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjC,IAAI,OAA2C,CAAC;YAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE;gBACxB,KAAK,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC;oBAAC,MAAM;gBACpC,KAAK,QAAQ;oBAAE,OAAO,GAAG,KAAK,CAAC;oBAAC,MAAM;gBACtC;oBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjE;YAED,IAAI,OAAwB,CAAC;YAC7B,IAAI;gBACH,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,OAAO,EACP,GAAG,EACH,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;aACF;YAAC,OAAO,EAAE,EAAE;gBACZ,kEAAkE;gBAClE,MAAM,EAAE,CAAC;aACT;YAED,IAAI,WAAW,EAAE;gBAChB,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;gBACxD,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;aACxC;YAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YAE3D,MAAM,OAAO,GAAoC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACrH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrH,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;YAEjE,MAAM,QAAQ,GAAa,EAAE,CAAC;YAE9B,IACC,OAAO,CAAC,UAAU,IAAI,GAAG;mBACtB,OAAO,CAAC,UAAU,GAAG,GAAG;mBACxB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;mBACvB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,0CAA0C;cAC/E;gBACD,iBAAiB;gBACjB,MAAM,QAAQ,GAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAY,EAAE,GAAG,CAAC,CAAC;gBAEvE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;oBAC3C,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;oBACzH,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAEpI,IAAI,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG;wBAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;iBACpE;aACD;YAED,IAAI,OAAO,CAAC,IAAI,EAAE;gBACjB,MAAM,IAAI,GAAW,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAClD,MAAM,QAAQ,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEpE,IAAI,IAAI,KAAK,QAAQ;oBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;aAC9D;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;gBACvE,MAAM,WAAW,GAAW,OAAO,CAAC,GAAG,CAAC,cAAc,CAAY,CAAC;gBACnE,MAAM,OAAO,GAAY,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAE5D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE;oBAChC,MAAM,MAAM,GAAY,IAAI,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;oBACnE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;wBAAE,SAAS;oBAElC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;wBAAE,SAAS;oBAErD,IAAI;wBACH,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;qBAClC;oBAAC,OAAO,EAAE,EAAE;wBACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;wBACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qBAChB;oBAED,IAAI,OAAO,EAAE;wBACZ,IAAI;4BACH,MAAM,KAAK,GAAa,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;4BAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gCAAE,SAAS;4BAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gCACzB,IAAI,IAAI,KAAK,GAAG;oCAAE,SAAS;gCAC3B,IAAI;oCACH,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iCACpB;gCAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;6BACjC;yBACD;wBAAC,OAAO,EAAE,EAAE;4BACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;yBAChB;qBACD;iBACD;aACD;YAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE;gBAC5B,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC3F,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;aACrH;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAExC,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAErD,OAAO,OAAO,CAAC,UAAU,CAAC;QAC3B,CAAC;KAAA;IAEY,KAAK;;YACjB,MAAM,eAAe,GAAW,+BAA+B,CAAC,GAAS,EAAE;gBAC1E,kBAAkB,CAAC,8CAA8C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBAEjF,IAAI,CAAC,KAAK,EAAE,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI;gBACH,IAAI;oBACH,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;iBAChE;gBAAC,OAAO,EAAE,EAAE;oBACZ,IAAK,EAAY,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE;wBACvF,kBAAkB,CAAC,qBAAqB,IAAI,CAAC,MAAM,yBAAyB,CAAC,CAAC;wBAC9E,OAAO;qBACP;oBAED,kBAAkB,CAAC,UAAU,IAAI,CAAC,MAAM,WAAW,CAAC,CAAC;oBACrD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC1C,OAAO;iBACP;gBAED,MAAM,MAAM,GAAU,IAAI,KAAK,CAC7B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CACb,CAAC;gBACF,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEpB,IAAI,KAAK,GAAW,CAAC,CAAC;gBACtB,OAAO,IAAI,EAAE;oBACZ,IAAI,IAAI,CAAC,SAAS;wBAAE,MAAM;oBAE1B,IAAI,IAAI,CAAC,QAAQ,EAAE;wBAClB,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBAChC,SAAS;qBACT;oBAED,MAAM,GAAG,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACpE,IAAI,CAAC,GAAG,EAAE;wBACT,sBAAsB,CAAC,aAAa,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;wBAC9D,MAAM;qBACN;oBAED,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBAEzC,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;oBAEzD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;wBACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;wBAEtD,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;qBACT;oBAED,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;wBAC7B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;wBACvD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;wBAE5D,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;qBACT;oBAED,IAAI,CAAC,CAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA,EAAE;wBACxD,kBAAkB,CAAC,sCAAsC,GAAG,qBAAqB,CAAC,CAAC;wBACnF,MAAM;qBACN;oBACD,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAExD,IAAI,KAAK,GAAG,CAAC,EAAE;wBACd,IAAI;4BACH,MAAM,OAAO,GAAW,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;4BAE/F,MAAM,mBAAmB,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;4BAE3D,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;gCACxC,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;6BAChC;yBACD;wBAAC,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE;qBAC7B;oBACD,IAAI,IAAI,CAAC,SAAS;wBAAE,MAAM;oBAE1B,IAAI,IAAI,CAAC,OAAO;wBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACnD,IAAI;wBACH,MAAM,UAAU,GAAW,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC;wBAE9D,IAAI,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;wBAC3D,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;qBACtD;oBAAC,OAAO,EAAE,EAAE;wBACZ,IAAI,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAEjD,MAAM,GAAG,GAAW,CAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAI,CAAC,CAAC;wBACzD,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;4BACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;4BACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;4BACnD,IAAI,IAAI,CAAC,OAAO;gCAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;yBACxD;6BAAM;4BACN,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;4BACnD,IAAI,IAAI,CAAC,OAAO;gCAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;4BAExD,IAAI,4BAA4B,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE;gCAC7C,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAAuB,CAAC,IAAI,CAAC,CAAC;6BACtE;iCAAM,IAAI,4BAA4B,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE;gCACvD,IAAI,mBAAmB,CAAC,IAAI,CAAE,EAA0B,CAAC,OAAO,CAAC,EAAE;oCAClE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;iCAClD;qCAAM;oCACN,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAA0B,CAAC,OAAO,CAAC,CAAC;iCAC5E;6BACD;yBACD;qBACD;4BAAS;wBACT,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;qBACzD;oBAED,KAAK,EAAE,CAAC;oBACR,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE;wBACjD,kBAAkB,CAAC,aAAa,IAAI,CAAC,MAAM,0DAA0D,CAAC,CAAC;wBACvG,MAAM;qBACN;iBACD;aACD;YAAC,OAAO,CAAC,EAAE;gBACX,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aACf;oBAAS;gBACT,kCAAkC,CAAC,eAAe,CAAC,CAAC;aACpD;QACF,CAAC;KAAA;CACD"}
|
|
1
|
+
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/classes/crawler.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAG/B,OAAO,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAErF,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnF,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3I,OAAO,EAAE,+BAA+B,EAAE,kCAAkC,EAAE,MAAM,wBAAwB,CAAC;AAC7G,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAGzD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAKzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAU3C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAE5B,MAAM,OAAO,OAAO;IACZ,MAAM,CAAC,sBAAsB,CAAC,OAA2B;QAC/D,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,uFAAuF,CAAC,CAAC;QACrH,OAAO,CAAC,SAAS,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,qJAAqJ,CAAC,CAAC;IACxL,CAAC;IAEM,MAAM,CAAC,OAAO,CACnB,OAAiC,EACjC,GAAW,EACX,cAAsB,EACtB,WAAmB,EACnB,OAAiB;QAElB,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAqC,EAAE,MAA0B,EAAQ,EAAE;YAC/G,MAAM,KAAK,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,GAAY,KAAK,CAAC;YAE9B,IAAI,SAAS,GAAY,KAAK,CAAC;YAC/B,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,eAAe,GAAW,uBAAuB,GAAG,EAAE,CAAC;YAE7D,MAAM,kBAAkB,GAAW,WAAW,GAAG,oBAAoB,CAAC;YACtE,mBAAmB,CACjB,cAAc,GAAG,KAAK,EACtB,kBAAkB,CACnB;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,kBAAkB,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;gBAEnD,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACnC,OAAO;YACR,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC;YACT,CAAC,CAAC,CAAC;YAEL,MAAM,GAAG,GAAuB,OAAO,CAAC,OAAO,CAC7C,GAAG,EACH,CAAC,GAAyB,EAAQ,EAAE;gBACnC,4EAA4E;gBAC5E,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAW,GAAG,GAAG,KAAK,CAAC;gBAEpC,gDAAgD;gBAEhD,IAAI,IAAI,GAAW,CAAC,CAAC;gBACrB,MAAM,IAAI,GAAa,EAAE,CAAC;gBAC1B,GAAG,CAAC,EAAE,CACJ,MAAM,EACN,CAAC,KAAa,EAAQ,EAAE;oBACvB,IAAI,IAAI,GAAG,WAAW,EAAE;wBACvB,IAAI,CAAC,QAAQ,EAAE;4BACd,QAAQ,GAAG,IAAI,CAAC;4BAChB,GAAG,CAAC,OAAO,EAAE,CAAC;4BAEd,kBAAkB,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;yBAC3D;wBAED,OAAO;qBACP;oBACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACjB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;oBACrB,IAAI,OAAO;wBAAE,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC,CACF,CAAC;gBACF,GAAG,CAAC,EAAE,CACJ,KAAK,EACL,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,SAAS,GAAG,IAAI,CAAC;oBAEjB,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,OAAO,CAAC;wBACN,OAAO,EAAE,OAAO;wBAChB,mEAAmE;wBACnE,EAAE,EAAE,GAAG,CAAC,UAAU,CAAC;wBACnB,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,GAAG;wBACjC,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;wBACzB,QAAQ,EAAE,QAAQ;qBACnB,CAAC,CAAC;gBACJ,CAAC,CACF,CAAC;gBAEF,wEAAwE;gBACxE,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;oBACpB,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;gBACb,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAE/B,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAEpC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAS,EAAE;gBAC5B,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,QAAQ,GAAG,IAAI,CAAC;gBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;gBAEd,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CACJ,UAAU,EACV,CAAC,GAAyB,EAAQ,EAAE;gBACnC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YAChD,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;gBACpB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;YACb,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,QAAQ,EACR,CAAC,MAAc,EAAQ,EAAE;gBACxB,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAClC,MAAM,CAAC,EAAE,CACP,SAAS,EACT,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,QAAQ,GAAG,IAAI,CAAC;oBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;oBAEd,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACpC,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,GAAG,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,OAA4B,EAAE,WAAqB;QAC9E,MAAM,GAAG,GAAoC,IAAI,GAAG,EAA8B,CAAC;QAEnF,MAAM,IAAI,GAAoC,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE;YAC5B,MAAM,EAAE,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACvC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACtD,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAE9D,MAAM,KAAK,GAAW,OAAO,CAAC,GAAG,CAAC,GAAG,CAAW,CAAC;YACjD,QAAQ,GAAG,CAAC,IAAI,EAAE;gBACjB,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAAC,MAAM;gBACzD,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;oBAAC,MAAM;gBAC5D,KAAK,MAAM;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAC,MAAM;gBACtD,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;aACpD;SACD;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;IAKD,YACU,MAAc,EACd,QAAyB,EACzB,WAAyB,EACzB,aAAwC,EACxC,YAA2B,EAC3B,OAAsB,EACtB,KAAY,EACZ,OAAiB;QAPjB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAA2B;QACxC,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAe;QACtB,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAU;QAXnB,aAAQ,GAAY,KAAK,CAAC;QAC1B,cAAS,GAAY,KAAK,CAAC;IAWhC,CAAC;IAEI,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAEM,KAAK;QACX,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtB,CAAC;IAEM,MAAM;QACZ,sBAAsB,CAAC,wBAAwB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACvB,CAAC;IAEY,KAAK,CAAC,GAAW,EAAE,WAAoB;;YACnD,iBAAiB,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YAErC,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjC,IAAI,OAA2C,CAAC;YAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE;gBACxB,KAAK,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC;oBAAC,MAAM;gBACpC,KAAK,QAAQ;oBAAE,OAAO,GAAG,KAAK,CAAC;oBAAC,MAAM;gBACtC;oBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjE;YAED,IAAI,OAAwB,CAAC;YAC7B,IAAI;gBACH,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,OAAO,EACP,GAAG,EACH,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;aACF;YAAC,OAAO,EAAE,EAAE;gBACZ,kEAAkE;gBAClE,MAAM,EAAE,CAAC;aACT;YAED,IAAI,WAAW,EAAE;gBAChB,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;gBACxD,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;aACxC;YAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YAE3D,MAAM,OAAO,GAAoC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACrH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrH,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;YAEjE,MAAM,QAAQ,GAAa,EAAE,CAAC;YAE9B,IACC,OAAO,CAAC,UAAU,IAAI,GAAG;mBACtB,OAAO,CAAC,UAAU,GAAG,GAAG;mBACxB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;mBACvB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,0CAA0C;cAC/E;gBACD,iBAAiB;gBACjB,MAAM,QAAQ,GAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAY,EAAE,GAAG,CAAC,CAAC;gBAEvE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;oBAC3C,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;oBACzH,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAEpI,IAAI,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG;wBAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;iBACpE;aACD;YAED,IAAI,OAAO,CAAC,IAAI,EAAE;gBACjB,MAAM,IAAI,GAAW,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAClD,MAAM,QAAQ,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEpE,IAAI,IAAI,KAAK,QAAQ;oBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;aAC9D;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;gBACvE,MAAM,WAAW,GAAW,OAAO,CAAC,GAAG,CAAC,cAAc,CAAY,CAAC;gBACnE,MAAM,OAAO,GAAY,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAE5D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE;oBAChC,MAAM,MAAM,GAAY,IAAI,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;oBACnE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;wBAAE,SAAS;oBAElC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;wBAAE,SAAS;oBAErD,IAAI;wBACH,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;qBAClC;oBAAC,OAAO,EAAE,EAAE;wBACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;wBACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qBAChB;oBAED,IAAI,OAAO,EAAE;wBACZ,IAAI;4BACH,MAAM,KAAK,GAAa,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;4BAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gCAAE,SAAS;4BAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gCACzB,IAAI,IAAI,KAAK,GAAG;oCAAE,SAAS;gCAC3B,IAAI;oCACH,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iCACpB;gCAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;6BACjC;yBACD;wBAAC,OAAO,EAAE,EAAE;4BACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;yBAChB;qBACD;iBACD;aACD;YAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE;gBAC5B,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC3F,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;aACrH;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAExC,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAErD,OAAO,OAAO,CAAC,UAAU,CAAC;QAC3B,CAAC;KAAA;IAEY,KAAK;;YACjB,MAAM,eAAe,GAAW,+BAA+B,CAAC,GAAS,EAAE;gBAC1E,kBAAkB,CAAC,8CAA8C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBAEjF,IAAI,CAAC,KAAK,EAAE,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI;gBACH,IAAI;oBACH,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;iBAChE;gBAAC,OAAO,EAAE,EAAE;oBACZ,IAAK,EAAY,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE;wBACvF,kBAAkB,CAAC,qBAAqB,IAAI,CAAC,MAAM,yBAAyB,CAAC,CAAC;wBAC9E,OAAO;qBACP;oBAED,kBAAkB,CAAC,UAAU,IAAI,CAAC,MAAM,WAAW,CAAC,CAAC;oBACrD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC1C,OAAO;iBACP;gBAED,MAAM,MAAM,GAAU,IAAI,KAAK,CAC7B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CACb,CAAC;gBACF,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEpB,IAAI,KAAK,GAAW,CAAC,CAAC;gBACtB,OAAO,IAAI,EAAE;oBACZ,IAAI,IAAI,CAAC,SAAS;wBAAE,MAAM;oBAE1B,IAAI,IAAI,CAAC,QAAQ,EAAE;wBAClB,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBAChC,SAAS;qBACT;oBAED,MAAM,GAAG,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACpE,IAAI,CAAC,GAAG,EAAE;wBACT,sBAAsB,CAAC,aAAa,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;wBAC9D,MAAM;qBACN;oBAED,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBAEzC,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;oBAEzD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;wBACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;wBAEtD,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;qBACT;oBAED,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;wBAC7B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;wBACvD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;wBAE5D,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;qBACT;oBAED,IAAI,CAAC,CAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA,EAAE;wBACxD,kBAAkB,CAAC,sCAAsC,GAAG,qBAAqB,CAAC,CAAC;wBACnF,MAAM;qBACN;oBACD,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAExD,IAAI,KAAK,GAAG,CAAC,EAAE;wBACd,IAAI;4BACH,MAAM,OAAO,GAAW,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;4BAE/F,MAAM,mBAAmB,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;4BAE3D,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;gCACxC,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;6BAChC;yBACD;wBAAC,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE;qBAC7B;oBACD,IAAI,IAAI,CAAC,SAAS;wBAAE,MAAM;oBAE1B,IAAI,IAAI,CAAC,OAAO;wBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACnD,IAAI;wBACH,MAAM,UAAU,GAAW,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC;wBAE9D,IAAI,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;wBAC3D,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;qBACtD;oBAAC,OAAO,EAAE,EAAE;wBACZ,IAAI,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAEjD,MAAM,GAAG,GAAW,CAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAI,CAAC,CAAC;wBACzD,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;4BACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;4BACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;4BACnD,IAAI,IAAI,CAAC,OAAO;gCAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;yBACxD;6BAAM;4BACN,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;4BACnD,IAAI,IAAI,CAAC,OAAO;gCAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;4BAExD,IAAI,4BAA4B,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE;gCAC7C,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAAuB,CAAC,IAAI,CAAC,CAAC;6BACtE;iCAAM,IAAI,4BAA4B,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE;gCACvD,IAAI,mBAAmB,CAAC,IAAI,CAAE,EAA0B,CAAC,OAAO,CAAC,EAAE;oCAClE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;iCAClD;qCAAM;oCACN,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAA0B,CAAC,OAAO,CAAC,CAAC;iCAC5E;6BACD;yBACD;qBACD;4BAAS;wBACT,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;qBACzD;oBAED,KAAK,EAAE,CAAC;oBACR,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE;wBACjD,kBAAkB,CAAC,aAAa,IAAI,CAAC,MAAM,0DAA0D,CAAC,CAAC;wBACvG,MAAM;qBACN;iBACD;aACD;YAAC,OAAO,CAAC,EAAE;gBACX,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aACf;oBAAS;gBACT,kCAAkC,CAAC,eAAe,CAAC,CAAC;aACpD;QACF,CAAC;KAAA;CACD"}
|
package/dist/classes/expirer.js
CHANGED
|
@@ -7,6 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
+
import mongodb from 'mongodb';
|
|
10
11
|
import { commonsArrayChunk, commonsDateDateToYmdHis, commonsTypeHasPropertyDate } from 'tscommons-es-core';
|
|
11
12
|
import { EStatus } from 'hydra-crawler-ts-assets';
|
|
12
13
|
import { commonsOutputAlert, commonsOutputDoing, commonsOutputProgress, commonsOutputSuccess } from 'nodecommons-es-cli';
|
|
@@ -74,7 +75,9 @@ export class Expirer {
|
|
|
74
75
|
commonsOutputDoing('Re-queuing expired');
|
|
75
76
|
tally = 0;
|
|
76
77
|
for (const batch of batches) {
|
|
77
|
-
|
|
78
|
+
const batchIds = batch
|
|
79
|
+
.map((id) => new mongodb.ObjectId(id));
|
|
80
|
+
yield this.database.getUrls().updateMany({ _id: { $in: batchIds } }, {
|
|
78
81
|
$set: { status: EStatus.QUEUED },
|
|
79
82
|
$unset: { ttl: true }
|
|
80
83
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expirer.js","sourceRoot":"","sources":["../../src/classes/expirer.ts"],"names":[],"mappings":";;;;;;;;;
|
|
1
|
+
{"version":3,"file":"expirer.js","sourceRoot":"","sources":["../../src/classes/expirer.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAIzH,OAAO,EAAgC,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE3F,MAAM,OAAO,OAAO;IACnB,YACU,MAAc,EACd,QAAyB;QADzB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;IAChC,CAAC;IAEU,mBAAmB;;YAChC,kBAAkB,CAAC,qCAAqC,CAAC,CAAC;YAC1D,MAAM,OAAO,GAA2C,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,SAAS,CAAc;gBACrG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE;gBACnE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;aAC/B,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3B,MAAM,MAAM,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAE3F,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpC,OAAO,MAAM;gBACX,gDAAgD;iBAC/C,GAAG,CAAC,CAAC,CAAc,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;KAAA;IAEY,MAAM,CAAC,KAAc;;YACjC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;YAE5C,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE1D,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,MAAM,OAAO,GAA6B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACpE,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE;gCAChB,OAAO,CAAC,QAAQ;gCAChB,OAAO,CAAC,MAAM;gCACd,OAAO,CAAC,MAAM;6BACf,EAAE,EAAE;oBACL,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;iBACjC,EAAC,EACF,EAAE,CACH,CAAC;YAEF,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,MAAM,QAAQ,GAAW,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE5D,IAAI,CAAC,0BAA0B,CAAC,GAAG,EAAE,WAAW,CAAC;oBAAE,SAAS;gBAE5D,MAAM,SAAS,GAAY,GAAG,CAAC,WAAW,CAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;gBACtE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,EAAE;oBACjC,iEAAiE;oBACjE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;oBACzB,KAAK,EAAE,CAAC;iBACR;aACD;YACD,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE5B,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE9B,MAAM,OAAO,GAAe,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAE5D,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,QAAQ,GAAuB,KAAK;qBACvC,GAAG,CAAC,CAAC,EAAU,EAAoB,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEnE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC1B;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;iBACtB,CACF,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;YAEvB,OAAO,IAAI,CAAC;QACb,CAAC;KAAA;IAEY,WAAW,CAAC,SAAe;;YACvC,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;YAEvD,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE1D,kBAAkB,CAAC,mCAAmC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACtD,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF,EAAE,CACH,CAAC,KAAK,EAAE,CAAC;YACV,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE5B,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE9B,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;gBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;aACtB,CACF,CAAC;YACF,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,OAAO,IAAI,CAAC;QACb,CAAC;KAAA;CACD"}
|
package/dist/classes/robot.js
CHANGED
|
@@ -12,13 +12,6 @@ import * as https from 'https';
|
|
|
12
12
|
import { URL } from 'url';
|
|
13
13
|
import { Crawler } from './crawler';
|
|
14
14
|
export class Robot {
|
|
15
|
-
constructor(domain, crawlConfig, robotsConfig, tracker) {
|
|
16
|
-
this.domain = domain;
|
|
17
|
-
this.crawlConfig = crawlConfig;
|
|
18
|
-
this.robotsConfig = robotsConfig;
|
|
19
|
-
this.tracker = tracker;
|
|
20
|
-
this.paths = new Map();
|
|
21
|
-
}
|
|
22
15
|
static parse(robotstxt, robotsConfig) {
|
|
23
16
|
const paths = [];
|
|
24
17
|
const sections = robotstxt.replace(/\r/g, '').split(/\n\n/);
|
|
@@ -38,6 +31,13 @@ export class Robot {
|
|
|
38
31
|
}
|
|
39
32
|
return [...new Set(paths)];
|
|
40
33
|
}
|
|
34
|
+
constructor(domain, crawlConfig, robotsConfig, tracker) {
|
|
35
|
+
this.domain = domain;
|
|
36
|
+
this.crawlConfig = crawlConfig;
|
|
37
|
+
this.robotsConfig = robotsConfig;
|
|
38
|
+
this.tracker = tracker;
|
|
39
|
+
this.paths = new Map();
|
|
40
|
+
}
|
|
41
41
|
checkDomain(protocol) {
|
|
42
42
|
return __awaiter(this, void 0, void 0, function* () {
|
|
43
43
|
let outcome;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robot.js","sourceRoot":"","sources":["../../src/classes/robot.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAO1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,OAAO,KAAK;
|
|
1
|
+
{"version":3,"file":"robot.js","sourceRoot":"","sources":["../../src/classes/robot.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAO1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,OAAO,KAAK;IACV,MAAM,CAAC,KAAK,CAAC,SAAiB,EAAE,YAA2B;QACjE,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,MAAM,QAAQ,GAAa,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtE,KAAK,IAAI,OAAO,IAAI,QAAQ,EAAE;YAC7B,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAEzB,MAAM,KAAK,GAA0B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAClF,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBAC/C,MAAM,KAAK,GAA0B,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAC9E,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,YAAY,CAAC,yBAAyB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;oBAAE,SAAS;gBACzE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;aACrB;SACD;QAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5B,CAAC;IAID,YACU,MAAc,EACd,WAAyB,EACzB,YAA2B,EAC3B,OAAiB;QAHjB,WAAM,GAAN,MAAM,CAAQ;QACd,gBAAW,GAAX,WAAW,CAAc;QACzB,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAU;QAE1B,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,CAAC;IAEa,WAAW,CAAC,QAAgB;;YACzC,IAAI,OAAkC,CAAC;YACvC,IAAI;gBACH,QAAQ,QAAQ,EAAE;oBACjB,KAAK,MAAM;wBACV,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,IAAI,EACJ,UAAU,IAAI,CAAC,MAAM,aAAa,EAClC,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;wBACF,MAAM;oBACP,KAAK,OAAO;wBACX,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,KAAK,EACL,WAAW,IAAI,CAAC,MAAM,aAAa,EACnC,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;wBACF,MAAM;oBACP,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;iBAC7C;aACD;YAAC,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE;YAE7B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI;gBAAE,OAAO,EAAE,CAAC;YAEzC,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAChE,CAAC;KAAA;IAEY,IAAI;;YAChB,KAAK,MAAM,QAAQ,IAAI,CAAE,MAAM,EAAE,OAAO,CAAE,EAAE;gBAC3C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;aAC3D;QACF,CAAC;KAAA;IAEM,YAAY,CAAC,GAAW;QAC9B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,MAAM,QAAQ,GAA0B,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjF,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE5B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAE,EAAE;YAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}
|
|
@@ -14,6 +14,18 @@ import { isIDictionaryConfig } from './dictionary.parser';
|
|
|
14
14
|
import { PhraseParser } from './phrase.parser';
|
|
15
15
|
let CACHE;
|
|
16
16
|
export class ComplexEnglishParser extends PhraseParser {
|
|
17
|
+
static loadKeyObject(config) {
|
|
18
|
+
const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
|
|
19
|
+
const json = commonsFileReadJsonFile(filename);
|
|
20
|
+
if (json === undefined)
|
|
21
|
+
throw new Error('Unable to read dictionary file');
|
|
22
|
+
if (!commonsTypeIsStringArrayKeyObject(json))
|
|
23
|
+
throw new Error('Dictionary file is not a JSON array');
|
|
24
|
+
return json;
|
|
25
|
+
}
|
|
26
|
+
static loadDictionary(config) {
|
|
27
|
+
return Object.keys(ComplexEnglishParser.loadKeyObject(config));
|
|
28
|
+
}
|
|
17
29
|
constructor(url, outcome, config) {
|
|
18
30
|
super(outcome, config, 'complexEnglish');
|
|
19
31
|
this.url = url;
|
|
@@ -29,18 +41,6 @@ export class ComplexEnglishParser extends PhraseParser {
|
|
|
29
41
|
const keys = Object.keys(this.dictionary);
|
|
30
42
|
this.setPhrases(keys, dictionaryConfig.caseSensitive || false);
|
|
31
43
|
}
|
|
32
|
-
static loadKeyObject(config) {
|
|
33
|
-
const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
|
|
34
|
-
const json = commonsFileReadJsonFile(filename);
|
|
35
|
-
if (json === undefined)
|
|
36
|
-
throw new Error('Unable to read dictionary file');
|
|
37
|
-
if (!commonsTypeIsStringArrayKeyObject(json))
|
|
38
|
-
throw new Error('Dictionary file is not a JSON array');
|
|
39
|
-
return json;
|
|
40
|
-
}
|
|
41
|
-
static loadDictionary(config) {
|
|
42
|
-
return Object.keys(ComplexEnglishParser.loadKeyObject(config));
|
|
43
|
-
}
|
|
44
44
|
getDictionary() {
|
|
45
45
|
return this.dictionary;
|
|
46
46
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complex-english.parser.js","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,IAAI,KAAqC,CAAC;AAE1C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;
|
|
1
|
+
{"version":3,"file":"complex-english.parser.js","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,IAAI,KAAqC,CAAC;AAE1C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;IACjE,MAAM,CAAC,aAAa,CAAC,MAAyB;QACpD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,IAAI,GAAsB,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAClE,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAE1E,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAErG,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,MAAyB;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;IAID,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAJhC,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,EAAE;YAC/F,kBAAkB,CAAC,yCAAyC,CAAC,CAAC;SAC9D;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,gBAAgB,CAAsB,CAAC;QAE1F,IAAI,CAAC,KAAK;YAAE,KAAK,GAAG,oBAAoB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB,MAAM,IAAI,GAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC;IAChE,CAAC;IAEM,aAAa;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAEe,YAAY,CAAC,QAAyB,EAAE,OAAiB;;YACxE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,kBAAkB,CAAC,SAAS,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAC;gBAC1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;aAC5D;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;aACrD;QACF,CAAC;KAAA;CACD"}
|
|
@@ -1,13 +1,6 @@
|
|
|
1
1
|
import * as cheerio from 'cheerio';
|
|
2
2
|
import { StringParser } from './string.parser';
|
|
3
3
|
export class HtmlParser extends StringParser {
|
|
4
|
-
constructor(outcome, config, configKey) {
|
|
5
|
-
super(outcome, config, configKey);
|
|
6
|
-
if (this.stringData && this.stringData.length) {
|
|
7
|
-
const collapsed = HtmlParser.collapseWhitespace(this.stringData);
|
|
8
|
-
this.dom = HtmlParser.loadDom(collapsed);
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
4
|
static collapseWhitespace(raw) {
|
|
12
5
|
return raw.replace(/\s+/g, ' ');
|
|
13
6
|
}
|
|
@@ -20,6 +13,13 @@ export class HtmlParser extends StringParser {
|
|
|
20
13
|
return undefined;
|
|
21
14
|
}
|
|
22
15
|
}
|
|
16
|
+
constructor(outcome, config, configKey) {
|
|
17
|
+
super(outcome, config, configKey);
|
|
18
|
+
if (this.stringData && this.stringData.length) {
|
|
19
|
+
const collapsed = HtmlParser.collapseWhitespace(this.stringData);
|
|
20
|
+
this.dom = HtmlParser.loadDom(collapsed);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
23
|
supports(contentType, isAllow) {
|
|
24
24
|
if (!isAllow)
|
|
25
25
|
return false;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html.parser.js","sourceRoot":"","sources":["../../src/parsers/html.parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,OAAgB,UAAkC,SAAQ,YAAe;
|
|
1
|
+
{"version":3,"file":"html.parser.js","sourceRoot":"","sources":["../../src/parsers/html.parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,OAAgB,UAAkC,SAAQ,YAAe;IACvE,MAAM,CAAC,kBAAkB,CAAC,GAAW;QAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,IAAY;QACjC,IAAI;YACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC1B;QAAC,OAAO,EAAE,EAAE;YACZ,SAAS;YACT,OAAO,SAAS,CAAC;SACjB;IACF,CAAC;IAID,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAC9C,MAAM,SAAS,GAAW,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;SACzC;IACF,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,4BAA4B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;CACD"}
|
|
@@ -19,11 +19,6 @@ export function isIHyperlinksConfig(test) {
|
|
|
19
19
|
return true;
|
|
20
20
|
}
|
|
21
21
|
export class HyperlinksParser extends HtmlParser {
|
|
22
|
-
constructor(url, outcome, config) {
|
|
23
|
-
super(outcome, config, 'hyperlinks');
|
|
24
|
-
this.url = url;
|
|
25
|
-
this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
|
|
26
|
-
}
|
|
27
22
|
static parseTagLinks(dom, tag, attribute) {
|
|
28
23
|
const links = [];
|
|
29
24
|
dom(`${tag}[${attribute}]`).each((_index, element) => {
|
|
@@ -48,6 +43,11 @@ export class HyperlinksParser extends HtmlParser {
|
|
|
48
43
|
catch (ex) { /* do nothing */ }
|
|
49
44
|
return undefined;
|
|
50
45
|
}
|
|
46
|
+
constructor(url, outcome, config) {
|
|
47
|
+
super(outcome, config, 'hyperlinks');
|
|
48
|
+
this.url = url;
|
|
49
|
+
this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
|
|
50
|
+
}
|
|
51
51
|
parse(database) {
|
|
52
52
|
return __awaiter(this, void 0, void 0, function* () {
|
|
53
53
|
if (!this.dom || !this.url || !this.hyperlinksConfig)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hyperlinks.parser.js","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAOtG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;
|
|
1
|
+
{"version":3,"file":"hyperlinks.parser.js","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAOtG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;IAE3D,MAAM,CAAC,aAAa,CACzB,GAAiB,EACjB,GAAW,EACX,SAAiB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACtF,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5D,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,GAAiB,EAAE,GAAW;QACvD,IAAI;YACH,MAAM,IAAI,GAAoB,GAAG,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAExC,MAAM,IAAI,GAAqB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAE5B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;SACzB;QAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QACjC,OAAO,SAAS,CAAC;IAClB,CAAC;IAID,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAJ5B,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAC7D,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,CAAC,gBAAgB;YAE9E,IAAI,KAAuB,CAAC;YAC5B,IAAI;gBACH,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACjC;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;YAEjC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE;gBAC1D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;aACjD;YAED,MAAM,KAAK,GAAkC,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO;oBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;aAClG;YAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;KAAA;IAED,4DAA4D;IAC/C,KAAK;;YACjB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,EAAE,CAAC,CAAC,gBAAgB;YAEjF,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO,EAAE;oBACZ,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;wBAAE,SAAS;oBACpC,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxF;aACD;YAED,MAAM,MAAM,GAAa,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAE7C,IAAI,QAAQ,GAAqB,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAClF,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,IAAI,MAAM,EAAE;gBACrB,MAAM,MAAM,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,KAAK,EAAE;oBAAE,SAAS;gBAEvB,IAAI;oBACH,MAAM,CAAC,GAAQ,IAAI,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;wBACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;qBAChB;oBACD,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;wBAAE,SAAS;oBAC/C,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,QAAQ,EAAE;wBAC3D,0CAA0C;wBAC1C,SAAS;qBACT;oBAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;iBAC3B;gBAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;aACjC;YAED,OAAO,OAAO,CAAC;QAChB,CAAC;KAAA;CACD"}
|
|
@@ -5,7 +5,7 @@ import { IParserConfig } from '../interfaces/iparser-config';
|
|
|
5
5
|
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
6
6
|
import { HtmlParser } from './html.parser';
|
|
7
7
|
import { IDataConfig } from './data.parser';
|
|
8
|
-
|
|
8
|
+
type TImageTag = {
|
|
9
9
|
src: string;
|
|
10
10
|
alt?: string;
|
|
11
11
|
title?: string;
|
|
@@ -10,10 +10,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
import { commonsOutputDebug } from 'nodecommons-es-cli';
|
|
11
11
|
import { HtmlParser } from './html.parser';
|
|
12
12
|
export class ImageTagsParser extends HtmlParser {
|
|
13
|
-
constructor(url, outcome, config) {
|
|
14
|
-
super(outcome, config, 'imageTags');
|
|
15
|
-
this.url = url;
|
|
16
|
-
}
|
|
17
13
|
static parseImageTags(dom) {
|
|
18
14
|
const images = [];
|
|
19
15
|
dom('img').each((_index, element) => {
|
|
@@ -34,6 +30,10 @@ export class ImageTagsParser extends HtmlParser {
|
|
|
34
30
|
});
|
|
35
31
|
return images;
|
|
36
32
|
}
|
|
33
|
+
constructor(url, outcome, config) {
|
|
34
|
+
super(outcome, config, 'imageTags');
|
|
35
|
+
this.url = url;
|
|
36
|
+
}
|
|
37
37
|
parse(database) {
|
|
38
38
|
return __awaiter(this, void 0, void 0, function* () {
|
|
39
39
|
if (!this.dom || !this.url)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image-tags.parser.js","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAS3C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;
|
|
1
|
+
{"version":3,"file":"image-tags.parser.js","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAS3C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;IACpD,MAAM,CAAC,cAAc,CAC1B,GAAiB;QAElB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACrE,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEnD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEhE,MAAM,KAAK,GAAc;gBACvB,GAAG,EAAE,GAAG;aACT,CAAC;YACF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAE/D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAJ3B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gBAAgB;YAEpD,MAAM,MAAM,GAAgB,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACtB,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACtD,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;aACnD;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;aAC7C;QACF,CAAC;KAAA;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complex-english.query.js","sourceRoot":"","sources":["../../src/queries/complex-english.query.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAc,MAAM,mBAAmB,CAAC;AAEnE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACrI,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAMzE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAUvC,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAAwC,EACzB,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,EAAE;QACZ,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO;KACP;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAmC,MAAM,CAAC,aAAa,EAAE,CAAC;IAC7E,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,kBAAkB,CAAC,uDAAuD,MAAM,EAAE,CAAC,CAAC;IAEpF,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"complex-english.query.js","sourceRoot":"","sources":["../../src/queries/complex-english.query.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAc,MAAM,mBAAmB,CAAC;AAEnE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACrI,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAMzE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAUvC,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAAwC,EACzB,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,EAAE;QACZ,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO;KACP;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAmC,MAAM,CAAC,aAAa,EAAE,CAAC;IAC7E,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,kBAAkB,CAAC,uDAAuD,MAAM,EAAE,CAAC,CAAC;IAEpF,MAAM,MAAM,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACrE;QACE,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO,CAAC,IAAI;QACpB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAClC,EACD,EAAE,CACH,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE;QACZ,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;YAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,MAAM,QAAQ,GAAkB,GAA+B,CAAC;QAEhE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,SAAS,EAAE;YACrF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAClB,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAChC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAClC;gBAAE,SAAS;SACZ;QAED,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,cAAc,EAAE,QAAQ,CAAC,cAAc;SACxC,CAAC,CAAC;KACH;IAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAqB,MAAM,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE;YACV,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;YACnD,SAAS;SACT;QAED,MAAM,GAAG,GAA2B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,EAAE;YACT,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;YACrD,SAAS;SACT;QAED,MAAM,KAAK,GAAoB,GAAG,CAAC,GAAG,CAAC;aACpC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAElG,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAQ,EAAE;YAClE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAa,WAAW;aAChC,IAAI,CAAC,IAAI,CAAC;aACV,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAa,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;YAC1B,IAAI,OAAO,GAAW,IAAI,CAAC;YAE3B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,cAAc,EAAE;gBAC3C,MAAM,KAAK,GAAqB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAEhC,MAAM,WAAW,GAAa,aAAa,CAAC,OAAO,CAAC,CAAC;gBAErD,MAAM,KAAK,GAAa,CAAE,KAAK,OAAO,EAAE,CAAE,CAAC;gBAC3C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAErE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aAC7D;YAED,IAAI,OAAO,KAAK,IAAI,EAAE;gBACrB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aAChB;SACD;KACD;AACF,CAAC,CAAA,CAAC;AAEF,wBAAwB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flash-content.query.js","sourceRoot":"","sources":["../../src/queries/flash-content.query.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACvI,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAgBjD,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEpD,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"flash-content.query.js","sourceRoot":"","sources":["../../src/queries/flash-content.query.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACvI,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAgBjD,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEpD,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACtE;QACE,GAAG,EAAE,eAAe;QACpB,MAAM,EAAE,EAAE,GAAG,EAAE;gBACb,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;gBAClB,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,MAAM;aACf,EAAE;KACJ,EACD,EAAE,CACH,CAAC;IACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,OAAO,EACP,MAAM,CACP,CAAC;IACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEjC,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,QAAQ,GAAuB,IAAI,GAAG,EAAiB,CAAC;IAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;YACE,QAAQ,EAAE,GAAG,CAAC,GAAG;SAClB,EACD,EAAE,CACH,CAAC;QACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;QAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;QACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAE7B,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;KACzB;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAW,EAAE,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE;YAC7C,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,IAAI,EAAE,GAAG,CAAC,GAAG;aACd,CAAC,CAAC;SACH;KACD;IACD,IAAI;SACD,IAAI,CAAC,CAAC,CAAO,EAAE,CAAO,EAAU,EAAE;QAClC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACV,CAAC,CAAC,CAAC;IAEL,oBAAoB,EAAE,CAAC;IAEvB,MAAM,GAAG,GAAe,IAAI,UAAU,CAAC;QACrC;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CACN,IAAI,EACJ,QAAQ,EACR,IAAI,CACL,CAAC;AACH,CAAC,CAAA,CAAC;AAEF,wBAAwB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"linking-to-domains.query.js","sourceRoot":"","sources":["../../src/queries/linking-to-domains.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAiB3J,MAAM,UAAU,GAAqC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;IACrF,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAqB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE;QACd,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,OAAO;KACP;IACD,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3C,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"linking-to-domains.query.js","sourceRoot":"","sources":["../../src/queries/linking-to-domains.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAiB3J,MAAM,UAAU,GAAqC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;IACrF,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAqB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE;QACd,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,OAAO;KACP;IACD,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3C,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAAgC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAC5E;QACE,MAAM,EAAE,KAAK;KACd,EACD,EAAE,CACH,CAAC;IACF,MAAM,OAAO,GAAc,MAAM,eAAe,CAAC,gBAAgB,CAC/D,OAAO,EACP,SAAS,CACV,CAAC;IACF,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpC,MAAM,UAAU,GAAyB,IAAI,GAAG,EAAmB,CAAC;IACpE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,kBAAkB,CAAC,0BAA0B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACvE;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;SAClC,EACD,EAAE,CACH,CAAC;QACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,QAAQ,EACR,MAAM,CACP,CAAC;QACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;KAC7B;IAED,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,cAAc,GAAqC,IAAI,GAAG,EAA+B,CAAC;IAChG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,MAAM,IAAI,GAAqB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAS;QAEpC,MAAM,GAAG,GAAuB,IAAI,GAAG,EAAiB,CAAC;QAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;gBACE,QAAQ,EAAE,GAAG,CAAC,GAAG;aAClB,EACD,EAAE,CACH,CAAC;YACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;YAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAE7B,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SACpB;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS;QAE7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE;QAC3C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAE5B,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,IAAI,EAAE,EAAE;YACrD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,GAAG,CAAE;iBACpE,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAC1C;KACD;IACD,oBAAoB,EAAE,CAAC;IAEvB,IAAI,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE;QACtC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;aACjB,IAAI,CAAC,UAAU,CAAC;aAChB,OAAO,CAAC,CAAC,MAAc,EAAQ,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;KACL;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,UAAU,GAA0B,IAAI,GAAG,EAAoB,CAAC;QAEtE,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE;gBAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACxE;aACD;SACD;QAED,MAAM,IAAI,GAAa,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aACjD,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEtC,IAAI,KAAK,GAAY,IAAI,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE;gBACvC,MAAM,GAAG,GAAa;oBACpB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAChB,GAAG;iBACJ,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE5B,KAAK,GAAG,KAAK,CAAC;aACd;SACD;KACD;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;KACrC;AACF,CAAC,CAAA,CAAC;AAEF,wBAAwB"}
|
|
@@ -11,11 +11,12 @@ declare enum EAction {
|
|
|
11
11
|
export declare function toEAction(value: string): EAction | undefined;
|
|
12
12
|
export declare class MaintenanceServer {
|
|
13
13
|
private crawl;
|
|
14
|
+
private hardLimit?;
|
|
14
15
|
private schedule;
|
|
15
16
|
private expirer;
|
|
16
17
|
private cleaner;
|
|
17
18
|
private isPaused;
|
|
18
|
-
constructor(times: TCommonsScheduleTime[], expiry: Expiry, lists: Lists, database: DatabaseService, crawl: CrawlServer);
|
|
19
|
+
constructor(times: TCommonsScheduleTime[], expiry: Expiry, lists: Lists, database: DatabaseService, crawl: CrawlServer, hardLimit?: number | undefined);
|
|
19
20
|
private perform;
|
|
20
21
|
start(): void;
|
|
21
22
|
}
|
|
@@ -30,8 +30,9 @@ export function toEAction(value) {
|
|
|
30
30
|
return undefined;
|
|
31
31
|
}
|
|
32
32
|
export class MaintenanceServer {
|
|
33
|
-
constructor(times, expiry, lists, database, crawl) {
|
|
33
|
+
constructor(times, expiry, lists, database, crawl, hardLimit) {
|
|
34
34
|
this.crawl = crawl;
|
|
35
|
+
this.hardLimit = hardLimit;
|
|
35
36
|
this.isPaused = false;
|
|
36
37
|
this.schedule = new CommonsSchedule('hydra-maintenance');
|
|
37
38
|
this.schedule.parse(times, (action) => toEAction(action), (action) => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -74,7 +75,7 @@ export class MaintenanceServer {
|
|
|
74
75
|
yield this.cleaner.purgeEmptyDomains();
|
|
75
76
|
break;
|
|
76
77
|
case EAction.PURGE_ORPHAN_URLS:
|
|
77
|
-
yield this.cleaner.purgeOrphanUrls();
|
|
78
|
+
yield this.cleaner.purgeOrphanUrls(this.hardLimit);
|
|
78
79
|
break;
|
|
79
80
|
}
|
|
80
81
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"maintenance.server.js","sourceRoot":"","sources":["../../src/servers/maintenance.server.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,oBAAoB,CAAC;AAEhG,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AAIzE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAM7C,IAAK,OAIJ;AAJD,WAAK,OAAO;IACX,4BAAiB,CAAA;IACjB,sDAA2C,CAAA;IAC3C,kDAAuC,CAAA;AACxC,CAAC,EAJI,OAAO,KAAP,OAAO,QAIX;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACtC,QAAQ,KAAK,EAAE;QACd,KAAK,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC7B,OAAO,OAAO,CAAC,MAAM,CAAC;QACvB,KAAK,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YAC1C,OAAO,OAAO,CAAC,mBAAmB,CAAC;QACpC,KAAK,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE;YACxC,OAAO,OAAO,CAAC,iBAAiB,CAAC;KAClC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAO7B,YACE,KAA6B,EAC7B,MAAc,EACd,KAAY,EACZ,QAAyB,EACjB,KAAkB;
|
|
1
|
+
{"version":3,"file":"maintenance.server.js","sourceRoot":"","sources":["../../src/servers/maintenance.server.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,oBAAoB,CAAC;AAEhG,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AAIzE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAM7C,IAAK,OAIJ;AAJD,WAAK,OAAO;IACX,4BAAiB,CAAA;IACjB,sDAA2C,CAAA;IAC3C,kDAAuC,CAAA;AACxC,CAAC,EAJI,OAAO,KAAP,OAAO,QAIX;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACtC,QAAQ,KAAK,EAAE;QACd,KAAK,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC7B,OAAO,OAAO,CAAC,MAAM,CAAC;QACvB,KAAK,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YAC1C,OAAO,OAAO,CAAC,mBAAmB,CAAC;QACpC,KAAK,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE;YACxC,OAAO,OAAO,CAAC,iBAAiB,CAAC;KAClC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAO7B,YACE,KAA6B,EAC7B,MAAc,EACd,KAAY,EACZ,QAAyB,EACjB,KAAkB,EAClB,SAAkB;QADlB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAS;QARpB,aAAQ,GAAY,KAAK,CAAC;QAUjC,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CACjB,KAAK,EACL,CAAC,MAAc,EAAqB,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EACxD,CAAO,MAAe,EAAiB,EAAE;YACxC,IAAI,YAAY,GAAY,KAAK,CAAC;YAElC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACrB,YAAY,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBAEnB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG;oBACjC,kBAAkB,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC;oBAE1D,IAAI;wBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;qBAChC;oBAAC,OAAO,EAAE,EAAE;wBACZ,gBAAgB;qBAChB;iBACD;aACD;YAED,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAE3B,IAAI,YAAY,EAAE;gBACjB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAEhD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;gBAEtB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;aACpB;QACF,CAAC,CAAA,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE5C,+BAA+B,CAAC,GAAS,EAAE;YAC1C,kBAAkB,CAAC,wDAAwD,CAAC,CAAC;YAC7E,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEa,OAAO,CAAC,MAAe;;YACpC,QAAQ,MAAM,EAAE;gBACf,KAAK,OAAO,CAAC,MAAM;oBAClB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAC5B,MAAM;gBACP,KAAK,OAAO,CAAC,mBAAmB;oBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;oBACvC,MAAM;gBACP,KAAK,OAAO,CAAC,iBAAiB;oBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACnD,MAAM;aACP;QACF,CAAC;KAAA;IAEM,KAAK;QACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACD"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
1
2
|
import { CommonsConfig } from 'tscommons-es-config';
|
|
2
|
-
import { CommonsStrictExpressServer } from 'nodecommons-es-express';
|
|
3
|
+
import { CommonsStrictExpressServer, ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
3
4
|
import { CommonsRestServer } from 'nodecommons-es-rest';
|
|
4
5
|
import { DatabaseService } from '../services/database.service';
|
|
5
|
-
export declare class RestServer extends CommonsRestServer {
|
|
6
|
+
export declare class RestServer extends CommonsRestServer<ICommonsRequestWithStrictParams, express.Response> {
|
|
6
7
|
constructor(expressServer: CommonsStrictExpressServer, database: DatabaseService, config: CommonsConfig);
|
|
7
8
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { commonsExpressNginxProxyPath } from 'nodecommons-es-express';
|
|
2
1
|
import { isICommonsExpressConfig } from 'nodecommons-es-express';
|
|
3
2
|
import { CommonsRestServer } from 'nodecommons-es-rest';
|
|
4
3
|
import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
|
|
4
|
+
import { commonsNginxProxyPath } from 'nodecommons-es-http';
|
|
5
5
|
import { TestApi } from '../apis/test.api';
|
|
6
6
|
import { StatisticsApi } from '../apis/statistics.api';
|
|
7
7
|
import { CrawlApi } from '../apis/crawl.api';
|
|
@@ -16,7 +16,7 @@ export class RestServer extends CommonsRestServer {
|
|
|
16
16
|
const expressConfig = config.getObject('express');
|
|
17
17
|
if (!isICommonsExpressConfig(expressConfig))
|
|
18
18
|
throw new Error('Express config is not valid');
|
|
19
|
-
const path =
|
|
19
|
+
const path = commonsNginxProxyPath(expressConfig.path);
|
|
20
20
|
commonsOutputDoing('Installing REST Test API');
|
|
21
21
|
new TestApi(this, path);
|
|
22
22
|
commonsOutputSuccess();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rest.server.js","sourceRoot":"","sources":["../../src/servers/rest.server.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rest.server.js","sourceRoot":"","sources":["../../src/servers/rest.server.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAI3D,MAAM,OAAO,UAAW,SAAQ,iBAAoE;IACnG,YACE,aAAyC,EACzC,QAAyB,EACzB,MAAqB;QAEtB,KAAK,CAAC,aAAa,CAAC,CAAC;QAErB,MAAM,aAAa,GAAoB,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAW,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAE/D,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;QACrD,IAAI,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACxC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACnC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,6BAA6B,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACrC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;QACjD,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACpC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;QACvD,IAAI,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC1C,oBAAoB,EAAE,CAAC;IACxB,CAAC;CAED"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { TStatusTallies } from 'hydra-crawler-ts-assets';
|
|
2
2
|
import { TFetch } from 'hydra-crawler-ts-assets';
|
|
3
3
|
import { TOutcome } from 'hydra-crawler-ts-assets';
|
|
4
|
-
import {
|
|
5
|
-
export declare class SocketIoServer extends
|
|
4
|
+
import { CommonsAppSocketIoServer } from 'nodecommons-es-app-socket-io';
|
|
5
|
+
export declare class SocketIoServer extends CommonsAppSocketIoServer {
|
|
6
6
|
fetching(fetch: TFetch): Promise<void>;
|
|
7
7
|
outcome(outcome: TOutcome): Promise<void>;
|
|
8
8
|
statusTallies(tallies: TStatusTallies): Promise<void>;
|
|
@@ -7,8 +7,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
-
import {
|
|
11
|
-
export class SocketIoServer extends
|
|
10
|
+
import { CommonsAppSocketIoServer } from 'nodecommons-es-app-socket-io';
|
|
11
|
+
export class SocketIoServer extends CommonsAppSocketIoServer {
|
|
12
12
|
fetching(fetch) {
|
|
13
13
|
const _super = Object.create(null, {
|
|
14
14
|
broadcast: { get: () => super.broadcast }
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-io.server.js","sourceRoot":"","sources":["../../src/servers/socket-io.server.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"socket-io.server.js","sourceRoot":"","sources":["../../src/servers/socket-io.server.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAExE,MAAM,OAAO,cAAe,SAAQ,wBAAwB;IAC9C,QAAQ,CAAC,KAAa;;;;;YAClC,MAAM,OAAM,SAAS,YAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC1C,CAAC;KAAA;IAEY,OAAO,CAAC,OAAiB;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;KAAA;IAEY,aAAa,CAAC,OAAuB;;;;;YACjD,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;KAAA;IAEY,WAAW,CAAC,KAAa;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;KAAA;IAEY,aAAa,CAAC,KAAa;;;;;YACvC,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;KAAA;IAEY,SAAS,CAAC,GAAW;;;;;YACjC,MAAM,OAAM,SAAS,YAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;KAAA;CACD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import mongodb from 'mongodb';
|
|
2
2
|
import { IUrlErrors } from 'hydra-crawler-ts-assets';
|
|
3
3
|
import { IImage } from 'hydra-crawler-ts-assets';
|
|
4
4
|
import { IUrl } from 'hydra-crawler-ts-assets';
|
|
@@ -11,11 +11,11 @@ import { EComparator } from 'hydra-crawler-ts-assets';
|
|
|
11
11
|
import { CommonsMongodbService } from 'nodecommons-es-database-mongodb';
|
|
12
12
|
import { TParserCtor } from '../types/tparser-ctor';
|
|
13
13
|
import { EAvailableStrategy } from '../enums/eavailable-strategy';
|
|
14
|
-
export
|
|
14
|
+
export type TMongoIdRow = {
|
|
15
15
|
_id: string;
|
|
16
16
|
};
|
|
17
17
|
export declare function isTMongoIdRow(test: unknown): test is TMongoIdRow;
|
|
18
|
-
export
|
|
18
|
+
export type TMongoIdTallyRow = {
|
|
19
19
|
_id: string;
|
|
20
20
|
tally: number;
|
|
21
21
|
};
|
|
@@ -25,13 +25,13 @@ export declare class DatabaseService extends CommonsMongodbService {
|
|
|
25
25
|
private urls;
|
|
26
26
|
private links;
|
|
27
27
|
private archiveds;
|
|
28
|
-
getDomains(): Collection<TDomain>;
|
|
29
|
-
getUrls(): Collection<IUrl>;
|
|
30
|
-
getLinks(): Collection<TLink>;
|
|
31
|
-
getArchiveds(): Collection<IUrl>;
|
|
28
|
+
getDomains(): mongodb.Collection<TDomain>;
|
|
29
|
+
getUrls(): mongodb.Collection<IUrl>;
|
|
30
|
+
getLinks(): mongodb.Collection<TLink>;
|
|
31
|
+
getArchiveds(): mongodb.Collection<IUrl>;
|
|
32
32
|
init(): Promise<void>;
|
|
33
33
|
initParser(ctor: TParserCtor): Promise<void>;
|
|
34
|
-
getRawDatabase(): Db;
|
|
34
|
+
getRawDatabase(): mongodb.Db;
|
|
35
35
|
wipe(): Promise<void>;
|
|
36
36
|
resetActive(): Promise<void>;
|
|
37
37
|
domain(domain: string, ip: string): Promise<boolean>;
|