hydra-crawler 2.2.4 → 2.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/autocomplete.api.d.ts +3 -1
- package/dist/apis/autocomplete.api.js.map +1 -1
- package/dist/apis/bugs.api.d.ts +3 -1
- package/dist/apis/bugs.api.js.map +1 -1
- package/dist/apis/crawl.api.d.ts +3 -1
- package/dist/apis/crawl.api.js.map +1 -1
- package/dist/apis/domains.api.d.ts +3 -1
- package/dist/apis/domains.api.js.map +1 -1
- package/dist/apis/images.api.d.ts +3 -1
- package/dist/apis/images.api.js.map +1 -1
- package/dist/apis/statistics.api.d.ts +3 -1
- package/dist/apis/statistics.api.js.map +1 -1
- package/dist/apis/test.api.d.ts +3 -1
- package/dist/apis/test.api.js.map +1 -1
- package/dist/apis/urls.api.d.ts +3 -1
- package/dist/apis/urls.api.js.map +1 -1
- package/dist/apps/cleanup.app.js +2 -2
- package/dist/apps/cleanup.app.js.map +1 -1
- package/dist/apps/cross-populate-export.app.js.map +1 -1
- package/dist/apps/cross-populate-import.app.js +21 -14
- package/dist/apps/cross-populate-import.app.js.map +1 -1
- package/dist/apps/denylist.app.js.map +1 -1
- package/dist/apps/hydra.app.d.ts +1 -0
- package/dist/apps/hydra.app.js +8 -1
- package/dist/apps/hydra.app.js.map +1 -1
- package/dist/apps/reattempt.app.js.map +1 -1
- package/dist/apps/startup.app.js.map +1 -1
- package/dist/apps/unarchive-urls.app.js.map +1 -1
- package/dist/classes/cleaner.js.map +1 -1
- package/dist/classes/crawler.js +12 -12
- package/dist/classes/crawler.js.map +1 -1
- package/dist/classes/expirer.js +4 -1
- package/dist/classes/expirer.js.map +1 -1
- package/dist/classes/robot.js +7 -7
- package/dist/classes/robot.js.map +1 -1
- package/dist/parsers/complex-english.parser.js +12 -12
- package/dist/parsers/complex-english.parser.js.map +1 -1
- package/dist/parsers/html.parser.js +7 -7
- package/dist/parsers/html.parser.js.map +1 -1
- package/dist/parsers/hyperlinks.parser.js +5 -5
- package/dist/parsers/hyperlinks.parser.js.map +1 -1
- package/dist/parsers/image-tags.parser.d.ts +1 -1
- package/dist/parsers/image-tags.parser.js +4 -4
- package/dist/parsers/image-tags.parser.js.map +1 -1
- package/dist/queries/complex-english.query.js.map +1 -1
- package/dist/queries/flash-content.query.js.map +1 -1
- package/dist/queries/linking-to-domains.query.js.map +1 -1
- package/dist/servers/rest.server.d.ts +3 -2
- package/dist/servers/rest.server.js +2 -2
- package/dist/servers/rest.server.js.map +1 -1
- package/dist/servers/socket-io.server.d.ts +2 -2
- package/dist/servers/socket-io.server.js +2 -2
- package/dist/servers/socket-io.server.js.map +1 -1
- package/dist/services/database.service.d.ts +8 -8
- package/dist/services/database.service.js +103 -34
- package/dist/services/database.service.js.map +1 -1
- package/dist/types/tcrawl-config.d.ts +1 -1
- package/dist/types/thydra-config.d.ts +1 -1
- package/dist/types/tparser-ctor.d.ts +1 -1
- package/dist/types/tquery.d.ts +1 -1
- package/dist/types/trobots-config.d.ts +1 -1
- package/package.json +32 -32
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"expirer.js","sourceRoot":"","sources":["../../src/classes/expirer.ts"],"names":[],"mappings":";;;;;;;;;
|
|
1
|
+
{"version":3,"file":"expirer.js","sourceRoot":"","sources":["../../src/classes/expirer.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAIzH,OAAO,EAAgC,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE3F,MAAM,OAAO,OAAO;IACnB,YACU,MAAc,EACd,QAAyB;QADzB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;IAChC,CAAC;IAEU,mBAAmB;;YAChC,kBAAkB,CAAC,qCAAqC,CAAC,CAAC;YAC1D,MAAM,OAAO,GAA2C,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,SAAS,CAAc;gBACrG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE;gBACnE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;aAC/B,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3B,MAAM,MAAM,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAE3F,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpC,OAAO,MAAM;gBACX,gDAAgD;iBAC/C,GAAG,CAAC,CAAC,CAAc,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;KAAA;IAEY,MAAM,CAAC,KAAc;;YACjC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;YAE5C,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE1D,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,MAAM,OAAO,GAA6B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACpE,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE;gCAChB,OAAO,CAAC,QAAQ;gCAChB,OAAO,CAAC,MAAM;gCACd,OAAO,CAAC,MAAM;6BACf,EAAE,EAAE;oBACL,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;iBACjC,EAAC,EACF,EAAE,CACH,CAAC;YAEF,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,MAAM,QAAQ,GAAW,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE5D,IAAI,CAAC,0BAA0B,CAAC,GAAG,EAAE,WAAW,CAAC;oBAAE,SAAS;gBAE5D,MAAM,SAAS,GAAY,GAAG,CAAC,WAAW,CAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;gBACtE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,EAAE;oBACjC,iEAAiE;oBACjE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;oBACzB,KAAK,EAAE,CAAC;iBACR;aACD;YACD,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE5B,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE9B,MAAM,OAAO,GAAe,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAE5D,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,QAAQ,GAAuB,KAAK;qBACvC,GAAG,CAAC,CAAC,EAAU,EAAoB,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEnE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC1B;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;iBACtB,CACF,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;YAEvB,OAAO,IAAI,CAAC;QACb,CAAC;KAAA;IAEY,WAAW,CAAC,SAAe;;YACvC,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;YAEvD,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE1D,kBAAkB,CAAC,mCAAmC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACtD,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF,EAAE,CACH,CAAC,KAAK,EAAE,CAAC;YACV,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE5B,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE9B,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;gBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;aACtB,CACF,CAAC;YACF,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,OAAO,IAAI,CAAC;QACb,CAAC;KAAA;CACD"}
|
package/dist/classes/robot.js
CHANGED
|
@@ -12,13 +12,6 @@ import * as https from 'https';
|
|
|
12
12
|
import { URL } from 'url';
|
|
13
13
|
import { Crawler } from './crawler';
|
|
14
14
|
export class Robot {
|
|
15
|
-
constructor(domain, crawlConfig, robotsConfig, tracker) {
|
|
16
|
-
this.domain = domain;
|
|
17
|
-
this.crawlConfig = crawlConfig;
|
|
18
|
-
this.robotsConfig = robotsConfig;
|
|
19
|
-
this.tracker = tracker;
|
|
20
|
-
this.paths = new Map();
|
|
21
|
-
}
|
|
22
15
|
static parse(robotstxt, robotsConfig) {
|
|
23
16
|
const paths = [];
|
|
24
17
|
const sections = robotstxt.replace(/\r/g, '').split(/\n\n/);
|
|
@@ -38,6 +31,13 @@ export class Robot {
|
|
|
38
31
|
}
|
|
39
32
|
return [...new Set(paths)];
|
|
40
33
|
}
|
|
34
|
+
constructor(domain, crawlConfig, robotsConfig, tracker) {
|
|
35
|
+
this.domain = domain;
|
|
36
|
+
this.crawlConfig = crawlConfig;
|
|
37
|
+
this.robotsConfig = robotsConfig;
|
|
38
|
+
this.tracker = tracker;
|
|
39
|
+
this.paths = new Map();
|
|
40
|
+
}
|
|
41
41
|
checkDomain(protocol) {
|
|
42
42
|
return __awaiter(this, void 0, void 0, function* () {
|
|
43
43
|
let outcome;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robot.js","sourceRoot":"","sources":["../../src/classes/robot.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAO1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,OAAO,KAAK;
|
|
1
|
+
{"version":3,"file":"robot.js","sourceRoot":"","sources":["../../src/classes/robot.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAO1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,OAAO,KAAK;IACV,MAAM,CAAC,KAAK,CAAC,SAAiB,EAAE,YAA2B;QACjE,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,MAAM,QAAQ,GAAa,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtE,KAAK,IAAI,OAAO,IAAI,QAAQ,EAAE;YAC7B,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAEzB,MAAM,KAAK,GAA0B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAClF,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBAC/C,MAAM,KAAK,GAA0B,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAC9E,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,YAAY,CAAC,yBAAyB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;oBAAE,SAAS;gBACzE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;aACrB;SACD;QAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5B,CAAC;IAID,YACU,MAAc,EACd,WAAyB,EACzB,YAA2B,EAC3B,OAAiB;QAHjB,WAAM,GAAN,MAAM,CAAQ;QACd,gBAAW,GAAX,WAAW,CAAc;QACzB,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAU;QAE1B,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,CAAC;IAEa,WAAW,CAAC,QAAgB;;YACzC,IAAI,OAAkC,CAAC;YACvC,IAAI;gBACH,QAAQ,QAAQ,EAAE;oBACjB,KAAK,MAAM;wBACV,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,IAAI,EACJ,UAAU,IAAI,CAAC,MAAM,aAAa,EAClC,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;wBACF,MAAM;oBACP,KAAK,OAAO;wBACX,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,KAAK,EACL,WAAW,IAAI,CAAC,MAAM,aAAa,EACnC,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;wBACF,MAAM;oBACP,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;iBAC7C;aACD;YAAC,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE;YAE7B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI;gBAAE,OAAO,EAAE,CAAC;YAEzC,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAChE,CAAC;KAAA;IAEY,IAAI;;YAChB,KAAK,MAAM,QAAQ,IAAI,CAAE,MAAM,EAAE,OAAO,CAAE,EAAE;gBAC3C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;aAC3D;QACF,CAAC;KAAA;IAEM,YAAY,CAAC,GAAW;QAC9B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,MAAM,QAAQ,GAA0B,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjF,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE5B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAE,EAAE;YAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}
|
|
@@ -14,6 +14,18 @@ import { isIDictionaryConfig } from './dictionary.parser';
|
|
|
14
14
|
import { PhraseParser } from './phrase.parser';
|
|
15
15
|
let CACHE;
|
|
16
16
|
export class ComplexEnglishParser extends PhraseParser {
|
|
17
|
+
static loadKeyObject(config) {
|
|
18
|
+
const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
|
|
19
|
+
const json = commonsFileReadJsonFile(filename);
|
|
20
|
+
if (json === undefined)
|
|
21
|
+
throw new Error('Unable to read dictionary file');
|
|
22
|
+
if (!commonsTypeIsStringArrayKeyObject(json))
|
|
23
|
+
throw new Error('Dictionary file is not a JSON array');
|
|
24
|
+
return json;
|
|
25
|
+
}
|
|
26
|
+
static loadDictionary(config) {
|
|
27
|
+
return Object.keys(ComplexEnglishParser.loadKeyObject(config));
|
|
28
|
+
}
|
|
17
29
|
constructor(url, outcome, config) {
|
|
18
30
|
super(outcome, config, 'complexEnglish');
|
|
19
31
|
this.url = url;
|
|
@@ -29,18 +41,6 @@ export class ComplexEnglishParser extends PhraseParser {
|
|
|
29
41
|
const keys = Object.keys(this.dictionary);
|
|
30
42
|
this.setPhrases(keys, dictionaryConfig.caseSensitive || false);
|
|
31
43
|
}
|
|
32
|
-
static loadKeyObject(config) {
|
|
33
|
-
const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
|
|
34
|
-
const json = commonsFileReadJsonFile(filename);
|
|
35
|
-
if (json === undefined)
|
|
36
|
-
throw new Error('Unable to read dictionary file');
|
|
37
|
-
if (!commonsTypeIsStringArrayKeyObject(json))
|
|
38
|
-
throw new Error('Dictionary file is not a JSON array');
|
|
39
|
-
return json;
|
|
40
|
-
}
|
|
41
|
-
static loadDictionary(config) {
|
|
42
|
-
return Object.keys(ComplexEnglishParser.loadKeyObject(config));
|
|
43
|
-
}
|
|
44
44
|
getDictionary() {
|
|
45
45
|
return this.dictionary;
|
|
46
46
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complex-english.parser.js","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,IAAI,KAAqC,CAAC;AAE1C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;
|
|
1
|
+
{"version":3,"file":"complex-english.parser.js","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,IAAI,KAAqC,CAAC;AAE1C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;IACjE,MAAM,CAAC,aAAa,CAAC,MAAyB;QACpD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,IAAI,GAAsB,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAClE,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAE1E,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAErG,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,MAAyB;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;IAID,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAJhC,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,EAAE;YAC/F,kBAAkB,CAAC,yCAAyC,CAAC,CAAC;SAC9D;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,gBAAgB,CAAsB,CAAC;QAE1F,IAAI,CAAC,KAAK;YAAE,KAAK,GAAG,oBAAoB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB,MAAM,IAAI,GAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC;IAChE,CAAC;IAEM,aAAa;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAEe,YAAY,CAAC,QAAyB,EAAE,OAAiB;;YACxE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,kBAAkB,CAAC,SAAS,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAC;gBAC1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;aAC5D;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;aACrD;QACF,CAAC;KAAA;CACD"}
|
|
@@ -1,13 +1,6 @@
|
|
|
1
1
|
import * as cheerio from 'cheerio';
|
|
2
2
|
import { StringParser } from './string.parser';
|
|
3
3
|
export class HtmlParser extends StringParser {
|
|
4
|
-
constructor(outcome, config, configKey) {
|
|
5
|
-
super(outcome, config, configKey);
|
|
6
|
-
if (this.stringData && this.stringData.length) {
|
|
7
|
-
const collapsed = HtmlParser.collapseWhitespace(this.stringData);
|
|
8
|
-
this.dom = HtmlParser.loadDom(collapsed);
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
4
|
static collapseWhitespace(raw) {
|
|
12
5
|
return raw.replace(/\s+/g, ' ');
|
|
13
6
|
}
|
|
@@ -20,6 +13,13 @@ export class HtmlParser extends StringParser {
|
|
|
20
13
|
return undefined;
|
|
21
14
|
}
|
|
22
15
|
}
|
|
16
|
+
constructor(outcome, config, configKey) {
|
|
17
|
+
super(outcome, config, configKey);
|
|
18
|
+
if (this.stringData && this.stringData.length) {
|
|
19
|
+
const collapsed = HtmlParser.collapseWhitespace(this.stringData);
|
|
20
|
+
this.dom = HtmlParser.loadDom(collapsed);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
23
|
supports(contentType, isAllow) {
|
|
24
24
|
if (!isAllow)
|
|
25
25
|
return false;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html.parser.js","sourceRoot":"","sources":["../../src/parsers/html.parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,OAAgB,UAAkC,SAAQ,YAAe;
|
|
1
|
+
{"version":3,"file":"html.parser.js","sourceRoot":"","sources":["../../src/parsers/html.parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,OAAgB,UAAkC,SAAQ,YAAe;IACvE,MAAM,CAAC,kBAAkB,CAAC,GAAW;QAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,IAAY;QACjC,IAAI;YACH,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC1B;QAAC,OAAO,EAAE,EAAE;YACZ,SAAS;YACT,OAAO,SAAS,CAAC;SACjB;IACF,CAAC;IAID,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;YAC9C,MAAM,SAAS,GAAW,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEzE,IAAI,CAAC,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;SACzC;IACF,CAAC;IAEM,QAAQ,CAAC,WAAmB,EAAE,OAAgB;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAC3B,OAAO,4BAA4B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvD,CAAC;CACD"}
|
|
@@ -19,11 +19,6 @@ export function isIHyperlinksConfig(test) {
|
|
|
19
19
|
return true;
|
|
20
20
|
}
|
|
21
21
|
export class HyperlinksParser extends HtmlParser {
|
|
22
|
-
constructor(url, outcome, config) {
|
|
23
|
-
super(outcome, config, 'hyperlinks');
|
|
24
|
-
this.url = url;
|
|
25
|
-
this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
|
|
26
|
-
}
|
|
27
22
|
static parseTagLinks(dom, tag, attribute) {
|
|
28
23
|
const links = [];
|
|
29
24
|
dom(`${tag}[${attribute}]`).each((_index, element) => {
|
|
@@ -48,6 +43,11 @@ export class HyperlinksParser extends HtmlParser {
|
|
|
48
43
|
catch (ex) { /* do nothing */ }
|
|
49
44
|
return undefined;
|
|
50
45
|
}
|
|
46
|
+
constructor(url, outcome, config) {
|
|
47
|
+
super(outcome, config, 'hyperlinks');
|
|
48
|
+
this.url = url;
|
|
49
|
+
this.hyperlinksConfig = this.getConfig(isIHyperlinksConfig);
|
|
50
|
+
}
|
|
51
51
|
parse(database) {
|
|
52
52
|
return __awaiter(this, void 0, void 0, function* () {
|
|
53
53
|
if (!this.dom || !this.url || !this.hyperlinksConfig)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hyperlinks.parser.js","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAOtG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;
|
|
1
|
+
{"version":3,"file":"hyperlinks.parser.js","sourceRoot":"","sources":["../../src/parsers/hyperlinks.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAOtG,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAe,aAAa,EAAE,MAAM,eAAe,CAAC;AAK3D,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAChD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,CAAC,iCAAiC,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAEtE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAA6B;IAE3D,MAAM,CAAC,aAAa,CACzB,GAAiB,EACjB,GAAW,EACX,SAAiB;QAElB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,GAAG,CAAC,GAAG,GAAG,IAAI,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACtF,MAAM,IAAI,GAAqB,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5D,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,GAAiB,EAAE,GAAW;QACvD,IAAI;YACH,MAAM,IAAI,GAAoB,GAAG,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAExC,MAAM,IAAI,GAAqB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAE5B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;SACzB;QAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;QACjC,OAAO,SAAS,CAAC;IAClB,CAAC;IAID,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAJ5B,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IAC7D,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,CAAC,gBAAgB;YAE9E,IAAI,KAAuB,CAAC;YAC5B,IAAI;gBACH,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACjC;YAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;YAEjC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,EAAE;gBAC1D,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;aACjD;YAED,MAAM,KAAK,GAAkC,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO;oBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;aAClG;YAED,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;KAAA;IAED,4DAA4D;IAC/C,KAAK;;YACjB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,OAAO,EAAE,CAAC,CAAC,gBAAgB;YAEjF,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;gBACnD,MAAM,OAAO,GAA0B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACvF,IAAI,OAAO,EAAE;oBACZ,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM;wBAAE,SAAS;oBACpC,KAAK,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACxF;aACD;YAED,MAAM,MAAM,GAAa,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAE7C,IAAI,QAAQ,GAAqB,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAClF,IAAI,QAAQ,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,IAAI,MAAM,EAAE;gBACrB,MAAM,MAAM,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,KAAK,EAAE;oBAAE,SAAS;gBAEvB,IAAI;oBACH,MAAM,CAAC,GAAQ,IAAI,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE;wBACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;qBAChB;oBACD,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;wBAAE,SAAS;oBAC/C,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,QAAQ,EAAE;wBAC3D,0CAA0C;wBAC1C,SAAS;qBACT;oBAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;iBAC3B;gBAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;aACjC;YAED,OAAO,OAAO,CAAC;QAChB,CAAC;KAAA;CACD"}
|
|
@@ -5,7 +5,7 @@ import { IParserConfig } from '../interfaces/iparser-config';
|
|
|
5
5
|
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
6
6
|
import { HtmlParser } from './html.parser';
|
|
7
7
|
import { IDataConfig } from './data.parser';
|
|
8
|
-
|
|
8
|
+
type TImageTag = {
|
|
9
9
|
src: string;
|
|
10
10
|
alt?: string;
|
|
11
11
|
title?: string;
|
|
@@ -10,10 +10,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
import { commonsOutputDebug } from 'nodecommons-es-cli';
|
|
11
11
|
import { HtmlParser } from './html.parser';
|
|
12
12
|
export class ImageTagsParser extends HtmlParser {
|
|
13
|
-
constructor(url, outcome, config) {
|
|
14
|
-
super(outcome, config, 'imageTags');
|
|
15
|
-
this.url = url;
|
|
16
|
-
}
|
|
17
13
|
static parseImageTags(dom) {
|
|
18
14
|
const images = [];
|
|
19
15
|
dom('img').each((_index, element) => {
|
|
@@ -34,6 +30,10 @@ export class ImageTagsParser extends HtmlParser {
|
|
|
34
30
|
});
|
|
35
31
|
return images;
|
|
36
32
|
}
|
|
33
|
+
constructor(url, outcome, config) {
|
|
34
|
+
super(outcome, config, 'imageTags');
|
|
35
|
+
this.url = url;
|
|
36
|
+
}
|
|
37
37
|
parse(database) {
|
|
38
38
|
return __awaiter(this, void 0, void 0, function* () {
|
|
39
39
|
if (!this.dom || !this.url)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image-tags.parser.js","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAS3C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;
|
|
1
|
+
{"version":3,"file":"image-tags.parser.js","sourceRoot":"","sources":["../../src/parsers/image-tags.parser.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAS3C,MAAM,OAAO,eAAgB,SAAQ,UAAuB;IACpD,MAAM,CAAC,cAAc,CAC1B,GAAiB;QAElB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAE/B,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE;YACrE,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEnD,MAAM,GAAG,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEhE,MAAM,KAAK,GAAc;gBACvB,GAAG,EAAE,GAAG;aACT,CAAC;YACF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;YACvD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;gBAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YAE/D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAJ3B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gBAAgB;YAEpD,MAAM,MAAM,GAAgB,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACtB,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBACtD,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;aACnD;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;aAC7C;QACF,CAAC;KAAA;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"complex-english.query.js","sourceRoot":"","sources":["../../src/queries/complex-english.query.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAc,MAAM,mBAAmB,CAAC;AAEnE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACrI,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAMzE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAUvC,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAAwC,EACzB,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,EAAE;QACZ,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO;KACP;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAmC,MAAM,CAAC,aAAa,EAAE,CAAC;IAC7E,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,kBAAkB,CAAC,uDAAuD,MAAM,EAAE,CAAC,CAAC;IAEpF,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"complex-english.query.js","sourceRoot":"","sources":["../../src/queries/complex-english.query.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,kBAAkB,EAAc,MAAM,mBAAmB,CAAC;AAEnE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACrI,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAMzE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAUvC,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,KAAY,EACZ,OAAe,EACf,aAAwC,EACzB,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,EAAE;QACZ,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO;KACP;IAED,MAAM,MAAM,GAAyB,IAAI,oBAAoB,CAC3D,SAAS,EACT,SAAS,EACT,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAwB,MAAM,CAAC,SAAS,EAAE,CAAC;IAEvD,MAAM,aAAa,GAAmC,MAAM,CAAC,aAAa,EAAE,CAAC;IAC7E,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAE/D,kBAAkB,CAAC,uDAAuD,MAAM,EAAE,CAAC,CAAC;IAEpF,MAAM,MAAM,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACrE;QACE,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO,CAAC,IAAI;QACpB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAClC,EACD,EAAE,CACH,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE;QACZ,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;YAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,MAAM,QAAQ,GAAkB,GAA+B,CAAC;QAEhE,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,SAAS,EAAE;YACrF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAClB,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,EAChC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAClC;gBAAE,SAAS;SACZ;QAED,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,cAAc,EAAE,QAAQ,CAAC,cAAc;SACxC,CAAC,CAAC;KACH;IAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,IAAI,GAAqB,MAAM,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE;YACV,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;YACnD,SAAS;SACT;QAED,MAAM,GAAG,GAA2B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,EAAE;YACT,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;YACrD,SAAS;SACT;QAED,MAAM,KAAK,GAAoB,GAAG,CAAC,GAAG,CAAC;aACpC,QAAQ,EAAE;YACX,sEAAsE;aACrE,MAAM,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAW,EAAE,CAAE,OAAe,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAElG,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,MAAc,EAAE,OAAwB,EAAQ,EAAE;YAClE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAa,WAAW;aAChC,IAAI,CAAC,IAAI,CAAC;aACV,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAE5C,MAAM,MAAM,GAAa,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAEnD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;YAC1B,IAAI,OAAO,GAAW,IAAI,CAAC;YAE3B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,cAAc,EAAE;gBAC3C,MAAM,KAAK,GAAqB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAEhC,MAAM,WAAW,GAAa,aAAa,CAAC,OAAO,CAAC,CAAC;gBAErD,MAAM,KAAK,GAAa,CAAE,KAAK,OAAO,EAAE,CAAE,CAAC;gBAC3C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAErE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aAC7D;YAED,IAAI,OAAO,KAAK,IAAI,EAAE;gBACrB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aAChB;SACD;KACD;AACF,CAAC,CAAA,CAAC;AAEF,wBAAwB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flash-content.query.js","sourceRoot":"","sources":["../../src/queries/flash-content.query.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACvI,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAgBjD,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEpD,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"flash-content.query.js","sourceRoot":"","sources":["../../src/queries/flash-content.query.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACvI,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAgBjD,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEpD,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACtE;QACE,GAAG,EAAE,eAAe;QACpB,MAAM,EAAE,EAAE,GAAG,EAAE;gBACb,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;gBAClB,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,MAAM;aACf,EAAE;KACJ,EACD,EAAE,CACH,CAAC;IACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,OAAO,EACP,MAAM,CACP,CAAC;IACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEjC,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,QAAQ,GAAuB,IAAI,GAAG,EAAiB,CAAC;IAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;YACE,QAAQ,EAAE,GAAG,CAAC,GAAG;SAClB,EACD,EAAE,CACH,CAAC;QACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;QAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;QACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAE7B,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;KACzB;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAW,EAAE,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE;YAC7C,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,IAAI,EAAE,GAAG,CAAC,GAAG;aACd,CAAC,CAAC;SACH;KACD;IACD,IAAI;SACD,IAAI,CAAC,CAAC,CAAO,EAAE,CAAO,EAAU,EAAE;QAClC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACV,CAAC,CAAC,CAAC;IAEL,oBAAoB,EAAE,CAAC;IAEvB,MAAM,GAAG,GAAe,IAAI,UAAU,CAAC;QACrC;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CACN,IAAI,EACJ,QAAQ,EACR,IAAI,CACL,CAAC;AACH,CAAC,CAAA,CAAC;AAEF,wBAAwB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"linking-to-domains.query.js","sourceRoot":"","sources":["../../src/queries/linking-to-domains.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAiB3J,MAAM,UAAU,GAAqC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;IACrF,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAqB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE;QACd,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,OAAO;KACP;IACD,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3C,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,
|
|
1
|
+
{"version":3,"file":"linking-to-domains.query.js","sourceRoot":"","sources":["../../src/queries/linking-to-domains.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAiB3J,MAAM,UAAU,GAAqC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;IACrF,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAqB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE;QACd,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,OAAO;KACP;IACD,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3C,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAAgC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAC5E;QACE,MAAM,EAAE,KAAK;KACd,EACD,EAAE,CACH,CAAC;IACF,MAAM,OAAO,GAAc,MAAM,eAAe,CAAC,gBAAgB,CAC/D,OAAO,EACP,SAAS,CACV,CAAC;IACF,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpC,MAAM,UAAU,GAAyB,IAAI,GAAG,EAAmB,CAAC;IACpE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,kBAAkB,CAAC,0BAA0B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACvE;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;SAClC,EACD,EAAE,CACH,CAAC;QACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,QAAQ,EACR,MAAM,CACP,CAAC;QACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;KAC7B;IAED,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,cAAc,GAAqC,IAAI,GAAG,EAA+B,CAAC;IAChG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,MAAM,IAAI,GAAqB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAS;QAEpC,MAAM,GAAG,GAAuB,IAAI,GAAG,EAAiB,CAAC;QAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;gBACE,QAAQ,EAAE,GAAG,CAAC,GAAG;aAClB,EACD,EAAE,CACH,CAAC;YACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;YAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAE7B,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SACpB;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS;QAE7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE;QAC3C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAE5B,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,IAAI,EAAE,EAAE;YACrD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,GAAG,CAAE;iBACpE,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAC1C;KACD;IACD,oBAAoB,EAAE,CAAC;IAEvB,IAAI,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE;QACtC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;aACjB,IAAI,CAAC,UAAU,CAAC;aAChB,OAAO,CAAC,CAAC,MAAc,EAAQ,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;KACL;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,UAAU,GAA0B,IAAI,GAAG,EAAoB,CAAC;QAEtE,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE;gBAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACxE;aACD;SACD;QAED,MAAM,IAAI,GAAa,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aACjD,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEtC,IAAI,KAAK,GAAY,IAAI,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE;gBACvC,MAAM,GAAG,GAAa;oBACpB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAChB,GAAG;iBACJ,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE5B,KAAK,GAAG,KAAK,CAAC;aACd;SACD;KACD;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;KACrC;AACF,CAAC,CAAA,CAAC;AAEF,wBAAwB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import * as express from 'express';
|
|
1
2
|
import { CommonsConfig } from 'tscommons-es-config';
|
|
2
|
-
import { CommonsStrictExpressServer } from 'nodecommons-es-express';
|
|
3
|
+
import { CommonsStrictExpressServer, ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
|
|
3
4
|
import { CommonsRestServer } from 'nodecommons-es-rest';
|
|
4
5
|
import { DatabaseService } from '../services/database.service';
|
|
5
|
-
export declare class RestServer extends CommonsRestServer {
|
|
6
|
+
export declare class RestServer extends CommonsRestServer<ICommonsRequestWithStrictParams, express.Response> {
|
|
6
7
|
constructor(expressServer: CommonsStrictExpressServer, database: DatabaseService, config: CommonsConfig);
|
|
7
8
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { commonsExpressNginxProxyPath } from 'nodecommons-es-express';
|
|
2
1
|
import { isICommonsExpressConfig } from 'nodecommons-es-express';
|
|
3
2
|
import { CommonsRestServer } from 'nodecommons-es-rest';
|
|
4
3
|
import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
|
|
4
|
+
import { commonsNginxProxyPath } from 'nodecommons-es-http';
|
|
5
5
|
import { TestApi } from '../apis/test.api';
|
|
6
6
|
import { StatisticsApi } from '../apis/statistics.api';
|
|
7
7
|
import { CrawlApi } from '../apis/crawl.api';
|
|
@@ -16,7 +16,7 @@ export class RestServer extends CommonsRestServer {
|
|
|
16
16
|
const expressConfig = config.getObject('express');
|
|
17
17
|
if (!isICommonsExpressConfig(expressConfig))
|
|
18
18
|
throw new Error('Express config is not valid');
|
|
19
|
-
const path =
|
|
19
|
+
const path = commonsNginxProxyPath(expressConfig.path);
|
|
20
20
|
commonsOutputDoing('Installing REST Test API');
|
|
21
21
|
new TestApi(this, path);
|
|
22
22
|
commonsOutputSuccess();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rest.server.js","sourceRoot":"","sources":["../../src/servers/rest.server.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rest.server.js","sourceRoot":"","sources":["../../src/servers/rest.server.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAI3D,MAAM,OAAO,UAAW,SAAQ,iBAAoE;IACnG,YACE,aAAyC,EACzC,QAAyB,EACzB,MAAqB;QAEtB,KAAK,CAAC,aAAa,CAAC,CAAC;QAErB,MAAM,aAAa,GAAoB,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAW,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAE/D,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;QACrD,IAAI,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACxC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACnC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,6BAA6B,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACrC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;QACjD,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACpC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;QACvD,IAAI,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC1C,oBAAoB,EAAE,CAAC;IACxB,CAAC;CAED"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { TStatusTallies } from 'hydra-crawler-ts-assets';
|
|
2
2
|
import { TFetch } from 'hydra-crawler-ts-assets';
|
|
3
3
|
import { TOutcome } from 'hydra-crawler-ts-assets';
|
|
4
|
-
import {
|
|
5
|
-
export declare class SocketIoServer extends
|
|
4
|
+
import { CommonsAppSocketIoServer } from 'nodecommons-es-app-socket-io';
|
|
5
|
+
export declare class SocketIoServer extends CommonsAppSocketIoServer {
|
|
6
6
|
fetching(fetch: TFetch): Promise<void>;
|
|
7
7
|
outcome(outcome: TOutcome): Promise<void>;
|
|
8
8
|
statusTallies(tallies: TStatusTallies): Promise<void>;
|
|
@@ -7,8 +7,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
-
import {
|
|
11
|
-
export class SocketIoServer extends
|
|
10
|
+
import { CommonsAppSocketIoServer } from 'nodecommons-es-app-socket-io';
|
|
11
|
+
export class SocketIoServer extends CommonsAppSocketIoServer {
|
|
12
12
|
fetching(fetch) {
|
|
13
13
|
const _super = Object.create(null, {
|
|
14
14
|
broadcast: { get: () => super.broadcast }
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-io.server.js","sourceRoot":"","sources":["../../src/servers/socket-io.server.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"socket-io.server.js","sourceRoot":"","sources":["../../src/servers/socket-io.server.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAExE,MAAM,OAAO,cAAe,SAAQ,wBAAwB;IAC9C,QAAQ,CAAC,KAAa;;;;;YAClC,MAAM,OAAM,SAAS,YAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC1C,CAAC;KAAA;IAEY,OAAO,CAAC,OAAiB;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;KAAA;IAEY,aAAa,CAAC,OAAuB;;;;;YACjD,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;KAAA;IAEY,WAAW,CAAC,KAAa;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;KAAA;IAEY,aAAa,CAAC,KAAa;;;;;YACvC,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;KAAA;IAEY,SAAS,CAAC,GAAW;;;;;YACjC,MAAM,OAAM,SAAS,YAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;KAAA;CACD"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import mongodb from 'mongodb';
|
|
2
2
|
import { IUrlErrors } from 'hydra-crawler-ts-assets';
|
|
3
3
|
import { IImage } from 'hydra-crawler-ts-assets';
|
|
4
4
|
import { IUrl } from 'hydra-crawler-ts-assets';
|
|
@@ -11,11 +11,11 @@ import { EComparator } from 'hydra-crawler-ts-assets';
|
|
|
11
11
|
import { CommonsMongodbService } from 'nodecommons-es-database-mongodb';
|
|
12
12
|
import { TParserCtor } from '../types/tparser-ctor';
|
|
13
13
|
import { EAvailableStrategy } from '../enums/eavailable-strategy';
|
|
14
|
-
export
|
|
14
|
+
export type TMongoIdRow = {
|
|
15
15
|
_id: string;
|
|
16
16
|
};
|
|
17
17
|
export declare function isTMongoIdRow(test: unknown): test is TMongoIdRow;
|
|
18
|
-
export
|
|
18
|
+
export type TMongoIdTallyRow = {
|
|
19
19
|
_id: string;
|
|
20
20
|
tally: number;
|
|
21
21
|
};
|
|
@@ -25,13 +25,13 @@ export declare class DatabaseService extends CommonsMongodbService {
|
|
|
25
25
|
private urls;
|
|
26
26
|
private links;
|
|
27
27
|
private archiveds;
|
|
28
|
-
getDomains(): Collection<TDomain>;
|
|
29
|
-
getUrls(): Collection<IUrl>;
|
|
30
|
-
getLinks(): Collection<TLink>;
|
|
31
|
-
getArchiveds(): Collection<IUrl>;
|
|
28
|
+
getDomains(): mongodb.Collection<TDomain>;
|
|
29
|
+
getUrls(): mongodb.Collection<IUrl>;
|
|
30
|
+
getLinks(): mongodb.Collection<TLink>;
|
|
31
|
+
getArchiveds(): mongodb.Collection<IUrl>;
|
|
32
32
|
init(): Promise<void>;
|
|
33
33
|
initParser(ctor: TParserCtor): Promise<void>;
|
|
34
|
-
getRawDatabase(): Db;
|
|
34
|
+
getRawDatabase(): mongodb.Db;
|
|
35
35
|
wipe(): Promise<void>;
|
|
36
36
|
resetActive(): Promise<void>;
|
|
37
37
|
domain(domain: string, ip: string): Promise<boolean>;
|
|
@@ -7,7 +7,15 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
7
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
8
|
});
|
|
9
9
|
};
|
|
10
|
+
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
11
|
+
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
12
|
+
var m = o[Symbol.asyncIterator], i;
|
|
13
|
+
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
14
|
+
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
15
|
+
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
16
|
+
};
|
|
10
17
|
import { URL } from 'url';
|
|
18
|
+
import mongodb from 'mongodb';
|
|
11
19
|
import { commonsObjectStripNulls, commonsStringRegexLike, commonsTypeAttemptNumber, commonsTypeHasPropertyNumber, commonsTypeHasPropertyString, commonsTypeHasPropertyTArray } from 'tscommons-es-core';
|
|
12
20
|
import { isIUrl } from 'hydra-crawler-ts-assets';
|
|
13
21
|
import { isTPhpError } from 'hydra-crawler-ts-assets';
|
|
@@ -574,38 +582,78 @@ export class DatabaseService extends CommonsMongodbService {
|
|
|
574
582
|
});
|
|
575
583
|
}
|
|
576
584
|
insertArchiveBatch(documents) {
|
|
585
|
+
var _a, documents_1, documents_1_1;
|
|
586
|
+
var _b, e_1, _c, _d;
|
|
577
587
|
return __awaiter(this, void 0, void 0, function* () {
|
|
578
588
|
const bulkInsert = this.getArchiveds().initializeUnorderedBulkOp();
|
|
579
589
|
const insertIds = [];
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
590
|
+
try {
|
|
591
|
+
for (_a = true, documents_1 = __asyncValues(documents); documents_1_1 = yield documents_1.next(), _b = documents_1_1.done, !_b;) {
|
|
592
|
+
_d = documents_1_1.value;
|
|
593
|
+
_a = false;
|
|
594
|
+
try {
|
|
595
|
+
const document = _d;
|
|
596
|
+
const typecast = document;
|
|
597
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
598
|
+
const id = typecast._id;
|
|
599
|
+
// Insert without raising an error for duplicates
|
|
600
|
+
bulkInsert
|
|
601
|
+
.find({ _id: id })
|
|
602
|
+
.upsert()
|
|
603
|
+
.replaceOne(document);
|
|
604
|
+
insertIds.push(id);
|
|
605
|
+
}
|
|
606
|
+
finally {
|
|
607
|
+
_a = true;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
612
|
+
finally {
|
|
613
|
+
try {
|
|
614
|
+
if (!_a && !_b && (_c = documents_1.return)) yield _c.call(documents_1);
|
|
615
|
+
}
|
|
616
|
+
finally { if (e_1) throw e_1.error; }
|
|
617
|
+
}
|
|
618
|
+
if (insertIds.length > 0)
|
|
592
619
|
yield bulkInsert.execute();
|
|
593
620
|
return insertIds;
|
|
594
621
|
});
|
|
595
622
|
}
|
|
596
623
|
deleteUrlsBatch(documents) {
|
|
624
|
+
var _a, documents_2, documents_2_1;
|
|
625
|
+
var _b, e_2, _c, _d;
|
|
597
626
|
return __awaiter(this, void 0, void 0, function* () {
|
|
598
627
|
// NB, this presumes that the links to and from this have already been removed by the cleaner
|
|
599
628
|
const bulkDelete = this.getUrls().initializeUnorderedBulkOp();
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
629
|
+
let tally = 0;
|
|
630
|
+
try {
|
|
631
|
+
for (_a = true, documents_2 = __asyncValues(documents); documents_2_1 = yield documents_2.next(), _b = documents_2_1.done, !_b;) {
|
|
632
|
+
_d = documents_2_1.value;
|
|
633
|
+
_a = false;
|
|
634
|
+
try {
|
|
635
|
+
const document = _d;
|
|
636
|
+
const typecast = document;
|
|
637
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
638
|
+
const id = typecast._id;
|
|
639
|
+
bulkDelete
|
|
640
|
+
.find({ _id: id })
|
|
641
|
+
.deleteOne();
|
|
642
|
+
tally++;
|
|
643
|
+
}
|
|
644
|
+
finally {
|
|
645
|
+
_a = true;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
650
|
+
finally {
|
|
651
|
+
try {
|
|
652
|
+
if (!_a && !_b && (_c = documents_2.return)) yield _c.call(documents_2);
|
|
653
|
+
}
|
|
654
|
+
finally { if (e_2) throw e_2.error; }
|
|
655
|
+
}
|
|
656
|
+
if (tally > 0)
|
|
609
657
|
yield bulkDelete.execute();
|
|
610
658
|
});
|
|
611
659
|
}
|
|
@@ -615,7 +663,8 @@ export class DatabaseService extends CommonsMongodbService {
|
|
|
615
663
|
while (true) {
|
|
616
664
|
commonsOutputDoing(`Inserting up to ${batchSize} urls entries into archiveds`);
|
|
617
665
|
const existings = this.getUrls().find({ status: EStatus.ARCHIVED }).limit(batchSize);
|
|
618
|
-
const done = yield this.insertArchiveBatch(existings)
|
|
666
|
+
const done = (yield this.insertArchiveBatch(existings))
|
|
667
|
+
.map((id) => new mongodb.ObjectId(id));
|
|
619
668
|
yield existings.close();
|
|
620
669
|
total += done.length;
|
|
621
670
|
commonsOutputResult(`${done.length}, ${total} total`);
|
|
@@ -696,25 +745,45 @@ export class DatabaseService extends CommonsMongodbService {
|
|
|
696
745
|
});
|
|
697
746
|
}
|
|
698
747
|
purgeArchiveDuplicates(url) {
|
|
748
|
+
var _a, e_3, _b, _c;
|
|
699
749
|
return __awaiter(this, void 0, void 0, function* () {
|
|
700
750
|
const archiveds = this.getArchiveds().find({
|
|
701
751
|
url: url
|
|
702
752
|
}, {});
|
|
703
753
|
const bulkDelete = this.getArchiveds().initializeUnorderedBulkOp();
|
|
704
754
|
let first = true;
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
755
|
+
let tally = 0;
|
|
756
|
+
try {
|
|
757
|
+
for (var _d = true, archiveds_1 = __asyncValues(archiveds), archiveds_1_1; archiveds_1_1 = yield archiveds_1.next(), _a = archiveds_1_1.done, !_a;) {
|
|
758
|
+
_c = archiveds_1_1.value;
|
|
759
|
+
_d = false;
|
|
760
|
+
try {
|
|
761
|
+
const document = _c;
|
|
762
|
+
if (first) {
|
|
763
|
+
first = false;
|
|
764
|
+
continue;
|
|
765
|
+
}
|
|
766
|
+
const typecast = document;
|
|
767
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
768
|
+
const id = typecast._id;
|
|
769
|
+
bulkDelete
|
|
770
|
+
.find({ _id: id })
|
|
771
|
+
.deleteOne();
|
|
772
|
+
tally++;
|
|
773
|
+
}
|
|
774
|
+
finally {
|
|
775
|
+
_d = true;
|
|
776
|
+
}
|
|
709
777
|
}
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
778
|
+
}
|
|
779
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
780
|
+
finally {
|
|
781
|
+
try {
|
|
782
|
+
if (!_d && !_a && (_b = archiveds_1.return)) yield _b.call(archiveds_1);
|
|
783
|
+
}
|
|
784
|
+
finally { if (e_3) throw e_3.error; }
|
|
785
|
+
}
|
|
786
|
+
if (tally > 0)
|
|
718
787
|
yield bulkDelete.execute();
|
|
719
788
|
yield archiveds.close();
|
|
720
789
|
});
|