hydra-crawler 1.4.6 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/autocomplete.api.d.ts +7 -0
- package/dist/apis/autocomplete.api.js +15 -9
- package/dist/apis/autocomplete.api.js.map +1 -0
- package/dist/apis/bugs.api.d.ts +7 -0
- package/dist/apis/bugs.api.js +21 -15
- package/dist/apis/bugs.api.js.map +1 -0
- package/dist/apis/crawl.api.d.ts +7 -0
- package/dist/apis/crawl.api.js +15 -9
- package/dist/apis/crawl.api.js.map +1 -0
- package/dist/apis/domains.api.d.ts +7 -0
- package/dist/apis/domains.api.js +24 -19
- package/dist/apis/domains.api.js.map +1 -0
- package/dist/apis/images.api.d.ts +7 -0
- package/dist/apis/images.api.js +20 -14
- package/dist/apis/images.api.js.map +1 -0
- package/dist/apis/statistics.api.d.ts +8 -0
- package/dist/apis/statistics.api.js +27 -20
- package/dist/apis/statistics.api.js.map +1 -0
- package/dist/apis/test.api.d.ts +5 -0
- package/dist/apis/test.api.js +15 -9
- package/dist/apis/test.api.js.map +1 -0
- package/dist/apis/urls.api.d.ts +7 -0
- package/dist/apis/urls.api.js +21 -15
- package/dist/apis/urls.api.js.map +1 -0
- package/dist/apps/cleanup.app.d.ts +19 -0
- package/dist/apps/cleanup.app.js +118 -100
- package/dist/apps/cleanup.app.js.map +1 -0
- package/dist/apps/cross-populate-export.app.d.ts +12 -0
- package/dist/apps/cross-populate-export.app.js +60 -47
- package/dist/apps/cross-populate-export.app.js.map +1 -0
- package/dist/apps/cross-populate-import.app.d.ts +12 -0
- package/dist/apps/cross-populate-import.app.js +64 -51
- package/dist/apps/cross-populate-import.app.js.map +1 -0
- package/dist/apps/denylist.app.d.ts +17 -0
- package/dist/apps/denylist.app.js +115 -98
- package/dist/apps/denylist.app.js.map +1 -0
- package/dist/apps/expire.app.d.ts +19 -0
- package/dist/apps/expire.app.js +44 -31
- package/dist/apps/expire.app.js.map +1 -0
- package/dist/apps/extract-text.app.d.ts +8 -0
- package/dist/apps/extract-text.app.js +43 -35
- package/dist/apps/extract-text.app.js.map +1 -0
- package/dist/apps/hydra.app.d.ts +34 -0
- package/dist/apps/hydra.app.js +150 -137
- package/dist/apps/hydra.app.js.map +1 -0
- package/dist/apps/import.app.d.ts +11 -0
- package/dist/apps/import.app.js +44 -32
- package/dist/apps/import.app.js.map +1 -0
- package/dist/apps/internal-hydra-common.app.d.ts +28 -0
- package/dist/apps/internal-hydra-common.app.js +5 -11
- package/dist/apps/internal-hydra-common.app.js.map +1 -0
- package/dist/apps/query.app.d.ts +20 -0
- package/dist/apps/query.app.js +63 -49
- package/dist/apps/query.app.js.map +1 -0
- package/dist/apps/reattempt.app.d.ts +17 -0
- package/dist/apps/reattempt.app.js +66 -53
- package/dist/apps/reattempt.app.js.map +1 -0
- package/dist/apps/requeue-domain.app.d.ts +13 -0
- package/dist/apps/requeue-domain.app.js +50 -37
- package/dist/apps/requeue-domain.app.js.map +1 -0
- package/dist/apps/seed.app.d.ts +15 -0
- package/dist/apps/seed.app.js +53 -40
- package/dist/apps/seed.app.js.map +1 -0
- package/dist/apps/startup.app.d.ts +11 -0
- package/dist/apps/startup.app.js +51 -38
- package/dist/apps/startup.app.js.map +1 -0
- package/dist/apps/unarchive.app.d.ts +15 -0
- package/dist/apps/unarchive.app.js +67 -54
- package/dist/apps/unarchive.app.js.map +1 -0
- package/dist/classes/cleaner.d.ts +12 -0
- package/dist/classes/cleaner.js +227 -207
- package/dist/classes/cleaner.js.map +1 -0
- package/dist/classes/crawler.d.ts +34 -0
- package/dist/classes/crawler.js +248 -241
- package/dist/classes/crawler.js.map +1 -0
- package/dist/classes/dns.d.ts +3 -0
- package/dist/classes/dns.js +10 -13
- package/dist/classes/dns.js.map +1 -0
- package/dist/classes/expirer.d.ts +10 -0
- package/dist/classes/expirer.js +107 -94
- package/dist/classes/expirer.js.map +1 -0
- package/dist/classes/expiry.d.ts +8 -0
- package/dist/classes/expiry.js +16 -19
- package/dist/classes/expiry.js.map +1 -0
- package/dist/classes/lists.d.ts +9 -0
- package/dist/classes/lists.js +13 -18
- package/dist/classes/lists.js.map +1 -0
- package/dist/classes/robot.d.ts +15 -0
- package/dist/classes/robot.js +40 -30
- package/dist/classes/robot.js.map +1 -0
- package/dist/classes/tracker.d.ts +25 -0
- package/dist/classes/tracker.js +82 -64
- package/dist/classes/tracker.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +72 -65
- package/dist/cli.js.map +1 -0
- package/dist/enums/eavailable-strategy.d.ts +4 -0
- package/dist/enums/eavailable-strategy.js +3 -5
- package/dist/enums/eavailable-strategy.js.map +1 -0
- package/dist/enums/elist.d.ts +7 -0
- package/dist/enums/elist.js +7 -11
- package/dist/enums/elist.js.map +1 -0
- package/dist/enums/eserver.d.ts +8 -0
- package/dist/enums/eserver.js +3 -5
- package/dist/enums/eserver.js.map +1 -0
- package/dist/enums/ex-powered-by.d.ts +6 -0
- package/dist/enums/ex-powered-by.js +3 -5
- package/dist/enums/ex-powered-by.js.map +1 -0
- package/dist/helpers/matcher.d.ts +5 -0
- package/dist/helpers/matcher.js +2 -5
- package/dist/helpers/matcher.js.map +1 -0
- package/dist/helpers/random.d.ts +4 -0
- package/dist/helpers/random.js +2 -5
- package/dist/helpers/random.js.map +1 -0
- package/dist/helpers/utf-decoder.d.ts +4 -0
- package/dist/helpers/utf-decoder.js +3 -6
- package/dist/helpers/utf-decoder.js.map +1 -0
- package/dist/interfaces/iexpiry.d.ts +7 -0
- package/dist/interfaces/iexpiry.js +9 -13
- package/dist/interfaces/iexpiry.js.map +1 -0
- package/dist/interfaces/imatch.d.ts +6 -0
- package/dist/interfaces/imatch.js +6 -9
- package/dist/interfaces/imatch.js.map +1 -0
- package/dist/interfaces/iparser-config.d.ts +4 -0
- package/dist/interfaces/iparser-config.js +4 -7
- package/dist/interfaces/iparser-config.js.map +1 -0
- package/dist/interfaces/iparser.d.ts +8 -0
- package/dist/interfaces/iparser.js +2 -2
- package/dist/interfaces/iparser.js.map +1 -0
- package/dist/interfaces/irequest-outcome.d.ts +11 -0
- package/dist/interfaces/irequest-outcome.js +2 -2
- package/dist/interfaces/irequest-outcome.js.map +1 -0
- package/dist/interfaces/iserver.d.ts +4 -0
- package/dist/interfaces/iserver.js +2 -2
- package/dist/interfaces/iserver.js.map +1 -0
- package/dist/parsers/accessibility-metrics.parser.d.ts +11 -0
- package/dist/parsers/accessibility-metrics.parser.js +34 -26
- package/dist/parsers/accessibility-metrics.parser.js.map +1 -0
- package/dist/parsers/asp-error.parser.d.ts +12 -0
- package/dist/parsers/asp-error.parser.js +36 -28
- package/dist/parsers/asp-error.parser.js.map +1 -0
- package/dist/parsers/bad-words.parser.d.ts +10 -0
- package/dist/parsers/bad-words.parser.js +21 -13
- package/dist/parsers/bad-words.parser.js.map +1 -0
- package/dist/parsers/complex-english.parser.d.ts +15 -0
- package/dist/parsers/complex-english.parser.js +33 -25
- package/dist/parsers/complex-english.parser.js.map +1 -0
- package/dist/parsers/data.parser.d.ts +14 -0
- package/dist/parsers/data.parser.js +12 -16
- package/dist/parsers/data.parser.js.map +1 -0
- package/dist/parsers/dictionary.parser.d.ts +19 -0
- package/dist/parsers/dictionary.parser.js +47 -39
- package/dist/parsers/dictionary.parser.js.map +1 -0
- package/dist/parsers/html.parser.d.ts +13 -0
- package/dist/parsers/html.parser.js +4 -8
- package/dist/parsers/html.parser.js.map +1 -0
- package/dist/parsers/hyperlinks.parser.d.ts +20 -0
- package/dist/parsers/hyperlinks.parser.js +82 -77
- package/dist/parsers/hyperlinks.parser.js.map +1 -0
- package/dist/parsers/image-tags.parser.d.ts +20 -0
- package/dist/parsers/image-tags.parser.js +38 -34
- package/dist/parsers/image-tags.parser.js.map +1 -0
- package/dist/parsers/jpeg.parser.d.ts +11 -0
- package/dist/parsers/jpeg.parser.js +28 -20
- package/dist/parsers/jpeg.parser.js.map +1 -0
- package/dist/parsers/paragraphs.parser.d.ts +13 -0
- package/dist/parsers/paragraphs.parser.js +33 -40
- package/dist/parsers/paragraphs.parser.js.map +1 -0
- package/dist/parsers/parser.d.ts +19 -0
- package/dist/parsers/parser.js +30 -17
- package/dist/parsers/parser.js.map +1 -0
- package/dist/parsers/php-error.parser.d.ts +12 -0
- package/dist/parsers/php-error.parser.js +42 -34
- package/dist/parsers/php-error.parser.js.map +1 -0
- package/dist/parsers/phrase.parser.d.ts +8 -0
- package/dist/parsers/phrase.parser.js +16 -11
- package/dist/parsers/phrase.parser.js.map +1 -0
- package/dist/parsers/regex.parser.d.ts +10 -0
- package/dist/parsers/regex.parser.js +30 -22
- package/dist/parsers/regex.parser.js.map +1 -0
- package/dist/parsers/server.parser.d.ts +12 -0
- package/dist/parsers/server.parser.js +66 -56
- package/dist/parsers/server.parser.js.map +1 -0
- package/dist/parsers/spelling.parser.d.ts +10 -0
- package/dist/parsers/spelling.parser.js +21 -13
- package/dist/parsers/spelling.parser.js.map +1 -0
- package/dist/parsers/string.parser.d.ts +8 -0
- package/dist/parsers/string.parser.js +5 -8
- package/dist/parsers/string.parser.js.map +1 -0
- package/dist/parsers/text.parser.d.ts +8 -0
- package/dist/parsers/text.parser.js +24 -18
- package/dist/parsers/text.parser.js.map +1 -0
- package/dist/parsers/words.parser.d.ts +11 -0
- package/dist/parsers/words.parser.js +32 -28
- package/dist/parsers/words.parser.js.map +1 -0
- package/dist/queries/complex-english.query.d.ts +2 -0
- package/dist/queries/complex-english.query.js +37 -38
- package/dist/queries/complex-english.query.js.map +1 -0
- package/dist/queries/flash-content.query.d.ts +2 -0
- package/dist/queries/flash-content.query.js +39 -30
- package/dist/queries/flash-content.query.js.map +1 -0
- package/dist/queries/linking-to-domains.query.d.ts +2 -0
- package/dist/queries/linking-to-domains.query.js +35 -27
- package/dist/queries/linking-to-domains.query.js.map +1 -0
- package/dist/queries/readability-score.query.d.ts +2 -0
- package/dist/queries/readability-score.query.js +21 -13
- package/dist/queries/readability-score.query.js.map +1 -0
- package/dist/servers/crawl.server.d.ts +35 -0
- package/dist/servers/crawl.server.js +133 -121
- package/dist/servers/crawl.server.js.map +1 -0
- package/dist/servers/express.server.d.ts +8 -0
- package/dist/servers/express.server.js +7 -10
- package/dist/servers/express.server.js.map +1 -0
- package/dist/servers/maintenance.server.d.ts +22 -0
- package/dist/servers/maintenance.server.js +42 -36
- package/dist/servers/maintenance.server.js.map +1 -0
- package/dist/servers/rest.server.d.ts +7 -0
- package/dist/servers/rest.server.js +40 -51
- package/dist/servers/rest.server.js.map +1 -0
- package/dist/servers/socket-io.server.d.ts +12 -0
- package/dist/servers/socket-io.server.js +48 -15
- package/dist/servers/socket-io.server.js.map +1 -0
- package/dist/services/database.service.d.ts +68 -0
- package/dist/services/database.service.js +528 -462
- package/dist/services/database.service.js.map +1 -0
- package/dist/types/tcrawl-config.d.ts +14 -0
- package/dist/types/tcrawl-config.js +14 -17
- package/dist/types/tcrawl-config.js.map +1 -0
- package/dist/types/thydra-config.d.ts +4 -0
- package/dist/types/thydra-config.js +4 -7
- package/dist/types/thydra-config.js.map +1 -0
- package/dist/types/tparser-ctor.d.ts +7 -0
- package/dist/types/tparser-ctor.js +2 -2
- package/dist/types/tparser-ctor.js.map +1 -0
- package/dist/types/tquery.d.ts +7 -0
- package/dist/types/tquery.js +2 -2
- package/dist/types/tquery.js.map +1 -0
- package/dist/types/trobots-config.d.ts +4 -0
- package/dist/types/trobots-config.js +4 -7
- package/dist/types/trobots-config.js.map +1 -0
- package/package.json +41 -29
- package/angular/10-es2015.bacd4ae5dd7913ce55f0.js +0 -1
- package/angular/10-es5.bacd4ae5dd7913ce55f0.js +0 -1
- package/angular/11-es2015.0f031dcf752d1e8eda6b.js +0 -1
- package/angular/11-es5.0f031dcf752d1e8eda6b.js +0 -1
- package/angular/3rdpartylicenses.txt +0 -1127
- package/angular/5-es2015.951498ca9c1bc74e57bf.js +0 -1
- package/angular/5-es5.951498ca9c1bc74e57bf.js +0 -1
- package/angular/6-es2015.65f680261a3506b88381.js +0 -1
- package/angular/6-es5.65f680261a3506b88381.js +0 -1
- package/angular/7-es2015.625197f3af1dbf3e805d.js +0 -1
- package/angular/7-es5.625197f3af1dbf3e805d.js +0 -1
- package/angular/8-es2015.55518901987a5b834309.js +0 -1
- package/angular/8-es5.55518901987a5b834309.js +0 -1
- package/angular/9-es2015.6cc9bde262564e7836f2.js +0 -1
- package/angular/9-es5.6cc9bde262564e7836f2.js +0 -1
- package/angular/Roboto-Black.41ed1105a6ebb8ffe34e.woff2 +0 -0
- package/angular/Roboto-Black.937491dfcbe64ca9a9f1.woff +0 -0
- package/angular/Roboto-BlackItalic.2e1ee657996854c6f427.woff +0 -0
- package/angular/Roboto-BlackItalic.50ca4c51ebc27e7e7d2f.woff2 +0 -0
- package/angular/Roboto-Bold.73288d91c325e82a5b92.woff +0 -0
- package/angular/Roboto-Bold.92fbd4e93cf0a5dbebaa.woff2 +0 -0
- package/angular/Roboto-BoldItalic.5f600d98a73d800ae575.woff2 +0 -0
- package/angular/Roboto-BoldItalic.6d89acbd21d7e3fbecb2.woff +0 -0
- package/angular/Roboto-Light.c27d89ac77468ae18f28.woff2 +0 -0
- package/angular/Roboto-Light.d923dfafc0c5183b59aa.woff +0 -0
- package/angular/Roboto-LightItalic.506274c7228cf81cae4d.woff2 +0 -0
- package/angular/Roboto-LightItalic.d4b8c137518d9d92bb28.woff +0 -0
- package/angular/Roboto-Medium.092c6130df8fd2199888.woff +0 -0
- package/angular/Roboto-Medium.1d3bced88509b0838984.woff2 +0 -0
- package/angular/Roboto-MediumItalic.18ff1628c628080166c1.woff +0 -0
- package/angular/Roboto-MediumItalic.d620b8f53f75966fe42e.woff2 +0 -0
- package/angular/Roboto-Regular.64cfb66c866ea50cad47.woff2 +0 -0
- package/angular/Roboto-Regular.e02e9d6ff5547f7e9962.woff +0 -0
- package/angular/Roboto-RegularItalic.4dd2af1e8df532f41db8.woff2 +0 -0
- package/angular/Roboto-RegularItalic.5ea38fff9eebef99c5df.woff +0 -0
- package/angular/Roboto-Thin.dbd56bd3357dc3617fe5.woff2 +0 -0
- package/angular/Roboto-Thin.e7f7c82374bd0ebef14b.woff +0 -0
- package/angular/Roboto-ThinItalic.5dd9349c940073834e9a.woff +0 -0
- package/angular/Roboto-ThinItalic.a8cef84f735ef887abdc.woff2 +0 -0
- package/angular/assets/config/app-config.json +0 -16
- package/angular/assets/images/splashbg.jpg +0 -0
- package/angular/assets/web-app-commons/fonts/material-icons/MaterialDesignIcons-Community-2.7.94.woff +0 -0
- package/angular/assets/web-app-commons/fonts/material-icons/MaterialDesignIcons-Community-2.7.94.woff2 +0 -0
- package/angular/assets/web-app-commons/fonts/material-icons/material-design-icons-community.css +0 -11293
- package/angular/favicon.ico +0 -0
- package/angular/flUhRq6tzZclQEJ-Vdg-IuiaDsNa.f2a0933406f783065152.woff +0 -0
- package/angular/flUhRq6tzZclQEJ-Vdg-IuiaDsNc.6467d9a24f234e8e8e07.woff2 +0 -0
- package/angular/index.html +0 -16
- package/angular/main-es2015.3a582572476c7f292e52.js +0 -1
- package/angular/main-es5.3a582572476c7f292e52.js +0 -1
- package/angular/polyfills-es2015.7df68534018bc2f6cb09.js +0 -1
- package/angular/polyfills-es5.e79468f406fae2989221.js +0 -1
- package/angular/runtime-es2015.6d2cff76cdb2790d3308.js +0 -1
- package/angular/runtime-es5.6d2cff76cdb2790d3308.js +0 -1
- package/angular/styles.c5c6c2534225b85c4ff0.css +0 -1
- package/config/bad-words.json +0 -1
- package/config/complex-english.json +0 -400
- package/config/hydra-auth.json +0 -8
- package/config/hydra-crawler.json +0 -84
- package/config/list-allow.json +0 -171
- package/config/list-deny.json +0 -248
- package/config/list-expiry.json +0 -7
- package/config/schedule.json +0 -25
- package/config/spelling.json +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utf-decoder.js","sourceRoot":"","sources":["../../src/helpers/utf-decoder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,OAAgB,UAAU;IACxB,MAAM,CAAC,UAAU,CAAC,QAAgB;QACxC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAE7C,IAAI;YACH,MAAM,MAAM,GAAW,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAE5B,MAAM,KAAK,GAAW,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACjD,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;YACnC,QAAQ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBAC9B,KAAK,UAAU;oBACd,OAAO,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACrC,KAAK,UAAU;oBACd,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAC3D;oBACC,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;aAC5B;SACD;QAAC,OAAO,EAAE,EAAE;YACZ,kDAAkD;YAClD,IAAI;gBACH,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;aAC3B;YAAC,OAAO,GAAG,EAAE;gBACb,SAAS;aACT;SACD;QAED,OAAO,SAAS,CAAC;IAClB,CAAC;CACD"}
|
|
@@ -1,19 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const tscommons_core_1 = require("tscommons-core");
|
|
5
|
-
const imatch_1 = require("./imatch");
|
|
6
|
-
function isIExpiry(test) {
|
|
1
|
+
import { commonsTypeHasPropertyNumber, commonsTypeHasPropertyObject, commonsTypeHasPropertyString } from 'tscommons-es-core';
|
|
2
|
+
import { isIMatch } from './imatch';
|
|
3
|
+
export function isIExpiry(test) {
|
|
7
4
|
// Can't use hasPropertyT as we need to pass a true parameter
|
|
8
|
-
if (!
|
|
5
|
+
if (!commonsTypeHasPropertyObject(test, 'match'))
|
|
9
6
|
return false;
|
|
10
|
-
if (!
|
|
7
|
+
if (!isIMatch(test.match, true))
|
|
11
8
|
return false;
|
|
12
|
-
if (!
|
|
9
|
+
if (!commonsTypeHasPropertyNumber(test, 'expiry'))
|
|
13
10
|
return false;
|
|
14
11
|
return true;
|
|
15
12
|
}
|
|
16
|
-
exports.isIExpiry = isIExpiry;
|
|
17
13
|
var EInterval;
|
|
18
14
|
(function (EInterval) {
|
|
19
15
|
EInterval["SECOND"] = "second";
|
|
@@ -58,12 +54,12 @@ function calculateExpiry(expiry) {
|
|
|
58
54
|
throw new Error(`Remaining expiry: ${expiry.trim()}`);
|
|
59
55
|
return total;
|
|
60
56
|
}
|
|
61
|
-
function toIExpiry(data) {
|
|
62
|
-
if (
|
|
57
|
+
export function toIExpiry(data) {
|
|
58
|
+
if (commonsTypeHasPropertyString(data, 'expiry')) {
|
|
63
59
|
data['expiry'] = calculateExpiry(data['expiry']);
|
|
64
60
|
}
|
|
65
61
|
if (!isIExpiry(data))
|
|
66
62
|
throw new Error('Object is not an instance of IExpiry');
|
|
67
63
|
return data;
|
|
68
64
|
}
|
|
69
|
-
|
|
65
|
+
//# sourceMappingURL=iexpiry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iexpiry.js","sourceRoot":"","sources":["../../src/interfaces/iexpiry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,4BAA4B,EAAE,4BAA4B,EAAe,MAAM,mBAAmB,CAAC;AAE1I,OAAO,EAAU,QAAQ,EAAE,MAAM,UAAU,CAAC;AAO5C,MAAM,UAAU,SAAS,CAAC,IAAa;IACtC,6DAA6D;IAC7D,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,IAAK,SAQJ;AARD,WAAK,SAAS;IACZ,8BAAiB,CAAA;IACjB,8BAAiB,CAAA;IACjB,0BAAa,CAAA;IACb,wBAAW,CAAA;IACX,0BAAa,CAAA;IACb,4BAAe,CAAA;IACf,0BAAa,CAAA;AACf,CAAC,EARI,SAAS,KAAT,SAAS,QAQb;AAED,MAAM,eAAe,GAAW,CAAC,CAAC;AAClC,MAAM,eAAe,GAAW,eAAe,GAAG,EAAE,CAAC;AACrD,MAAM,aAAa,GAAW,eAAe,GAAG,EAAE,CAAC;AACnD,MAAM,YAAY,GAAW,aAAa,GAAG,EAAE,CAAC;AAChD,MAAM,aAAa,GAAW,YAAY,GAAG,CAAC,CAAC;AAC/C,MAAM,aAAa,GAAW,YAAY,GAAG,MAAM,CAAC;AACpD,MAAM,cAAc,GAAW,aAAa,GAAG,EAAE,CAAC;AAElD,MAAM,cAAc,GAAmC;IACrD,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,eAAe;IACnC,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,eAAe;IACnC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;IAC/B,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,YAAY;IAC7B,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;IAC/B,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,cAAc;IACjC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;CAChC,CAAC;AAEF,SAAS,eAAe,CAAC,MAAc;IACtC,IAAI,KAAK,GAAW,CAAC,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;QAC/C,OAAO,IAAI,EAAE;YACZ,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,4BAA4B,IAAI,mBAAmB,EAAE,GAAG,CAAC,CAAC;YAC3F,MAAM,MAAM,GAAyB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM;YAE3B,IAAI,KAAK,GAAW,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;YAC9B,KAAK,IAAI,KAAK,CAAC;YAEf,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;SACxC;KACD;IAED,IAAI,EAAE,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAEhF,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAa;IACtC,IAAI,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAW,CAAC,CAAC;KAC3D;IAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAE9E,OAAO,IAAI,CAAC;AACb,CAAC"}
|
|
@@ -1,16 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const tscommons_core_1 = require("tscommons-core");
|
|
5
|
-
function isIMatch(test, allowBlank = false) {
|
|
6
|
-
if (!tscommons_core_1.CommonsType.hasPropertyStringOrUndefined(test, 'hostname'))
|
|
1
|
+
import { commonsTypeHasPropertyStringOrUndefined } from 'tscommons-es-core';
|
|
2
|
+
export function isIMatch(test, allowBlank = false) {
|
|
3
|
+
if (!commonsTypeHasPropertyStringOrUndefined(test, 'hostname'))
|
|
7
4
|
return false;
|
|
8
|
-
if (!
|
|
5
|
+
if (!commonsTypeHasPropertyStringOrUndefined(test, 'pathname'))
|
|
9
6
|
return false;
|
|
10
|
-
if (!
|
|
7
|
+
if (!commonsTypeHasPropertyStringOrUndefined(test, 'search'))
|
|
11
8
|
return false;
|
|
12
9
|
if (!test.hostname && !test.pathname && !test.search && !allowBlank)
|
|
13
10
|
return false;
|
|
14
11
|
return true;
|
|
15
12
|
}
|
|
16
|
-
|
|
13
|
+
//# sourceMappingURL=imatch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"imatch.js","sourceRoot":"","sources":["../../src/interfaces/imatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uCAAuC,EAAE,MAAM,mBAAmB,CAAC;AAQ5E,MAAM,UAAU,QAAQ,CAAC,IAAa,EAAE,aAAsB,KAAK;IAClE,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3E,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAElF,OAAO,IAAI,CAAC;AACb,CAAC"}
|
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const tscommons_core_1 = require("tscommons-core");
|
|
5
|
-
function isIParserConfig(test) {
|
|
6
|
-
if (!tscommons_core_1.CommonsType.hasPropertyBoolean(test, 'enabled'))
|
|
1
|
+
import { commonsTypeHasPropertyBoolean } from 'tscommons-es-core';
|
|
2
|
+
export function isIParserConfig(test) {
|
|
3
|
+
if (!commonsTypeHasPropertyBoolean(test, 'enabled'))
|
|
7
4
|
return false;
|
|
8
5
|
return true;
|
|
9
6
|
}
|
|
10
|
-
|
|
7
|
+
//# sourceMappingURL=iparser-config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iparser-config.js","sourceRoot":"","sources":["../../src/interfaces/iparser-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AAQlE,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,6BAA6B,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAElE,OAAO,IAAI,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { DatabaseService } from '../services/database.service';
|
|
2
|
+
export interface IParser {
|
|
3
|
+
isEnabled(): boolean;
|
|
4
|
+
supports(contentType: string, isAllow: boolean): boolean;
|
|
5
|
+
init(database: DatabaseService): Promise<void>;
|
|
6
|
+
parse(database: DatabaseService): Promise<void>;
|
|
7
|
+
links(): Promise<string[]>;
|
|
8
|
+
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
export {};
|
|
2
|
+
//# sourceMappingURL=iparser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iparser.js","sourceRoot":"","sources":["../../src/interfaces/iparser.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
import { IncomingHttpHeaders } from 'http';
|
|
4
|
+
export interface IRequestOutcome {
|
|
5
|
+
latency: number;
|
|
6
|
+
ip: string;
|
|
7
|
+
statusCode: number;
|
|
8
|
+
headers: IncomingHttpHeaders;
|
|
9
|
+
data?: Buffer;
|
|
10
|
+
exceeded: boolean;
|
|
11
|
+
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
export {};
|
|
2
|
+
//# sourceMappingURL=irequest-outcome.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"irequest-outcome.js","sourceRoot":"","sources":["../../src/interfaces/irequest-outcome.ts"],"names":[],"mappings":""}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
export {};
|
|
2
|
+
//# sourceMappingURL=iserver.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iserver.js","sourceRoot":"","sources":["../../src/interfaces/iserver.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { TKeyObject } from 'tscommons-es-core';
|
|
2
|
+
import { DatabaseService } from '../services/database.service';
|
|
3
|
+
import { IParserConfig } from '../interfaces/iparser-config';
|
|
4
|
+
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
5
|
+
import { TextParser } from './text.parser';
|
|
6
|
+
import { IDataConfig } from './data.parser';
|
|
7
|
+
export declare class AccessibilityMetricsParser extends TextParser<IDataConfig> {
|
|
8
|
+
private url?;
|
|
9
|
+
constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
|
|
10
|
+
protected parseText(database: DatabaseService, text: string): Promise<void>;
|
|
11
|
+
}
|
|
@@ -1,32 +1,40 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { commonsStringAutomatedReadabilityIndex, commonsStringFleschKincaidReadingEase, commonsStringSplitSentences, commonsStringSplitWords } from 'tscommons-es-core';
|
|
11
|
+
import { commonsOutputDebug } from 'nodecommons-es-cli';
|
|
12
|
+
import { TextParser } from './text.parser';
|
|
13
|
+
export class AccessibilityMetricsParser extends TextParser {
|
|
8
14
|
constructor(url, outcome, config) {
|
|
9
15
|
super(outcome, config, 'accessibilityMetrics');
|
|
10
16
|
this.url = url;
|
|
11
17
|
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
18
|
+
parseText(database, text) {
|
|
19
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
20
|
+
if (!this.url)
|
|
21
|
+
return;
|
|
22
|
+
if (text.length === 0) {
|
|
23
|
+
yield database.unsetData(this.url, 'accessibilityMetrics');
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
const paragraphs = text.split('\n');
|
|
27
|
+
const all = paragraphs.join('. ');
|
|
28
|
+
const metrics = {
|
|
29
|
+
paragraphs: paragraphs.length,
|
|
30
|
+
sentences: commonsStringSplitSentences(all).length,
|
|
31
|
+
words: commonsStringSplitWords(all).length,
|
|
32
|
+
fkre: commonsStringFleschKincaidReadingEase(all),
|
|
33
|
+
ari: commonsStringAutomatedReadabilityIndex(all)
|
|
34
|
+
};
|
|
35
|
+
commonsOutputDebug(`Accessibility metrics are: ${JSON.stringify(metrics)}`);
|
|
36
|
+
yield database.setData(this.url, 'accessibilityMetrics', metrics);
|
|
37
|
+
});
|
|
30
38
|
}
|
|
31
39
|
}
|
|
32
|
-
|
|
40
|
+
//# sourceMappingURL=accessibility-metrics.parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"accessibility-metrics.parser.js","sourceRoot":"","sources":["../../src/parsers/accessibility-metrics.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,sCAAsC,EAAE,qCAAqC,EAAE,2BAA2B,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAEpL,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAW3C,MAAM,OAAO,0BAA2B,SAAQ,UAAuB;IACtE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,sBAAsB,CACvB,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;IAStB,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,CAAC,CAAC;gBAC3D,OAAO;aACP;YAED,MAAM,UAAU,GAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,GAAG,GAAW,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE1C,MAAM,OAAO,GAA0B;gBACrC,UAAU,EAAE,UAAU,CAAC,MAAM;gBAC7B,SAAS,EAAE,2BAA2B,CAAC,GAAG,CAAC,CAAC,MAAM;gBAClD,KAAK,EAAE,uBAAuB,CAAC,GAAG,CAAC,CAAC,MAAM;gBAC1C,IAAI,EAAE,qCAAqC,CAAC,GAAG,CAAC;gBAChD,GAAG,EAAE,sCAAsC,CAAC,GAAG,CAAC;aACjD,CAAC;YAEF,kBAAkB,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC5E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,EAAE,OAAO,CAAC,CAAC;QACnE,CAAC;KAAA;CACD"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { TKeyObject } from 'tscommons-es-core';
|
|
2
|
+
import { DatabaseService } from '../services/database.service';
|
|
3
|
+
import { IParserConfig } from '../interfaces/iparser-config';
|
|
4
|
+
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
5
|
+
import { StringParser } from './string.parser';
|
|
6
|
+
import { IDataConfig } from './data.parser';
|
|
7
|
+
export declare class AspErrorParser extends StringParser<IDataConfig> {
|
|
8
|
+
private url?;
|
|
9
|
+
constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
|
|
10
|
+
parse(database: DatabaseService): Promise<void>;
|
|
11
|
+
supports(_contentType: string, isAllow: boolean): boolean;
|
|
12
|
+
}
|
|
@@ -1,40 +1,48 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { StringParser } from './string.parser';
|
|
11
|
+
const ASP_ERROR = 'Server Error in \'[^\']+\' Application[.]';
|
|
12
|
+
const ASP_ERROR_PARSE = 'Server Error in \'([^\']+)\' Application[.].*<h2>\\s*<i>(.+?)</i>\\s*</h2>.+?<code><pre>(.+?)</pre></code>';
|
|
13
|
+
export class AspErrorParser extends StringParser {
|
|
8
14
|
constructor(url, outcome, config) {
|
|
9
15
|
super(outcome, config, 'aspError');
|
|
10
16
|
this.url = url;
|
|
11
17
|
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
else {
|
|
22
|
-
const detail = new RegExp(ASP_ERROR_PARSE, 'g');
|
|
23
|
-
const result2 = detail.exec(singlified);
|
|
24
|
-
if (result2 === null) {
|
|
25
|
-
await database.setData(this.url, 'aspErrors', true);
|
|
18
|
+
parse(database) {
|
|
19
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
20
|
+
if (!this.url || !this.stringData)
|
|
21
|
+
return;
|
|
22
|
+
const singlified = this.stringData.replace(/[\t\n\r]/g, ' ');
|
|
23
|
+
const pattern = new RegExp(ASP_ERROR, 'g');
|
|
24
|
+
const result = pattern.exec(singlified);
|
|
25
|
+
if (result === null) {
|
|
26
|
+
yield database.unsetData(this.url, 'aspErrors');
|
|
26
27
|
}
|
|
27
28
|
else {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
}
|
|
29
|
+
const detail = new RegExp(ASP_ERROR_PARSE, 'g');
|
|
30
|
+
const result2 = detail.exec(singlified);
|
|
31
|
+
if (result2 === null) {
|
|
32
|
+
yield database.setData(this.url, 'aspErrors', true);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
yield database.setData(this.url, 'aspErrors', {
|
|
36
|
+
application: result2[1],
|
|
37
|
+
message: result2[2],
|
|
38
|
+
stack: result2[3]
|
|
39
|
+
});
|
|
40
|
+
}
|
|
33
41
|
}
|
|
34
|
-
}
|
|
42
|
+
});
|
|
35
43
|
}
|
|
36
44
|
supports(_contentType, isAllow) {
|
|
37
45
|
return isAllow; // everything allowlisted
|
|
38
46
|
}
|
|
39
47
|
}
|
|
40
|
-
|
|
48
|
+
//# sourceMappingURL=asp-error.parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"asp-error.parser.js","sourceRoot":"","sources":["../../src/parsers/asp-error.parser.ts"],"names":[],"mappings":";;;;;;;;;AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,SAAS,GAAW,2CAA2C,CAAC;AACtE,MAAM,eAAe,GAAW,4GAA4G,CAAC;AAE7I,MAAM,OAAO,cAAe,SAAQ,YAAyB;IAC5D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU;gBAAE,OAAO;YAE1C,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;YAErE,MAAM,OAAO,GAAW,IAAI,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAEnD,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9D,IAAI,MAAM,KAAK,IAAI,EAAE;gBACpB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;aAChD;iBAAM;gBACN,MAAM,MAAM,GAAW,IAAI,MAAM,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;gBACxD,MAAM,OAAO,GAAyB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE9D,IAAI,OAAO,KAAK,IAAI,EAAE;oBACrB,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;iBACpD;qBAAM;oBACN,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE;wBAC5C,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;wBACvB,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;wBACnB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;qBAClB,CAAC,CAAC;iBACH;aACD;QACF,CAAC;KAAA;IAEM,QAAQ,CAAC,YAAoB,EAAE,OAAgB;QACrD,OAAO,OAAO,CAAC,CAAC,yBAAyB;IAC1C,CAAC;CACD"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { TKeyObject } from 'tscommons-es-core';
|
|
2
|
+
import { DatabaseService } from '../services/database.service';
|
|
3
|
+
import { IParserConfig } from '../interfaces/iparser-config';
|
|
4
|
+
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
5
|
+
import { DictionaryParser, IDictionaryConfig } from './dictionary.parser';
|
|
6
|
+
export declare class BadWordsParser extends DictionaryParser<IDictionaryConfig> {
|
|
7
|
+
private url?;
|
|
8
|
+
constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
|
|
9
|
+
protected parseMatches(database: DatabaseService, matches: string[], _nonMatches: string[]): Promise<void>;
|
|
10
|
+
}
|
|
@@ -1,19 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { DictionaryParser } from './dictionary.parser';
|
|
11
|
+
export class BadWordsParser extends DictionaryParser {
|
|
6
12
|
constructor(url, outcome, config) {
|
|
7
13
|
super(outcome, config, 'badWords');
|
|
8
14
|
this.url = url;
|
|
9
15
|
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
parseMatches(database, matches, _nonMatches) {
|
|
17
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
18
|
+
if (!this.url)
|
|
19
|
+
return;
|
|
20
|
+
if (matches.length > 0)
|
|
21
|
+
yield database.setData(this.url, 'badWords', matches);
|
|
22
|
+
else
|
|
23
|
+
yield database.unsetData(this.url, 'badWords');
|
|
24
|
+
});
|
|
17
25
|
}
|
|
18
26
|
}
|
|
19
|
-
|
|
27
|
+
//# sourceMappingURL=bad-words.parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bad-words.parser.js","sourceRoot":"","sources":["../../src/parsers/bad-words.parser.ts"],"names":[],"mappings":";;;;;;;;;AAOA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,qBAAqB,CAAC;AAE1E,MAAM,OAAO,cAAe,SAAQ,gBAAmC;IACtE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,YAAY,CAAC,QAAyB,EAAE,OAAiB,EAAE,WAAqB;;YAC/F,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;;gBACzE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACrD,CAAC;KAAA;CACD"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { TKeyObject } from 'tscommons-es-core';
|
|
2
|
+
import { DatabaseService } from '../services/database.service';
|
|
3
|
+
import { IParserConfig } from '../interfaces/iparser-config';
|
|
4
|
+
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
5
|
+
import { IDictionaryConfig } from './dictionary.parser';
|
|
6
|
+
import { PhraseParser } from './phrase.parser';
|
|
7
|
+
export declare class ComplexEnglishParser extends PhraseParser<IDictionaryConfig> {
|
|
8
|
+
private url?;
|
|
9
|
+
static loadKeyObject(config: IDictionaryConfig): TKeyObject<string[]>;
|
|
10
|
+
static loadDictionary(config: IDictionaryConfig): string[];
|
|
11
|
+
private dictionary;
|
|
12
|
+
constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
|
|
13
|
+
getDictionary(): TKeyObject<string[]> | undefined;
|
|
14
|
+
protected parsePhrases(database: DatabaseService, matches: string[]): Promise<void>;
|
|
15
|
+
}
|
|
@@ -1,20 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { commonsTypeHasPropertyT, commonsTypeIsStringArrayKeyObject } from 'tscommons-es-core';
|
|
11
|
+
import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
|
|
12
|
+
import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-es-file';
|
|
13
|
+
import { isIDictionaryConfig } from './dictionary.parser';
|
|
14
|
+
import { PhraseParser } from './phrase.parser';
|
|
9
15
|
let CACHE;
|
|
10
|
-
class ComplexEnglishParser extends
|
|
16
|
+
export class ComplexEnglishParser extends PhraseParser {
|
|
11
17
|
constructor(url, outcome, config) {
|
|
12
18
|
super(outcome, config, 'complexEnglish');
|
|
13
19
|
this.url = url;
|
|
14
20
|
if (!config)
|
|
15
21
|
return;
|
|
16
|
-
if (!
|
|
17
|
-
|
|
22
|
+
if (!commonsTypeHasPropertyT(config, 'complexEnglish', isIDictionaryConfig)) {
|
|
23
|
+
commonsOutputError('Invalid config for ComplexEnglishParser');
|
|
18
24
|
}
|
|
19
25
|
const dictionaryConfig = config['complexEnglish'];
|
|
20
26
|
if (!CACHE)
|
|
@@ -24,11 +30,11 @@ class ComplexEnglishParser extends phrase_parser_1.PhraseParser {
|
|
|
24
30
|
this.setPhrases(keys, dictionaryConfig.caseSensitive || false);
|
|
25
31
|
}
|
|
26
32
|
static loadKeyObject(config) {
|
|
27
|
-
const filename =
|
|
28
|
-
const json =
|
|
33
|
+
const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
|
|
34
|
+
const json = commonsFileReadJsonFile(filename);
|
|
29
35
|
if (json === undefined)
|
|
30
36
|
throw new Error('Unable to read dictionary file');
|
|
31
|
-
if (!
|
|
37
|
+
if (!commonsTypeIsStringArrayKeyObject(json))
|
|
32
38
|
throw new Error('Dictionary file is not a JSON array');
|
|
33
39
|
return json;
|
|
34
40
|
}
|
|
@@ -38,16 +44,18 @@ class ComplexEnglishParser extends phrase_parser_1.PhraseParser {
|
|
|
38
44
|
getDictionary() {
|
|
39
45
|
return this.dictionary;
|
|
40
46
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
47
|
+
parsePhrases(database, matches) {
|
|
48
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
49
|
+
if (!this.url)
|
|
50
|
+
return;
|
|
51
|
+
if (matches.length > 0) {
|
|
52
|
+
commonsOutputDebug(`Found ${matches.length} potentially complex phrases`);
|
|
53
|
+
yield database.setData(this.url, 'complexEnglish', matches);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
yield database.unsetData(this.url, 'complexEnglish');
|
|
57
|
+
}
|
|
58
|
+
});
|
|
51
59
|
}
|
|
52
60
|
}
|
|
53
|
-
|
|
61
|
+
//# sourceMappingURL=complex-english.parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"complex-english.parser.js","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,IAAI,KAAqC,CAAC;AAE1C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;IAkBxE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAJhC,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,EAAE;YAC/F,kBAAkB,CAAC,yCAAyC,CAAC,CAAC;SAC9D;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,gBAAgB,CAAsB,CAAC;QAE1F,IAAI,CAAC,KAAK;YAAE,KAAK,GAAG,oBAAoB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB,MAAM,IAAI,GAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC;IAChE,CAAC;IApCM,MAAM,CAAC,aAAa,CAAC,MAAyB;QACpD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,IAAI,GAAsB,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAClE,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAE1E,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAErG,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,MAAyB;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;IAyBM,aAAa;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAEe,YAAY,CAAC,QAAyB,EAAE,OAAiB;;YACxE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,kBAAkB,CAAC,SAAS,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAC;gBAC1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;aAC5D;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;aACrD;QACF,CAAC;KAAA;CACD"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { TKeyObject } from 'tscommons-es-core';
|
|
3
|
+
import { IRequestOutcome } from '../interfaces/irequest-outcome';
|
|
4
|
+
import { IParserConfig } from '../interfaces/iparser-config';
|
|
5
|
+
import { Parser } from './parser';
|
|
6
|
+
export interface IDataConfig extends IParserConfig {
|
|
7
|
+
maxDataBytes?: number;
|
|
8
|
+
allowExceeded?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare function isIDataConfig(test: unknown): test is IDataConfig;
|
|
11
|
+
export declare abstract class DataParser<T extends IDataConfig> extends Parser<T> {
|
|
12
|
+
protected data: Buffer | undefined;
|
|
13
|
+
constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
|
|
14
|
+
}
|
|
@@ -1,22 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
const parser_1 = require("./parser");
|
|
8
|
-
function isIDataConfig(test) {
|
|
9
|
-
if (!iparser_config_1.isIParserConfig(test))
|
|
1
|
+
import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-es-core';
|
|
2
|
+
import { commonsOutputError } from 'nodecommons-es-cli';
|
|
3
|
+
import { isIParserConfig } from '../interfaces/iparser-config';
|
|
4
|
+
import { Parser } from './parser';
|
|
5
|
+
export function isIDataConfig(test) {
|
|
6
|
+
if (!isIParserConfig(test))
|
|
10
7
|
return false;
|
|
11
|
-
if (!
|
|
8
|
+
if (!commonsTypeHasPropertyNumberOrUndefined(test, 'maxDataBytes'))
|
|
12
9
|
return false;
|
|
13
|
-
if (!
|
|
10
|
+
if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'allowExceeded'))
|
|
14
11
|
return false;
|
|
15
12
|
return true;
|
|
16
13
|
}
|
|
17
|
-
exports.isIDataConfig = isIDataConfig;
|
|
18
14
|
const MAX_DATA_BYTES_HARD_LIMIT = 1024 * 1024 * 1024; // 1Gb
|
|
19
|
-
class DataParser extends
|
|
15
|
+
export class DataParser extends Parser {
|
|
20
16
|
constructor(outcome, config, configKey) {
|
|
21
17
|
super(outcome, config, configKey);
|
|
22
18
|
if (!outcome || !outcome.data)
|
|
@@ -24,17 +20,17 @@ class DataParser extends parser_1.Parser {
|
|
|
24
20
|
const dataConfig = this.getConfig(isIDataConfig);
|
|
25
21
|
const maxByteSize = (dataConfig && dataConfig.maxDataBytes) ? dataConfig.maxDataBytes : MAX_DATA_BYTES_HARD_LIMIT;
|
|
26
22
|
if (outcome.data.byteLength >= maxByteSize) {
|
|
27
|
-
|
|
23
|
+
commonsOutputError(`Data to be parsed exceeded parser limit of ${maxByteSize}. Skipping parse.`);
|
|
28
24
|
this.data = undefined;
|
|
29
25
|
return;
|
|
30
26
|
}
|
|
31
27
|
if ((!dataConfig || !dataConfig.allowExceeded) && outcome.exceeded) {
|
|
32
28
|
// by default don't parse data which has exceeded Hydra's upper download limit, so will be corrupt. But there is a parser option to force it to do so if desired for some reason
|
|
33
|
-
|
|
29
|
+
commonsOutputError('Data to be parsed exceeded Hydra\'s upper download limit and allowExceed it set. Skipping parse.');
|
|
34
30
|
this.data = undefined;
|
|
35
31
|
return;
|
|
36
32
|
}
|
|
37
33
|
this.data = outcome.data;
|
|
38
34
|
}
|
|
39
35
|
}
|
|
40
|
-
|
|
36
|
+
//# sourceMappingURL=data.parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"data.parser.js","sourceRoot":"","sources":["../../src/parsers/data.parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,uCAAuC,EAAc,MAAM,mBAAmB,CAAC;AAElI,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAGxD,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAE9E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAOlC,MAAM,UAAU,aAAa,CAAC,IAAa;IAC1C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEzC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IACjF,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,yBAAyB,GAAW,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AAEpE,MAAM,OAAgB,UAAkC,SAAQ,MAAS;IAGxE,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAEtC,MAAM,UAAU,GAA0B,IAAI,CAAC,SAAS,CAAc,aAAa,CAAC,CAAC;QACrF,MAAM,WAAW,GAAW,CAAC,UAAU,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,yBAAyB,CAAC;QAC1H,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,WAAW,EAAE;YAC3C,kBAAkB,CAAC,8CAA8C,WAAW,mBAAmB,CAAC,CAAC;YACjG,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;SACP;QAED,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE;YACnE,gLAAgL;YAEhL,kBAAkB,CAAC,kGAAkG,CAAC,CAAC;YACvH,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;SACP;QAED,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,CAAC;CACD"}
|