hydra-crawler 3.0.9 → 3.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/classes/expiry.mjs +7 -2
  2. package/dist/classes/expiry.mjs.map +1 -1
  3. package/dist/helpers/matcher.mjs +10 -5
  4. package/dist/helpers/matcher.mjs.map +1 -1
  5. package/package.json +1 -1
package/dist/classes/expiry.mjs CHANGED
@@ -45,8 +45,13 @@ export class Expiry {
45
45
  let best;
46
46
  let bestScore;
47
47
  for (const expiry of this.expiries) {
48
- if (!Matcher.matches(whatwg, expiry.match))
49
- continue;
48
+ try {
49
+ if (!Matcher.matches(whatwg, expiry.match))
50
+ continue;
51
+ }
52
+ catch (_e) {
53
+ // continue;
54
+ }
50
55
  const score = Expiry.getScore(expiry);
51
56
  if (bestScore === undefined || score > bestScore) {
52
57
  best = expiry;
package/dist/classes/expiry.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"expiry.mjs","sourceRoot":"","sources":["../../src/classes/expiry.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,4BAA4B,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE3G,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAEjD,OAAO,EAAW,SAAS,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAE1E,MAAM,OAAO,MAAM;IACX,MAAM,CAAC,YAAY,CAAC,IAAY;QACtC,MAAM,IAAI,GAAY,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAEtE,MAAM,QAAQ,GAA0B,IAAI;aACzC,GAAG,CAAC,CAAC,KAAc,EAAqB,EAAE;YAC1C,IAAI,CAAC;gBACJ,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,OAAO,SAAS,CAAC;YAClB,CAAC;QACF,CAAC,CAAC,CAAC;QAEL,IAAI,CAAC,mBAAmB,CAAU,QAAQ,EAAE,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAExG,OAAO,QAAQ,CAAC;IACjB,CAAC;IAEO,MAAM,CAAC,QAAQ,CAAC,MAAe;QACtC,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC1D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAO,CAAC,MAAM,CAAC;QAC7C,CAAC;QACD,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;YAC5D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAS,CAAC,MAAM,CAAC;QAC/C,CAAC;QACD,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;YAC5D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAS,CAAC,MAAM,CAAC;QAC/C,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;IAEO,QAAQ,GAAc,EAAE,CAAC;IAE1B,GAAG,CAAC,QAAmB;QAC7B,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;IACF,CAAC;IAEM,aAAa,CAAC,GAAW;QAC/B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,IAAuB,CAAC;QAC5B,IAAI,SAA2B,CAAC;QAEhC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;gBAAE,SAAS;YAErD,MAAM,KAAK,GAAW,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAE9C,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBAClD,IAAI,GAAG,MAAM,CAAC;gBACd,SAAS,GAAG,KAAK,CAAC;YACnB,CAAC;QACF,CAAC;QAED,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAEtE,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;CACD"}
1
+ {"version":3,"file":"expiry.mjs","sourceRoot":"","sources":["../../src/classes/expiry.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,4BAA4B,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE3G,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAEjD,OAAO,EAAW,SAAS,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAE1E,MAAM,OAAO,MAAM;IACX,MAAM,CAAC,YAAY,CAAC,IAAY;QACtC,MAAM,IAAI,GAAY,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAEtE,MAAM,QAAQ,GAA0B,IAAI;aACzC,GAAG,CAAC,CAAC,KAAc,EAAqB,EAAE;YAC1C,IAAI,CAAC;gBACJ,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,OAAO,SAAS,CAAC;YAClB,CAAC;QACF,CAAC,CAAC,CAAC;QAEL,IAAI,CAAC,mBAAmB,CAAU,QAAQ,EAAE,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAExG,OAAO,QAAQ,CAAC;IACjB,CAAC;IAEO,MAAM,CAAC,QAAQ,CAAC,MAAe;QACtC,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC1D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAO,CAAC,MAAM,CAAC;QAC7C,CAAC;QACD,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;YAC5D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAS,CAAC,MAAM,CAAC;QAC/C,CAAC;QACD,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;YAC5D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAS,CAAC,MAAM,CAAC;QAC/C,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;IAEO,QAAQ,GAAc,EAAE,CAAC;IAE1B,GAAG,CAAC,QAAmB;QAC7B,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;IACF,CAAC;IAEM,aAAa,CAAC,GAAW;QAC/B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,IAAuB,CAAC;QAC5B,IAAI,SAA2B,CAAC;QAEhC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC;gBACJ,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;oBAAE,SAAS;YACtD,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,YAAY;YACb,CAAC;YAED,MAAM,KAAK,GAAW,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAE9C,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBAClD,IAAI,GAAG,MAAM,CAAC;gBACd,SAAS,GAAG,KAAK,CAAC;YACnB,CAAC;QACF,CAAC;QAED,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAEtE,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;CACD"}
package/dist/helpers/matcher.mjs CHANGED
@@ -8,12 +8,17 @@ export class Matcher {
8
8
  return true;
9
9
  }
10
10
  static isAnyMatch(url, list) {
11
- const whatwg = new URL(url);
12
- for (const comparators of list) {
13
- if (Matcher.matches(whatwg, comparators))
14
- return true;
11
+ try {
12
+ const whatwg = new URL(url);
13
+ for (const comparators of list) {
14
+ if (Matcher.matches(whatwg, comparators))
15
+ return true;
16
+ }
17
+ return false;
18
+ }
19
+ catch (_e) {
20
+ return false;
15
21
  }
16
- return false;
17
22
  }
18
23
  }
19
24
  //# sourceMappingURL=matcher.mjs.map
package/dist/helpers/matcher.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"matcher.mjs","sourceRoot":"","sources":["../../src/helpers/matcher.mts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,OAAO;IACrB,MAAM,CAAC,OAAO,CAAC,GAAQ,EAAE,IAAY;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACrC,MAAM,KAAK,GAAW,IAAI,MAAM,CAAE,IAAgC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAE,GAA0C,CAAC,GAAG,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;QACjF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,UAAU,CAAC,GAAW,EAAE,IAAc;QACnD,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,WAAW,IAAI,IAAI,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC;gBAAE,OAAO,IAAI,CAAC;QACvD,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}
1
+ {"version":3,"file":"matcher.mjs","sourceRoot":"","sources":["../../src/helpers/matcher.mts"],"names":[],"mappings":"AAIA,MAAM,OAAgB,OAAO;IACrB,MAAM,CAAC,OAAO,CAAC,GAAQ,EAAE,IAAY;QAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACrC,MAAM,KAAK,GAAW,IAAI,MAAM,CAAE,IAAgC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAE,GAA0C,CAAC,GAAG,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;QACjF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,UAAU,CAAC,GAAW,EAAE,IAAc;QACnD,IAAI,CAAC;YACJ,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjC,KAAK,MAAM,WAAW,IAAI,IAAI,EAAE,CAAC;gBAChC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC;oBAAE,OAAO,IAAI,CAAC;YACvD,CAAC;YAED,OAAO,KAAK,CAAC;QACd,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;CACD"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hydra-crawler",
3
- "version": "3.0.9",
3
+ "version": "3.0.10",
4
4
  "description": "Node.js Hydra web crawler",
5
5
  "author": "Pete Morris",
6
6
  "license": "ISC",