hydra-crawler 3.2.5 → 3.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,51 +12,37 @@ export class Cleaner {
12
12
  async detectOrphans(detectingCaption, clauses, matcher, actionCaption, action, hardLimit) {
13
13
  if (!this.databaseService)
14
14
  throw new Error('Database service has not been set yet');
15
- let tally = 0;
16
15
  let found = 0;
17
16
  const orphanUrls = [];
18
17
  { // scope
19
18
  const log = commonsLogDoing('cleaner', detectingCaption);
19
+ const urls = this.databaseService.getUrls()
20
+ .aggregate([
21
+ { $match: { $and: [
22
+ clauses,
23
+ { $or: [
24
+ { orphan: null },
25
+ { orphan: false }
26
+ ] }
27
+ ] } },
28
+ { $lookup: { from: 'links', localField: 'url', foreignField: 'outgoing', as: 'links' } },
29
+ { $match: { 'links.0': { $exists: false } } },
30
+ { $project: { links: false } }
31
+ ]);
20
32
  while (true) {
21
- const urls = this.databaseService.getUrls()
22
- .find({
23
- $and: [
24
- clauses,
25
- { $or: [
26
- { orphan: null },
27
- { orphan: false }
28
- ] }
29
- ]
30
- }, {})
31
- .sort({ _id: 1 })
32
- .skip(tally);
33
- try {
34
- while (true) {
35
- tally++;
36
- if ((tally % 100) === 0)
37
- log.progress(`${tally} urls, ${found} orphans`);
38
- if (hardLimit && tally >= hardLimit)
39
- break;
40
- const row = await urls.next();
41
- if (row === null)
42
- break;
43
- if (!matcher(row))
44
- continue;
45
- const incoming = await this.databaseService.getLinks().findOne({ outgoing: row.url }, { limit: 1 });
46
- if (incoming)
47
- continue;
48
- orphanUrls.push(row.url);
49
- found++;
50
- }
33
+ if ((found % 100) === 0)
34
+ log.progress(`${found} orphans`);
35
+ if (hardLimit && found >= hardLimit)
51
36
  break;
52
- }
53
- catch (err) {
54
- if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
55
- throw err;
56
- commonsLogDebug('Code 43 encountered. Continuing');
57
- }
37
+ const row = await urls.next();
38
+ if (row === null)
39
+ break;
40
+ if (!matcher(row))
41
+ continue;
42
+ orphanUrls.push(row.url);
43
+ found++;
58
44
  }
59
- log.result(`${tally} urls, ${found} orphans`);
45
+ log.result(`${found} orphans`);
60
46
  }
61
47
  { // scope
62
48
  const log = commonsLogDoing('cleaner', actionCaption);
@@ -70,28 +56,28 @@ export class Cleaner {
70
56
  log.success();
71
57
  }
72
58
  }
73
- async detectAndDeleteOrphans(detectingCaption, clauses, matcher, hardLimit) {
59
+ detectAndDeleteOrphans(detectingCaption, clauses, matcher, hardLimit) {
74
60
  return this.detectOrphans(detectingCaption, clauses, matcher, 'Deleting orphan urls', async (url) => {
75
61
  await this.databaseService.getUrls().deleteOne({ url: url });
76
62
  }, hardLimit);
77
63
  }
78
- async detectAndFlagOrphans(detectingCaption, clauses, matcher, hardLimit) {
64
+ detectAndFlagOrphans(detectingCaption, clauses, matcher, hardLimit) {
79
65
  return this.detectOrphans(detectingCaption, clauses, matcher, 'Flagging urls as orphans', async (url) => {
80
66
  await this.databaseService.getUrls().updateOne({ url: url }, {
81
67
  $set: { orphan: true }
82
68
  });
83
69
  }, hardLimit);
84
70
  }
85
- async detectAndDeleteStatusOrphans(statuses, hardLimit) {
86
- return this.detectAndDeleteOrphans(`Detecting deletable ${statuses.join(', ')} orphan urls`, statuses, (_url) => true, hardLimit);
71
+ detectAndDeleteStatusOrphans(statuses, hardLimit) {
72
+ return this.detectAndDeleteOrphans(`Detecting deletable ${statuses.join(', ')} orphan urls`, { status: { $in: statuses } }, (_url) => true, hardLimit);
87
73
  }
88
- async detectAndFlagStatusOrphans(statuses, hardLimit) {
89
- return this.detectAndFlagOrphans(`Detecting orphans for ${statuses.join(', ')} urls`, statuses, (_url) => true, hardLimit);
74
+ detectAndFlagStatusOrphans(statuses, hardLimit) {
75
+ return this.detectAndFlagOrphans(`Detecting orphans for ${statuses.join(', ')} urls`, { status: { $in: statuses } }, (_url) => true, hardLimit);
90
76
  }
91
- async detectNonAllowlistOrphans(statuses, hardLimit) {
77
+ detectNonAllowlistOrphans(statuses, hardLimit) {
92
78
  return this.detectAndFlagOrphans(`Detecting non-allowlist ${statuses.join(', ')} orphan urls`, { status: { $in: statuses } }, (url) => !this.lists.match(EList.ALLOW, url.url), hardLimit);
93
79
  }
94
- async detectStatusCodeOrphans(gt, lt, hardLimit) {
80
+ detectStatusCodeOrphans(gt, lt, hardLimit) {
95
81
  return this.detectAndFlagOrphans(`Detecting DONE status code between ${gt}<${lt} orphan urls`, { $and: [
96
82
  { status: EStatus.DONE },
97
83
  { statusCode: { $gt: gt } },
@@ -1 +1 @@
1
- {"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGnF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAE7E,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,OAAO;wBACP,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAEzE,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;4BAAE,SAAS;wBAE5B,MAAM,QAAQ,GAAe,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,CACxE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,EACrB,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;wBACF,IAAI,QAAQ;4BAAE,SAAS;wBAEvB,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACzB,KAAK,EAAE,CAAC;oBACT,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QAC/C,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC9B,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBAEvD,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAClC,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,sBAAsB,EACtB,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACvB,CACF,CAAC;QACH,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,4BAA4B,CACxC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,sBAAsB,CAChC,uBAAuB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EACxD,QAAQ,EACR,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACtC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EACnD,QAAQ,EACR,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CACrC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAC5D,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,EAAU,EACV,EAAU,EACV,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,sCAAsC,EAAE,IAAI,EAAE,cAAc,EAC5D,EAAE,IAAI,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;gBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;gBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;aAC5B,EAAE,EACH,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,SAAkB;QAC9C,MAAM,IAAI,CAAC,4BAA4B,CAAC,CAAE,OAAO,CAAC,OAAO,CAAE,EAAE,SAAS,CAAC,CAAC;QAExE,MAAM,IAAI,CAAC,0BAA0B,CAAC;YACpC,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;SACnB,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,yBAAyB,CAAC;YACnC,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QAExD,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAC;YAC7F,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAE7C,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBACpE,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAChF,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;oBACZ,oBAAoB,EAAE,IAAI;oBAC1B,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,QAAQ,EAAE,IAAI;oBACd,cAAc,EAAE,IAAI;oBACpB,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAElF,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,IAAI,CAAC;gBACJ,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAE7C,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,CAAC,CAAC;YACnG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAE3E,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;iBACxD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;CAED"}
1
+ {"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGnF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,aAAa,CACzB,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,aAAqB,EACrB,MAAsC,EACtC,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAoC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzE,SAAS,CAAO;gBACf,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE;4BAChB,OAAO;4BACP,EAAE,GAAG,EAAE;oCACL,EAAE,MAAM,EAAE,IAAI,EAAE;oCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;iCAClB,EAAE;yBACJ,EAAE,EAAE;gBACL,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE;gBACxF,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC7C,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;aAC/B,CAAC,CAAC;YAEL,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;gBAE1D,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;oBAAE,MAAM;gBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACzC,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAE5B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACzB,KAAK,EAAE,CAAC;YACT,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QAChC,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC9B,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBAEvD,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,sBAAsB,CAC5B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,sBAAsB,EACtB,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,gBAAwB,EACxB,OAAwB,EACxB,OAA+B,EAC/B,SAAkB;QAEnB,OAAO,IAAI,CAAC,aAAa,CACvB,gBAAgB,EAChB,OAAO,EACP,OAAO,EACP,0BAA0B,EAC1B,KAAK,EAAE,GAAW,EAAiB,EAAE;YACpC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACvB,CACF,CAAC;QACH,CAAC,EACD,SAAS,CACV,CAAC;IACH,CAAC;IAEO,4BAA4B,CAClC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,sBAAsB,CAChC,uBAAuB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EACxD,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEO,0BAA0B,CAChC,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EACnD,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEO,yBAAyB,CAC/B,QAAmB,EACnB,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAC5D,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC7B,CAAC,GAAS,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,EAC/D,SAAS,CACV,CAAC;IACH,CAAC;IAEO,uBAAuB,CAC7B,EAAU,EACV,EAAU,EACV,SAAkB;QAEnB,OAAO,IAAI,CAAC,oBAAoB,CAC9B,sCAAsC,EAAE,IAAI,EAAE,cAAc,EAC5D,EAAE,IAAI,EAAE;gBACN,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;gBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;gBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;aAC5B,EAAE,EACH,CAAC,IAAU,EAAW,EAAE,CAAC,IAAI,EAC7B,SAAS,CACV,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,SAAkB;QAC9C,MAAM,IAAI,CAAC,4BAA4B,CAAC,CAAE,OAAO,CAAC,OAAO,CAAE,EAAE,SAAS,CAAC,CAAC;QAExE,MAAM,IAAI,CAAC,0BAA0B,CAAC;YACpC,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;SACnB,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,yBAAyB,CAAC;YACnC,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QAExD,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAC;YAC7F,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAE7C,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBACpE,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAChF,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;oBACZ,oBAAoB,EAAE,IAAI;oBAC1B,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,QAAQ,EAAE,IAAI;oBACd,cAAc,EAAE,IAAI;oBACpB,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,IAAI;oBACX,MAAM,EAAE,IAAI;iBACb;aACF,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAElF,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,IAAI,CAAC;gBACJ,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAE7C,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,CAAC,CAAC;YACnG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAE3E,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAE,EAAE;iBACxD,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;CAED"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hydra-crawler",
3
- "version": "3.2.5",
3
+ "version": "3.2.7",
4
4
  "description": "Node.js Hydra web crawler",
5
5
  "author": "Pete Morris",
6
6
  "license": "ISC",