hydra-crawler 2.2.7 → 2.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
9
9
  };
10
10
  import { commonsTypeHasPropertyNumber } from 'tscommons-es-core';
11
11
  import { EStatus } from 'hydra-crawler-ts-assets';
12
- import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
12
+ import { commonsOutputDebug, commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
13
13
  import { EList } from '../enums/elist';
14
14
  export class Cleaner {
15
15
  constructor(lists, databaseService) {
@@ -46,10 +46,8 @@ export class Cleaner {
46
46
  const row = yield urls.next();
47
47
  if (row === null)
48
48
  break;
49
- const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
50
- .limit(1)
51
- .count();
52
- if (incoming > 0)
49
+ const incoming = yield this.databaseService.getLinks().findOne({ outgoing: row.url }, { limit: 1 });
50
+ if (incoming)
53
51
  continue;
54
52
  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
55
53
  yield this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
@@ -60,6 +58,7 @@ export class Cleaner {
60
58
  catch (err) {
61
59
  if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
62
60
  throw err;
61
+ commonsOutputDebug('Code 43 encountered. Continuing');
63
62
  }
64
63
  }
65
64
  commonsOutputResult(`${tally} urls, ${found} orphans`);
@@ -95,10 +94,8 @@ export class Cleaner {
95
94
  break;
96
95
  if (this.lists.match(EList.ALLOW, row.url))
97
96
  continue;
98
- const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
99
- .limit(1)
100
- .count();
101
- if (incoming > 0)
97
+ const incoming = yield this.databaseService.getLinks().findOne({ outgoing: row.url }, { limit: 1 });
98
+ if (incoming)
102
99
  continue;
103
100
  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
104
101
  yield this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
@@ -109,6 +106,7 @@ export class Cleaner {
109
106
  catch (err) {
110
107
  if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
111
108
  throw err;
109
+ commonsOutputDebug('Code 43 encountered. Continuing');
112
110
  }
113
111
  }
114
112
  commonsOutputResult(`${tally} urls, ${found} orphans`);
@@ -144,10 +142,8 @@ export class Cleaner {
144
142
  const row = yield urls.next();
145
143
  if (row === null)
146
144
  break;
147
- const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url })
148
- .limit(1)
149
- .count();
150
- if (incoming > 0)
145
+ const incoming = yield this.databaseService.getLinks().findOne({ outgoing: row.url }, { limit: 1 });
146
+ if (incoming)
151
147
  continue;
152
148
  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
153
149
  yield this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
@@ -158,6 +154,7 @@ export class Cleaner {
158
154
  catch (err) {
159
155
  if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
160
156
  throw err;
157
+ commonsOutputDebug('Code 43 encountered. Continuing');
161
158
  }
162
159
  }
163
160
  commonsOutputResult(`${tally} urls, ${found} orphans`);
@@ -199,6 +196,7 @@ export class Cleaner {
199
196
  catch (err) {
200
197
  if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
201
198
  throw err;
199
+ commonsOutputDebug('Code 43 encountered. Continuing');
202
200
  }
203
201
  }
204
202
  commonsOutputResult(tally);
@@ -252,13 +250,11 @@ export class Cleaner {
252
250
  if ((tally % 10) === 0)
253
251
  commonsOutputProgress(`${tally} domains, ${found} removed`);
254
252
  const urls = yield this.databaseService.getUrls()
255
- .find({
253
+ .findOne({
256
254
  domain: domain,
257
255
  status: { $ne: EStatus.ARCHIVED }
258
- }, {})
259
- .limit(1)
260
- .count();
261
- if (urls > 0)
256
+ }, { limit: 1 });
257
+ if (urls)
262
258
  continue;
263
259
  yield this.databaseService.getDomains().deleteOne({ domain: domain });
264
260
  found++;
@@ -1 +1 @@
1
- {"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../src/classes/cleaner.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAGjE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAM1H,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,MAAM,OAAO,OAAO;IACnB,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEU,mBAAmB,CAC/B,QAAmB,EACnB,SAAkB;;YAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEnE,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACjE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CACtB;6BACE,KAAK,CAAC,CAAC,CAAC;6BACR,KAAK,EAAE,CAAC;wBACX,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,yBAAyB,CACrC,QAAmB,EACnB,SAAkB;;YAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;4BAAE,SAAS;wBAErD,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACjE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CACtB;6BACE,KAAK,CAAC,CAAC,CAAC;6BACR,KAAK,EAAE,CAAC;wBACX,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,uBAAuB,CACnC,EAAU,EACV,EAAU,EACV,SAAkB;;YAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,sCAAsC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;wBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACjE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CACtB;6BACE,KAAK,CAAC,CAAC,CAAC;6BACR,KAAK,EAAE,CAAC;wBACX,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YAED,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEY,eAAe,CAAC,SAAkB;;YAC9C,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAC7B,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;aACnB,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,IAAI,CAAC,yBAAyB,CAAC;gBACnC,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;aACb,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;YACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;YAExD,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;YACrD,OAAO,IAAI,EAAE;gBACZ,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;qBACnE;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;iBACZ;aACF,CACF,CAAC;YACF,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,iBAAiB;;YAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;YAE1C,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI;gBACH,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE;oBACZ,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;oBAEtD,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;iBACzB;aACD;YAAC,OAAO,GAAG,EAAE;gBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;aAC7E;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;YAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAEpF,MAAM,IAAI,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACtD,IAAI,CAAO;oBACV,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;iBAClC,EAAE,EAAE,CAAC;qBACL,KAAK,CAAC,CAAC,CAAC;qBACR,KAAK,EAAE,CAAC;gBAEX,IAAI,IAAI,GAAG,CAAC;oBAAE,SAAS;gBAEvB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAC3D,CAAC;KAAA;CAED"}
1
+ {"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../src/classes/cleaner.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAGjE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAM9I,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,MAAM,OAAO,OAAO;IACnB,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEU,mBAAmB,CAC/B,QAAmB,EACnB,SAAkB;;YAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEnE,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAe,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,CACxE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,EACrB,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;wBACF,IAAI,QAAQ;4BAAE,SAAS;wBAEvB,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,kBAAkB,CAAC,iCAAiC,CAAC,CAAC;iBACtD;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,yBAAyB,CACrC,QAAmB,EACnB,SAAkB;;YAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;4BAAE,SAAS;wBAErD,MAAM,QAAQ,GAAe,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,CACxE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,EACrB,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;wBACF,IAAI,QAAQ;4BAAE,SAAS;wBAEvB,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,kBAAkB,CAAC,iCAAiC,CAAC,CAAC;iBACtD;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,uBAAuB,CACnC,EAAU,EACV,EAAU,EACV,SAAkB;;YAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,sCAAsC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBAClE,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;wBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;4BAAE,MAAM;wBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAe,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,CACxE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,EACrB,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;wBACF,IAAI,QAAQ;4BAAE,SAAS;wBAEvB,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,kBAAkB,CAAC,iCAAiC,CAAC,CAAC;iBACtD;aACD;YAED,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEY,eAAe,CAAC,SAAkB;;YAC9C,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAC7B,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;aACnB,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,IAAI,CAAC,yBAAyB,CAAC;gBACnC,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;aACb,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;YACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;YAExD,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;YACrD,OAAO,IAAI,EAAE;gBACZ,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;qBACnE;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,kBAAkB,CAAC,iCAAiC,CAAC,CAAC;iBACtD;aACD;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;iBACZ;aACF,CACF,CAAC;YACF,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,iBAAiB;;YAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;YAE1C,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI;gBACH,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE;oBACZ,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;oBAEtD,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;iBACzB;aACD;YAAC,OAAO,GAAG,EAAE;gBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;aAC7E;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;YAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAEpF,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;iBAClC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAC3D,CAAC;KAAA;CAED"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hydra-crawler",
3
- "version": "2.2.7",
3
+ "version": "2.2.9",
4
4
  "description": "Node.js Hydra web crawler",
5
5
  "author": "Pete Morris",
6
6
  "license": "ISC",