@cumulus/ingest 16.0.3-alpha.0 → 16.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/.nycrc.json +7 -0
  2. package/HttpProviderClient.d.ts +14 -2
  3. package/HttpProviderClient.d.ts.map +1 -1
  4. package/HttpProviderClient.js +29 -2
  5. package/HttpProviderClient.js.map +1 -1
  6. package/S3ProviderClient.d.ts +17 -3
  7. package/S3ProviderClient.d.ts.map +1 -1
  8. package/S3ProviderClient.js +21 -4
  9. package/S3ProviderClient.js.map +1 -1
  10. package/consumer.d.ts.map +1 -1
  11. package/consumer.js +15 -3
  12. package/consumer.js.map +1 -1
  13. package/lock.d.ts +27 -24
  14. package/lock.d.ts.map +1 -1
  15. package/lock.js +39 -36
  16. package/lock.js.map +1 -1
  17. package/package.json +15 -14
  18. package/queue.d.ts +7 -6
  19. package/queue.d.ts.map +1 -1
  20. package/queue.js +7 -16
  21. package/queue.js.map +1 -1
  22. package/src/HttpProviderClient.js +36 -2
  23. package/src/S3ProviderClient.ts +31 -4
  24. package/src/consumer.ts +13 -3
  25. package/src/lock.ts +44 -41
  26. package/src/queue.js +9 -27
  27. package/tsconfig.tsbuildinfo +1 -1
package/.nycrc.json ADDED
@@ -0,0 +1,7 @@
1
+ {
2
+ "extends": "../../nyc.config.js",
3
+ "statements": 83.0,
4
+ "functions": 82.0,
5
+ "branches": 72.0,
6
+ "lines": 84.0
7
+ }
package/HttpProviderClient.d.ts CHANGED
@@ -27,7 +27,7 @@ declare class HttpProviderClient {
27
27
  /**
28
28
  * Download a remote file to disk
29
29
  *
30
- * @param {Object} params
30
+ * @param {object} params
31
31
  * @param {string} params.remotePath - the full path to the remote file to be fetched
32
32
  * @param {string} params.localPath - the full local destination file path
33
33
  * @returns {Promise.<string>} - the path that the file was saved to
@@ -39,7 +39,7 @@ declare class HttpProviderClient {
39
39
  /**
40
40
  * Download the remote file to a given s3 location
41
41
  *
42
- * @param {Object} params
42
+ * @param {object} params
43
43
  * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
44
44
  * @param {string} params.destinationBucket - destination s3 bucket of the file
45
45
  * @param {string} params.destinationKey - destination s3 key of the file
@@ -54,6 +54,18 @@ declare class HttpProviderClient {
54
54
  s3uri: string;
55
55
  etag: string;
56
56
  }>;
57
+ /**
58
+ * Upload a file
59
+ *
60
+ * @param {object} params
61
+ * @param {string} params.localPath - the full local file path
62
+ * @param {string} params.uploadPath - the full remote file path for uploading file to
63
+ * @returns {Promise<string>} the uri of the uploaded file
64
+ */
65
+ upload(params: {
66
+ localPath: string;
67
+ uploadPath: string;
68
+ }): Promise<string>;
57
69
  connect(): Promise<void>;
58
70
  end(): Promise<void>;
59
71
  }
package/HttpProviderClient.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"HttpProviderClient.d.ts","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":";AAiCA;IACE,iCAsBC;IArBC,oBAAoC;IACpC,cAAuC;IACvC,UAA+B;IAC/B,UAA+B;IAC/B,qBAAqD;IACrD,eAAoB;IACpB,oBAAmD;IAInD,eAAyC;IAEzC,qBAA0E;IAC1E,sBAAkE;IAGlE,iBAIE;IAGJ,iCA2CC;IAzCG,cAA2D;IAC3D,cAA2D;IA0C/D,wCAUC;IANG,gCAAqE;IAQzE;;;;;OAKG;IACH,WAHW,MAAM,GACJ,cAAc,CAoF1B;IAED;;;;;;;OAOG;IACH;QAJ0B,UAAU,EAAzB,MAAM;QACS,SAAS,EAAxB,MAAM;QACJ,QAAS,MAAM,CAAC,CA8B5B;IAED;;;;;;;;;OASG;IACH;QAN0B,cAAc,EAA7B,MAAM;QACS,iBAAiB,EAAhC,MAAM;QACS,cAAc,EAA7B,MAAM;;eACc,MAAM;cAAQ,MAAM;OAyClD;IAGD,yBAAkB;IAElB,qBAAc;CAEf"}
1
+ {"version":3,"file":"HttpProviderClient.d.ts","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":";AAkCA;IACE,iCAsBC;IArBC,oBAAoC;IACpC,cAAuC;IACvC,UAA+B;IAC/B,UAA+B;IAC/B,qBAAqD;IACrD,eAAoB;IACpB,oBAAmD;IAInD,eAAyC;IAEzC,qBAA0E;IAC1E,sBAAkE;IAGlE,iBAIE;IAGJ,iCA2CC;IAzCG,cAA2D;IAC3D,cAA2D;IA0C/D,wCAUC;IANG,gCAAqE;IAQzE;;;;;OAKG;IACH,WAHW,MAAM,GACJ,cAAc,CAoF1B;IAED;;;;;;;OAOG;IACH;QAJ0B,UAAU,EAAzB,MAAM;QACS,SAAS,EAAxB,MAAM;QACJ,QAAS,MAAM,CAAC,CA8B5B;IAED;;;;;;;;;OASG;IACH;QAN0B,cAAc,EAA7B,MAAM;QACS,iBAAiB,EAAhC,MAAM;QACS,cAAc,EAA7B,MAAM;;eACc,MAAM;cAAQ,MAAM;OAyClD;IAED;;;;;;;OAOG;IACH;QAJ0B,SAAS,EAAxB,MAAM;QACS,UAAU,EAAzB,MAAM;QACJ,QAAQ,MAAM,CAAC,CAyB3B;IAGD,yBAAkB;IAElB,qBAAc;CAEf"}
package/HttpProviderClient.js CHANGED
@@ -8,6 +8,7 @@ const Crawler = require('simplecrawler');
8
8
  const got = require('got');
9
9
  const { CookieJar } = require('tough-cookie');
10
10
  const { promisify } = require('util');
11
+ const stream = require('node:stream');
11
12
  const { buildS3Uri, getTextObject, parseS3Uri, streamS3Upload, } = require('@cumulus/aws-client/S3');
12
13
  const log = require('@cumulus/common/log');
13
14
  const isValidHostname = require('is-valid-hostname');
@@ -172,7 +173,7 @@ class HttpProviderClient {
172
173
  /**
173
174
  * Download a remote file to disk
174
175
  *
175
- * @param {Object} params
176
+ * @param {object} params
176
177
  * @param {string} params.remotePath - the full path to the remote file to be fetched
177
178
  * @param {string} params.localPath - the full local destination file path
178
179
  * @returns {Promise.<string>} - the path that the file was saved to
@@ -206,7 +207,7 @@ class HttpProviderClient {
206
207
  /**
207
208
  * Download the remote file to a given s3 location
208
209
  *
209
- * @param {Object} params
210
+ * @param {object} params
210
211
  * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
211
212
  * @param {string} params.destinationBucket - destination s3 bucket of the file
212
213
  * @param {string} params.destinationKey - destination s3 key of the file
@@ -246,6 +247,32 @@ class HttpProviderClient {
246
247
  log.info('Uploading to s3 is complete (http)', s3uri);
247
248
  return { s3uri, etag };
248
249
  }
250
+ /**
251
+ * Upload a file
252
+ *
253
+ * @param {object} params
254
+ * @param {string} params.localPath - the full local file path
255
+ * @param {string} params.uploadPath - the full remote file path for uploading file to
256
+ * @returns {Promise<string>} the uri of the uploaded file
257
+ */
258
+ async upload(params) {
259
+ const { localPath, uploadPath } = params;
260
+ log.info(params);
261
+ await this.setUpGotOptions();
262
+ await this.downloadTLSCertificate();
263
+ const options = {
264
+ protocol: this.protocol,
265
+ host: this.host,
266
+ port: this.port,
267
+ path: uploadPath,
268
+ method: 'POST',
269
+ };
270
+ const remoteUrl = buildURL(options);
271
+ got.stream.options = options;
272
+ await promisify(pipeline)(fs.createReadStream(localPath), await got.stream.post(remoteUrl), new stream.PassThrough());
273
+ log.info(`Finishing uploading ${localPath} to ${remoteUrl}`);
274
+ return remoteUrl;
275
+ }
249
276
  /* eslint-disable no-empty-function */
250
277
  async connect() { }
251
278
  async end() { }
package/HttpProviderClient.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"HttpProviderClient.js","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACzB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC9B,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AACrC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACvC,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AACzC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;AAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AAC9C,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAEtC,MAAM,EACJ,UAAU,EACV,aAAa,EACb,UAAU,EACV,cAAc,GACf,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AACtC,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC3C,MAAM,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;AACrD,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;AACzD,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAE1C,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEtD,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,EAAE;IAC5B,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO;IAEhD,MAAM,IAAI,SAAS,CAAC,gDAAgD,IAAI,EAAE,CAAC,CAAC;AAC9E,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEnE,MAAM,kBAAkB;IACtB,YAAY,cAAc;QACxB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC,eAAe,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC,cAAc,CAAC;QACpD,IAAI,cAAc,CAAC,QAAQ,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE;YACvD,MAAM,IAAI,cAAc,CAAC,2EAA2E,CAAC,CAAC;SACvG;QACD,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC;QAE1C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3E,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACnE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEjD,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE;YAC3B,IAAI,CAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC5D,IAAI,CAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SAC7D;aAAM;YACL,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;SAC9C;QAED,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;QAE5C,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5D,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE5D,MAAM,eAAe,GAAG;YACtB,+CAA+C;YAC/C,gDAAgD;YAChD,YAAY;YACZ,oBAAoB,CAAC,OAAO,EAAE,QAAQ;gBACpC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBACrD,GAAG,CAAC,KAAK,CAAC;cACN,OAAO,CAAC,GAAG,CAAC,IAAI;iBACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC;;;WAG3C,CAAC,CAAC;oBACH,OAAO;iBACR;gBAED,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;oBAC1C,sCAAsC;oBACtC,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACrC,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACrC,qCAAqC;iBACtC;YACH,CAAC;SACF,CAAC;QACF,MAAM,yBAAyB,GAAG,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClF,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG;YACtB,cAAc,EAAE;gBACd,yBAAyB;aAC1B;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO;QACnE,IAAI;YACF,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,WAAW,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YACtE,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;SAC/D;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,IAAI,MAAM,CAAC,mBAAmB,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;SAClF;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,IAAI;QACb,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEpC,uDAAuD;QACvD,4BAA4B;QAC5B,MAAM,iBAAiB,GAAG,gCAAgC,CAAC;QAC3D,MAAM,0BAA0B,GAAG,MAAM,CAAC;QAE1C,MAAM,CAAC,GAAG,IAAI,OAAO,CACnB,QAAQ,CAAC;YACP,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;SACL,CAAC,CACH,CAAC;QAEF,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE;YAC/D,CAAC,CAAC,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;SAC1D;QACD,IAAI,IAAI,CAAC,eAAe,EAAE;YACxB,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;SACzC;QACD,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC,CAAC,cAAc,GAAG,EAAE,CAAC;QACtB,CAAC,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC3B,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;QACxB,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;QACf,MAAM,KAAK,GAAG,EAAE,CAAC;QAEjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,CAAC,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE;gBAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3D,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBACrB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;oBAChC,IAAI,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAEhD,OAAO,KAAK,KAAK,IAAI,EAAE;wBACrB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC5B,yDAAyD;wBACzD,MAAM,IAAI,GAAG,UAAU;6BACpB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;6BACjB,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC;6BACvC,SAAS,EAAE,CAAC;wBACf,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC3B,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBAC7C;gBACH,CAAC,CAAC,CAAC;gBAEH,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE,CACxB,MAAM,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAElE,CAAC,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACzC,IAAI,YAAY,GAAG,EAAE,CAAC;gBACtB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBAC5B,YAAY,IAAI,KAAK,CAAC;gBACxB,CAAC,CAAC,CAAC;gBAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,mBAAmB,CACxC,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,GAAG,6BAA6B,QAAQ,CAAC,UAAU,EAAE,CAC3F,CAAC;oBACF,GAAG,CAAC,OAAO,GAAG,YAAY,CAAC;oBAC3B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,CACxC,MAAM,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC,qBAAqB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE5F,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;gBAChC,MAAM,YAAY,GAAG,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAC;gBACnG,YAAY,CAAC,OAAO,GAAG,SAAS,CAAC;gBACjC,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAM;QACnB,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QACzC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEpC,MAAM,SAAS,GAAG,QAAQ,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,UAAU;SACjB,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,eAAe,SAAS,OAAO,SAAS,EAAE,CAAC,CAAC;QACrD,IAAI;YACF,MAAM,SAAS,CAAC,QAAQ,CAAC,CACvB,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,EACtC,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAChC,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE;gBAC/E,MAAM,OAAO,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,8CAA8C,CAAC;gBACtF,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;aACxC;;gBAAM,MAAM,KAAK,CAAC;SACpB;QACD,GAAG,CAAC,IAAI,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QAE/C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,IAAI,CAAC,MAAM;QACf,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;QACrE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,QAAQ,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QAC5D,GAAG,CAAC,IAAI,CAAC,QAAQ,SAAS,OAAO,KAAK,EAAE,CAAC,CAAC;QAE1C,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI;YACF,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAChE,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;SAChC;QAAC,OAAO,KAAK,EAAE;YACd,GAAG,CAAC,IAAI,CAAC,mBAAmB,SAAS,gBAAgB,KAAK,GAAG,CAAC,CAAC;SAChE;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;QAE9E,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5D,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,cAAc,CACzC,YAAY,EACZ;YACE,MAAM,EAAE;gBACN,MAAM,EAAE,iBAAiB;gBACzB,GAAG,EAAE,cAAc;gBACnB,WAAW,EAAE,WAAW;aACzB;SACF,CACF,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QACtD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,OAAO,KAAI,CAAC;IAElB,KAAK,CAAC,GAAG,KAAI,CAAC;CAEf;AAED,MAAM,CAAC,OAAO,GAAG,kBAAkB,CAAC"}
1
+ {"version":3,"file":"HttpProviderClient.js","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACzB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC9B,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AACrC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACvC,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AACzC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;AAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AAC9C,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AACtC,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAEtC,MAAM,EACJ,UAAU,EACV,aAAa,EACb,UAAU,EACV,cAAc,GACf,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AACtC,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC3C,MAAM,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;AACrD,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;AACzD,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAE1C,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEtD,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,EAAE;IAC5B,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO;IAEhD,MAAM,IAAI,SAAS,CAAC,gDAAgD,IAAI,EAAE,CAAC,CAAC;AAC9E,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEnE,MAAM,kBAAkB;IACtB,YAAY,cAAc;QACxB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC,eAAe,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC,cAAc,CAAC;QACpD,IAAI,cAAc,CAAC,QAAQ,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE;YACvD,MAAM,IAAI,cAAc,CAAC,2EAA2E,CAAC,CAAC;SACvG;QACD,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC;QAE1C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3E,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACnE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEjD,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE;YAC3B,IAAI,CAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC5D,IAAI,CAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SAC7D;aAAM;YACL,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;SAC9C;QAED,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;QAE5C,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5D,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE5D,MAAM,eAAe,GAAG;YACtB,+CAA+C;YAC/C,gDAAgD;YAChD,YAAY;YACZ,oBAAoB,CAAC,OAAO,EAAE,QAAQ;gBACpC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBACrD,GAAG,CAAC,KAAK,CAAC;cACN,OAAO,CAAC,GAAG,CAAC,IAAI;iBACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC;;;WAG3C,CAAC,CAAC;oBACH,OAAO;iBACR;gBAED,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;oBAC1C,sCAAsC;oBACtC,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACrC,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACrC,qCAAqC;iBACtC;YACH,CAAC;SACF,CAAC;QACF,MAAM,yBAAyB,GAAG,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClF,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG;YACtB,cAAc,EAAE;gBACd,yBAAyB;aAC1B;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO;QACnE,IAAI;YACF,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,WAAW,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YACtE,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;SAC/D;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,IAAI,MAAM,CAAC,mBAAmB,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;SAClF;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,IAAI;QACb,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEpC,uDAAuD;QACvD,4BAA4B;QAC5B,MAAM,iBAAiB,GAAG,gCAAgC,CAAC;QAC3D,MAAM,0BAA0B,GAAG,MAAM,CAAC;QAE1C,MAAM,CAAC,GAAG,IAAI,OAAO,CACnB,QAAQ,CAAC;YACP,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;SACL,CAAC,CACH,CAAC;QAEF,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE;YAC/D,CAAC,CAAC,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;SAC1D;QACD,IAAI,IAAI,CAAC,eAAe,EAAE;YACxB,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;SACzC;QACD,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC,CAAC,cAAc,GAAG,EAAE,CAAC;QACtB,CAAC,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC3B,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;QACxB,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;QACf,MAAM,KAAK,GAAG,EAAE,CAAC;QAEjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,CAAC,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE;gBAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3D,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBACrB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;oBAChC,IAAI,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAEhD,OAAO,KAAK,KAAK,IAAI,EAAE;wBACrB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC5B,yDAAyD;wBACzD,MAAM,IAAI,GAAG,UAAU;6BACpB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;6BACjB,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC;6BACvC,SAAS,EAAE,CAAC;wBACf,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC3B,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBAC7C;gBACH,CAAC,CAAC,CAAC;gBAEH,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE,CACxB,MAAM,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAElE,CAAC,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACzC,IAAI,YAAY,GAAG,EAAE,CAAC;gBACtB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBAC5B,YAAY,IAAI,KAAK,CAAC;gBACxB,CAAC,CAAC,CAAC;gBAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,mBAAmB,CACxC,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,GAAG,6BAA6B,QAAQ,CAAC,UAAU,EAAE,CAC3F,CAAC;oBACF,GAAG,CAAC,OAAO,GAAG,YAAY,CAAC;oBAC3B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,CACxC,MAAM,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC,qBAAqB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE5F,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;gBAChC,MAAM,YAAY,GAAG,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAC;gBACnG,YAAY,CAAC,OAAO,GAAG,SAAS,CAAC;gBACjC,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAM;QACnB,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QACzC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEpC,MAAM,SAAS,GAAG,QAAQ,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,UAAU;SACjB,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,eAAe,SAAS,OAAO,SAAS,EAAE,CAAC,CAAC;QACrD,IAAI;YACF,MAAM,SAAS,CAAC,QAAQ,CAAC,CACvB,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,EACtC,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAChC,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE;gBAC/E,MAAM,OAAO,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,8CAA8C,CAAC;gBACtF,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;aACxC;;gBAAM,MAAM,KAAK,CAAC;SACpB;QACD,GAAG,CAAC,IAAI,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QAE/C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,IAAI,CAAC,MAAM;QACf,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;QACrE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,QAAQ,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QAC5D,GAAG,CAAC,IAAI,CAAC,QAAQ,SAAS,OAAO,KAAK,EAAE,CAAC,CAAC;QAE1C,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI;YACF,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAChE,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;SAChC;QAAC,OAAO,KAAK,EAAE;YACd,GAAG,CAAC,IAAI,CAAC,mBAAmB,SAAS,gBAAgB,KAAK,GAAG,CAAC,CAAC;SAChE;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;QAE9E,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5D,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,cAAc,CACzC,YAAY,EACZ;YACE,MAAM,EAAE;gBACN,MAAM,EAAE,iBAAiB;gBACzB,GAAG,EAAE,cAAc;gBACnB,WAAW,EAAE,WAAW;aACzB;SACF,CACF,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QACtD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,MAAM,CAAC,MAAM;QACjB,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG;YACd,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,MAAM;SACf,CAAC;QAEF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpC,GAAG,CAAC,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QAC7B,MAAM,SAAS,CAAC,QAAQ,CAAC,CACvB,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAC9B,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAChC,IAAI,MAAM,CAAC,WAAW,EAAE,CACzB,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,uBAAuB,SAAS,OAAO,SAAS,EAAE,CAAC,CAAC;QAC7D,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,OAAO,KAAI,CAAC;IAElB,KAAK,CAAC,GAAG,KAAI,CAAC;CAEf;AAED,MAAM,CAAC,OAAO,GAAG,kBAAkB,CAAC"}
package/S3ProviderClient.d.ts CHANGED
@@ -7,7 +7,7 @@ declare class S3ProviderClient implements ProviderClient {
7
7
  /**
8
8
  * Download a remote file to disk
9
9
  *
10
- * @param {Object} params
10
+ * @param {object} params
11
11
  * @param {string} params.remotePath - the full path to the remote file to be fetched
12
12
  * @param {string} params.localPath - the full local destination file path
13
13
  * @param {string} params.remoteAltBucket - alternate per-file bucket override to this.bucket
@@ -30,8 +30,8 @@ declare class S3ProviderClient implements ProviderClient {
30
30
  /**
31
31
  * Download the remote file to a given s3 location
32
32
  *
33
- * @param {Object} params - the full path to the remote file to be fetched
34
- * @param {string} params.sourceKey - the full path to the remote file to be fetched
33
+ * @param {object} params
34
+ * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
35
35
  * @param {string} params.bucket - destination s3 bucket of the file
36
36
  * @param {string} params.destinationBucket - destination s3 bucket of the file
37
37
  * @param {string} params.destinationKey - destination s3 key of the file
@@ -47,6 +47,20 @@ declare class S3ProviderClient implements ProviderClient {
47
47
  s3uri: string;
48
48
  etag: string;
49
49
  }>;
50
+ /**
51
+ * Upload a file
52
+ *
53
+ * @param {object} params
54
+ * @param {string} params.localPath - the full local file path
55
+ * @param {string} params.uploadPath - the full remote file path for uploading file to
56
+ * @param {string} [params.remoteAltBucket] - alternate per-file bucket override to this.bucket
57
+ * @returns {Promise<string>} the uri of the uploaded file
58
+ */
59
+ upload(params: {
60
+ localPath: string;
61
+ uploadPath: string;
62
+ remoteAltBucket?: string;
63
+ }): Promise<string>;
50
64
  connect(): Promise<void>;
51
65
  end(): Promise<void>;
52
66
  }
package/S3ProviderClient.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"S3ProviderClient.d.ts","sourceRoot":"","sources":["src/S3ProviderClient.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAsB,MAAM,SAAS,CAAC;AAEvF,cAAM,gBAAiB,YAAW,cAAc;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,EAAE,MAAM,EAAE,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO;IAKhD;;;;;;;;;OASG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBnB;;;;;;OAMG;IACG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,EAAE,CAAC;IAwB7D;;;;;;;;;;OAUG;IACG,IAAI,CACR,MAAM,EAAE;QACN,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;KACxB,GACA,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAqDrC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAExB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAE3B;AAED,SAAS,gBAAgB,CAAC"}
1
+ {"version":3,"file":"S3ProviderClient.d.ts","sourceRoot":"","sources":["src/S3ProviderClient.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAsB,MAAM,SAAS,CAAC;AAEvF,cAAM,gBAAiB,YAAW,cAAc;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,EAAE,MAAM,EAAE,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO;IAKhD;;;;;;;;;OASG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC,MAAM,CAAC;IAmBnB;;;;;;OAMG;IACG,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,EAAE,CAAC;IAwB7D;;;;;;;;;;OAUG;IACG,IAAI,CACR,MAAM,EAAE;QACN,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;KACxB,GACA,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAmD3C;;;;;;;;OAQG;IACG,MAAM,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B;IAgBK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAExB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAE3B;AAED,SAAS,gBAAgB,CAAC"}
package/S3ProviderClient.js CHANGED
@@ -36,7 +36,7 @@ class S3ProviderClient {
36
36
  /**
37
37
  * Download a remote file to disk
38
38
  *
39
- * @param {Object} params
39
+ * @param {object} params
40
40
  * @param {string} params.remotePath - the full path to the remote file to be fetched
41
41
  * @param {string} params.localPath - the full local destination file path
42
42
  * @param {string} params.remoteAltBucket - alternate per-file bucket override to this.bucket
@@ -88,8 +88,8 @@ class S3ProviderClient {
88
88
  /**
89
89
  * Download the remote file to a given s3 location
90
90
  *
91
- * @param {Object} params - the full path to the remote file to be fetched
92
- * @param {string} params.sourceKey - the full path to the remote file to be fetched
91
+ * @param {object} params
92
+ * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
93
93
  * @param {string} params.bucket - destination s3 bucket of the file
94
94
  * @param {string} params.destinationBucket - destination s3 bucket of the file
95
95
  * @param {string} params.destinationKey - destination s3 key of the file
@@ -127,7 +127,6 @@ class S3ProviderClient {
127
127
  sourceObject,
128
128
  destinationBucket,
129
129
  destinationKey,
130
- ACL: 'private',
131
130
  copyTags: true,
132
131
  });
133
132
  return { s3uri, etag };
@@ -140,6 +139,24 @@ class S3ProviderClient {
140
139
  throw error;
141
140
  }
142
141
  }
142
+ /**
143
+ * Upload a file
144
+ *
145
+ * @param {object} params
146
+ * @param {string} params.localPath - the full local file path
147
+ * @param {string} params.uploadPath - the full remote file path for uploading file to
148
+ * @param {string} [params.remoteAltBucket] - alternate per-file bucket override to this.bucket
149
+ * @returns {Promise<string>} the uri of the uploaded file
150
+ */
151
+ async upload(params) {
152
+ const { localPath, uploadPath, remoteAltBucket } = params;
153
+ log.info(params);
154
+ const remoteBucket = remoteAltBucket || this.bucket;
155
+ await S3.putFile(remoteBucket, uploadPath, localPath);
156
+ const s3Uri = S3.buildS3Uri(remoteBucket, uploadPath);
157
+ log.info(`Finishing uploading ${localPath} to ${s3Uri}`);
158
+ return s3Uri;
159
+ }
143
160
  /* eslint-disable @typescript-eslint/no-empty-function */
144
161
  async connect() { }
145
162
  async end() { }
package/S3ProviderClient.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"S3ProviderClient.js","sourceRoot":"","sources":["src/S3ProviderClient.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,2DAA6C;AAC7C,yDAA2C;AAC3C,wDAA0C;AAC1C,+BAA+C;AAC/C,mCAAuF;AAEvF,MAAM,gBAAgB;IAGpB,YAAY,EAAE,MAAM,KAA0B,EAAE;QAC9C,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,SAAS,CAAC,oBAAoB,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,QAAQ,CAAC,MAId;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;QAE1D,MAAM,YAAY,GAAG,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC;QAEpD,MAAM,SAAS,GAAG,QAAQ,YAAY,IAAI,UAAU,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,eAAe,SAAS,OAAO,SAAS,EAAE,CAAC,CAAC;QAErD,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE,YAAY;YACpB,GAAG,EAAE,UAAU;SAChB,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACzD,GAAG,CAAC,IAAI,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QAE/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC;YACvC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5B,IAAI,CAAC,IAAA,0BAAkB,EAAC,MAAM,CAAC,EAAE;gBAC/B,MAAM,IAAI,SAAS,CAAC,aAAa,MAAM,6BAA6B,CAAC,CAAC;aACvE;YACD,OAAO;gBACL,IAAI,EAAE,IAAA,eAAQ,EAAC,MAAM,CAAC,GAAG,CAAC;gBAC1B,yEAAyE;gBACzE,iEAAiE;gBACjE,IAAI,EAAE,IAAA,cAAO,EAAC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAA,cAAO,EAAC,MAAM,CAAC,GAAG,CAAC;gBACnE,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,IAAI,CACR,MAKC;QAED,MAAM,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QAC7E,MAAM,YAAY,GAAG,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;QAC3C,MAAM,SAAS,GAAG,cAAc,CAAC;QAEjC,IAAI;YACF,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;YAClE,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;YAE/D,uDAAuD;YACvD,0BAA0B;YAC1B,IAAI,YAAY,CAAC,aAAa,KAAK,CAAC,EAAE;gBACpC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC;oBACjD,UAAU,EAAE,IAAA,WAAI,EAAC,YAAY,EAAE,SAAS,CAAC;oBACzC,MAAM,EAAE,iBAAiB;oBACzB,GAAG,EAAE,cAAc;iBACpB,CAAC,CAAC;gBAEH,kEAAkE;gBAClE,uDAAuD;gBACvD,EAAE;gBACF,gDAAgD;gBAChD,IAAI,CAAC,gBAAgB,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE;oBAC/C,MAAM,IAAI,KAAK,CACb,4CAA4C,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,SAAS,CAAC,OAAO,KAAK,EAAE,CACjG,CAAC;iBACH;gBAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC;gBACnC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;aACxB;YAED,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,mBAAmB,CAAC;gBAC5C,YAAY;gBACZ,SAAS;gBACT,YAAY;gBACZ,iBAAiB;gBACjB,cAAc;gBACd,GAAG,EAAE,SAAS;gBACd,QAAQ,EAAE,IAAI;aACf,CAAC,CAAC;YACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SACxB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE;gBAC3D,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;gBAC9D,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;aACrE;YAED,MAAM,KAAK,CAAC;SACb;IACH,CAAC;IAED,yDAAyD;IACzD,KAAK,CAAC,OAAO,KAAmB,CAAC;IAEjC,KAAK,CAAC,GAAG,KAAmB,CAAC;CAE9B;AAED,iBAAS,gBAAgB,CAAC"}
1
+ {"version":3,"file":"S3ProviderClient.js","sourceRoot":"","sources":["src/S3ProviderClient.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,2DAA6C;AAC7C,yDAA2C;AAC3C,wDAA0C;AAC1C,+BAA+C;AAC/C,mCAAuF;AAEvF,MAAM,gBAAgB;IAGpB,YAAY,EAAE,MAAM,KAA0B,EAAE;QAC9C,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,SAAS,CAAC,oBAAoB,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,QAAQ,CAAC,MAId;QACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;QAE1D,MAAM,YAAY,GAAG,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC;QAEpD,MAAM,SAAS,GAAG,QAAQ,YAAY,IAAI,UAAU,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,eAAe,SAAS,OAAO,SAAS,EAAE,CAAC,CAAC;QAErD,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE,YAAY;YACpB,GAAG,EAAE,UAAU;SAChB,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACzD,GAAG,CAAC,IAAI,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QAE/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC;YACvC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5B,IAAI,CAAC,IAAA,0BAAkB,EAAC,MAAM,CAAC,EAAE;gBAC/B,MAAM,IAAI,SAAS,CAAC,aAAa,MAAM,6BAA6B,CAAC,CAAC;aACvE;YACD,OAAO;gBACL,IAAI,EAAE,IAAA,eAAQ,EAAC,MAAM,CAAC,GAAG,CAAC;gBAC1B,yEAAyE;gBACzE,iEAAiE;gBACjE,IAAI,EAAE,IAAA,cAAO,EAAC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAA,cAAO,EAAC,MAAM,CAAC,GAAG,CAAC;gBACnE,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,IAAI,CACR,MAKC;QAED,MAAM,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QAC7E,MAAM,YAAY,GAAG,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;QAC3C,MAAM,SAAS,GAAG,cAAc,CAAC;QAEjC,IAAI;YACF,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;YAClE,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;YAE/D,uDAAuD;YACvD,0BAA0B;YAC1B,IAAI,YAAY,CAAC,aAAa,KAAK,CAAC,EAAE;gBACpC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC;oBACjD,UAAU,EAAE,IAAA,WAAI,EAAC,YAAY,EAAE,SAAS,CAAC;oBACzC,MAAM,EAAE,iBAAiB;oBACzB,GAAG,EAAE,cAAc;iBACpB,CAAC,CAAC;gBAEH,kEAAkE;gBAClE,uDAAuD;gBACvD,EAAE;gBACF,gDAAgD;gBAChD,IAAI,CAAC,gBAAgB,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE;oBAC/C,MAAM,IAAI,KAAK,CACb,4CAA4C,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,SAAS,CAAC,OAAO,KAAK,EAAE,CACjG,CAAC;iBACH;gBAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC;gBACnC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;aACxB;YAED,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,mBAAmB,CAAC;gBAC5C,YAAY;gBACZ,SAAS;gBACT,YAAY;gBACZ,iBAAiB;gBACjB,cAAc;gBACd,QAAQ,EAAE,IAAI;aACf,CAAC,CAAC;YACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SACxB;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE;gBAC3D,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;gBAC9D,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;aACrE;YAED,MAAM,KAAK,CAAC;SACb;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,CAAC,MAIZ;QACC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjB,MAAM,YAAY,GAAG,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC;QACpD,MAAM,EAAE,CAAC,OAAO,CACd,YAAY,EACZ,UAAU,EACV,SAAS,CACV,CAAC;QAEF,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QACtD,GAAG,CAAC,IAAI,CAAC,uBAAuB,SAAS,OAAO,KAAK,EAAE,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,yDAAyD;IACzD,KAAK,CAAC,OAAO,KAAmB,CAAC;IAEjC,KAAK,CAAC,GAAG,KAAmB,CAAC;CAE9B;AAED,iBAAS,gBAAgB,CAAC"}
package/consumer.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwC,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAG3F,oBAAY,uBAAuB,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAE/F,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,OAAO,CAAA;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;IACjD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;gBAE/B,EACV,QAAQ,EACR,YAAgB,EAChB,SAAc,EACd,iBAAiB,EACjB,sBAA6B,GAC9B,EAAE,yBAAyB;YAUd,cAAc;YAcd,eAAe;IAqBvB,OAAO,CAAC,EAAE,EAAE,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC;CA6B5D"}
1
+ {"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAIzE,oBAAY,uBAAuB,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAG/F,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,OAAO,CAAA;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;IACjD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;gBAE/B,EACV,QAAQ,EACR,YAAgB,EAChB,SAAc,EACd,iBAAiB,EACjB,sBAA6B,GAC9B,EAAE,yBAAyB;YAUd,cAAc;YAsBd,eAAe;IAqBvB,OAAO,CAAC,EAAE,EAAE,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC;CA6B5D"}
package/consumer.js CHANGED
@@ -22,10 +22,15 @@ var __importStar = (this && this.__importStar) || function (mod) {
22
22
  __setModuleDefault(result, mod);
23
23
  return result;
24
24
  };
25
+ var __importDefault = (this && this.__importDefault) || function (mod) {
26
+ return (mod && mod.__esModule) ? mod : { "default": mod };
27
+ };
25
28
  Object.defineProperty(exports, "__esModule", { value: true });
26
29
  exports.Consumer = void 0;
27
30
  const SQS_1 = require("@cumulus/aws-client/SQS");
28
- const log = __importStar(require("@cumulus/common/log"));
31
+ const sqs = __importStar(require("@cumulus/aws-client/SQS"));
32
+ const logger_1 = __importDefault(require("@cumulus/logger"));
33
+ const log = new logger_1.default({ sender: '@cumulus/ingest/consumer' });
29
34
  class Consumer {
30
35
  constructor({ queueUrl, messageLimit = 1, timeLimit = 90, visibilityTimeout, deleteProcessedMessage = true, }) {
31
36
  this.queueUrl = queueUrl;
@@ -39,11 +44,18 @@ class Consumer {
39
44
  async processMessage(message, fn) {
40
45
  try {
41
46
  await fn(this.queueUrl, message);
42
- if (this.deleteProcessedMessage)
43
- await (0, SQS_1.deleteSQSMessage)(this.queueUrl, message.ReceiptHandle);
47
+ if (this.deleteProcessedMessage) {
48
+ await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
49
+ }
44
50
  return 1;
45
51
  }
46
52
  catch (error) {
53
+ if (error.code === 'ExecutionAlreadyExists') {
54
+ log.debug('Deleting message for execution that already exists...');
55
+ await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
56
+ log.debug('Completed deleting message.');
57
+ return 1;
58
+ }
47
59
  log.error(error);
48
60
  return 0;
49
61
  }
package/consumer.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"consumer.js","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAA2F;AAC3F,yDAA2C;AAY3C,MAAa,QAAQ;IASnB,YAAY,EACV,QAAQ,EACR,YAAY,GAAG,CAAC,EAChB,SAAS,GAAG,EAAE,EACd,iBAAiB,EACjB,sBAAsB,GAAG,IAAI,GACH;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,OAAmB,EACnB,EAA2B;QAE3B,IAAI;YACF,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,sBAAsB;gBAAE,MAAM,IAAA,sBAAgB,EAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;YAC9F,OAAO,CAAC,CAAC;SACV;QAAC,OAAO,KAAK,EAAE;YACd,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjB,OAAO,CAAC,CAAC;SACV;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,EAA2B,EAC3B,YAAoB,EACpB,iBAAyB;QAEzB,IAAI,YAAY,GAAG,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,2EAA2E,YAAY,EAAE,CAAC,CAAC;QAElI,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAkB,EACvC,IAAI,CAAC,QAAQ,EACb,EAAE,aAAa,EAAE,YAAY,EAAE,iBAAiB,EAAE,CACnD,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YACvB,GAAG,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC9E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC7C,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAa,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;SACtE;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAA2B;QACvC,IAAI,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACrC,GAAG,CAAC,IAAI,CAAC,+BAA+B,YAAY,cAAc,CAAC,CAAC;QAEpE,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,qCAAqC;QACrC,8FAA8F;QAC9F,OAAO,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YAC3C,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,YAAY,GAAG,EAAE,EAAE;gBACrB,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBACrE,YAAY,IAAI,EAAE,CAAC;aACpB;iBAAM,IAAI,YAAY,GAAG,CAAC,EAAE;gBAC3B,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC/E,YAAY,IAAI,YAAY,CAAC;aAC9B;YACD,GAAG,IAAI,OAAO,CAAC;YACf,oEAAoE;YACpE,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE;gBAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,wCAAwC,CAAC,CAAC;aAC5E;SACF;QACD,oCAAoC;QAEpC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,yCAAyC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzE,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAzFD,4BAyFC"}
1
+ {"version":3,"file":"consumer.js","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAyE;AACzE,6DAA+C;AAC/C,6DAAqC;AAIrC,MAAM,GAAG,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAS/D,MAAa,QAAQ;IASnB,YAAY,EACV,QAAQ,EACR,YAAY,GAAG,CAAC,EAChB,SAAS,GAAG,EAAE,EACd,iBAAiB,EACjB,sBAAsB,GAAG,IAAI,GACH;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,OAAmB,EACnB,EAA2B;QAE3B,IAAI;YACF,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,sBAAsB,EAAE;gBAC/B,MAAM,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;aAClE;YACD,OAAO,CAAC,CAAC;SACV;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,CAAC,IAAI,KAAK,wBAAwB,EAAE;gBAC3C,GAAG,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBACnE,MAAM,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;gBACjE,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;gBACzC,OAAO,CAAC,CAAC;aACV;YACD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjB,OAAO,CAAC,CAAC;SACV;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,EAA2B,EAC3B,YAAoB,EACpB,iBAAyB;QAEzB,IAAI,YAAY,GAAG,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,2EAA2E,YAAY,EAAE,CAAC,CAAC;QAElI,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAkB,EACvC,IAAI,CAAC,QAAQ,EACb,EAAE,aAAa,EAAE,YAAY,EAAE,iBAAiB,EAAE,CACnD,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YACvB,GAAG,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC9E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC7C,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAa,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;SACtE;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAA2B;QACvC,IAAI,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACrC,GAAG,CAAC,IAAI,CAAC,+BAA+B,YAAY,cAAc,CAAC,CAAC;QAEpE,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,qCAAqC;QACrC,8FAA8F;QAC9F,OAAO,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YAC3C,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,YAAY,GAAG,EAAE,EAAE;gBACrB,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBACrE,YAAY,IAAI,EAAE,CAAC;aACpB;iBAAM,IAAI,YAAY,GAAG,CAAC,EAAE;gBAC3B,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC/E,YAAY,IAAI,YAAY,CAAC;aAC9B;YACD,GAAG,IAAI,OAAO,CAAC;YACf,oEAAoE;YACpE,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE;gBAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,wCAAwC,CAAC,CAAC;aAC5E;SACF;QACD,oCAAoC;QAEpC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,yCAAyC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzE,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAjGD,4BAiGC"}
package/lock.d.ts CHANGED
@@ -3,36 +3,39 @@ export interface Lock {
3
3
  LastModified?: Date;
4
4
  }
5
5
  /**
6
- * Checks all locks and removes those older than five minutes. Returns a count
7
- * of locks that are not older than five minutes.
8
- *
9
- * @param {Object} bucket - The AWS S3 bucket with the locks to check
10
- * @param {Array} locks - The list of locks in the bucket
11
- * @returns {integer} - Number of locks remaining in bucket
12
- **/
13
- export declare function checkOldLocks(bucket: string, locks?: Lock[]): Promise<number>;
6
+ * Checks all locks and removes those older than five minutes. Returns a count
7
+ * of locks that are not older than configured retention period or 5 minutes.
8
+ *
9
+ * @param {object} bucket - The AWS S3 bucket with the locks to check
10
+ * @param {Array} locks - The list of locks in the bucket
11
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
12
+ * @returns {integer} - Number of locks remaining in bucket
13
+ */
14
+ export declare function checkOldLocks(bucket: string, locks?: Lock[], retentionTimeInSeconds?: number): Promise<number>;
14
15
  /**
15
- * Counts the number of locks in a bucket.
16
- *
17
- * @param {Object} bucket - The AWS S3 bucket to check
18
- * @param {string} providerName - The provider name
19
- * @returns {integer} - Number of current locks in the bucket
20
- **/
21
- export declare function countLock(bucket: string, providerName: string): Promise<number>;
22
- export declare function removeLock(bucket: string, providerName: string, filename: string): Promise<void>;
16
+ * Counts the number of locks in a bucket.
17
+ *
18
+ * @param {object} bucket - The AWS S3 bucket to check
19
+ * @param {string} providerName - The provider name
20
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
21
+ * @returns {integer} - Number of current locks in the bucket
22
+ */
23
+ export declare function countLock(bucket: string, providerName: string, retentionTimeInSeconds?: number): Promise<number>;
24
+ export declare function removeLock(bucket: string, providerName: string, granuleId: string): Promise<void>;
23
25
  /**
24
26
  *
25
- * @param {string} bucket
26
- * @param {Object} provider
27
- * @param {string} provider.id
28
- * @param {number} provider.globalConnectionLimit
29
- * @param {string} filename
30
- * @param {number} counter
31
- * @param {string} ACL
27
+ * @param {string} bucket - system bucket to place the lock files
28
+ * @param {object} provider - provider object
29
+ * @param {string} provider.id - provider id
30
+ * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
31
+ * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
32
+ * @param {string} granuleId - id of downloading granule
33
+ * @param {number} counter - retry counter
32
34
  * @returns {Promise<boolean>}
33
35
  */
34
36
  export declare function proceed(bucket: string, provider: {
35
37
  id: string;
36
38
  globalConnectionLimit?: number;
37
- }, filename: string, counter: number | undefined, ACL: string): Promise<boolean>;
39
+ maxDownloadTime?: number;
40
+ }, granuleId: string, counter?: number): Promise<boolean>;
38
41
  //# sourceMappingURL=lock.d.ts.map
package/lock.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,IAAI,EAAO,GACjB,OAAO,CAAC,MAAM,CAAC,CAoBjB;AAED;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAOjB;AAiBD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IACR,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAA;CAC/B,EACD,QAAQ,EAAE,MAAM,EAChB,OAAO,oBAAI,EACX,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,OAAO,CAAC,CAwBlB"}
1
+ {"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,IAAI,EAAO,EAClB,sBAAsB,GAAE,MAAY,GACnC,OAAO,CAAC,MAAM,CAAC,CAqBjB;AAED;;;;;;;GAOG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,sBAAsB,CAAC,EAAE,MAAM,GAC9B,OAAO,CAAC,MAAM,CAAC,CAOjB;AAeD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IACR,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,EACD,SAAS,EAAE,MAAM,EACjB,OAAO,SAAI,GACV,OAAO,CAAC,OAAO,CAAC,CAwBlB"}
package/lock.js CHANGED
@@ -29,87 +29,90 @@ const common_1 = require("@cumulus/common");
29
29
  const log = __importStar(require("@cumulus/common/log"));
30
30
  const lockPrefix = 'lock';
31
31
  /**
32
- * Checks all locks and removes those older than five minutes. Returns a count
33
- * of locks that are not older than five minutes.
34
- *
35
- * @param {Object} bucket - The AWS S3 bucket with the locks to check
36
- * @param {Array} locks - The list of locks in the bucket
37
- * @returns {integer} - Number of locks remaining in bucket
38
- **/
39
- async function checkOldLocks(bucket, locks = []) {
40
- const fiveMinutesAgo = Date.now() - (5 * 60 * 1000);
32
+ * Checks all locks and removes those older than five minutes. Returns a count
33
+ * of locks that are not older than configured retention period or 5 minutes.
34
+ *
35
+ * @param {object} bucket - The AWS S3 bucket with the locks to check
36
+ * @param {Array} locks - The list of locks in the bucket
37
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
38
+ * @returns {integer} - Number of locks remaining in bucket
39
+ */
40
+ async function checkOldLocks(bucket, locks = [], retentionTimeInSeconds = 300) {
41
+ const expirationTimestamp = Date.now() - (retentionTimeInSeconds * 1000);
41
42
  const expiredLocks = locks.filter((lock) => {
42
43
  if (!lock.LastModified) {
43
44
  throw new TypeError(`Could not find LastModified on ${JSON.stringify(lock)}`);
44
45
  }
45
- return lock.LastModified.getTime() < fiveMinutesAgo;
46
+ return lock.LastModified.getTime() < expirationTimestamp;
46
47
  });
47
48
  await Promise.all(expiredLocks.map((lock) => {
48
49
  if (!lock.Key) {
49
50
  throw new TypeError(`Could not find Key on ${JSON.stringify(lock)}`);
50
51
  }
52
+ log.debug(`Removing expired lock ${JSON.stringify(lock)}`);
51
53
  return (0, S3_1.deleteS3Object)(bucket, lock.Key);
52
54
  }));
53
55
  return locks.length - expiredLocks.length;
54
56
  }
55
57
  exports.checkOldLocks = checkOldLocks;
56
58
  /**
57
- * Counts the number of locks in a bucket.
58
- *
59
- * @param {Object} bucket - The AWS S3 bucket to check
60
- * @param {string} providerName - The provider name
61
- * @returns {integer} - Number of current locks in the bucket
62
- **/
63
- async function countLock(bucket, providerName) {
59
+ * Counts the number of locks in a bucket.
60
+ *
61
+ * @param {object} bucket - The AWS S3 bucket to check
62
+ * @param {string} providerName - The provider name
63
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
64
+ * @returns {integer} - Number of current locks in the bucket
65
+ */
66
+ async function countLock(bucket, providerName, retentionTimeInSeconds) {
64
67
  const locks = await (0, S3_1.listS3ObjectsV2)({
65
68
  Bucket: bucket,
66
69
  Prefix: `${lockPrefix}/${providerName}`,
67
70
  });
68
- return checkOldLocks(bucket, locks);
71
+ return checkOldLocks(bucket, locks, retentionTimeInSeconds);
69
72
  }
70
73
  exports.countLock = countLock;
71
- async function addLock(bucket, providerName, filename, ACL) {
72
- const key = `${lockPrefix}/${providerName}/${filename}`;
74
+ async function addLock(bucket, providerName, granuleId) {
75
+ const key = `${lockPrefix}/${providerName}/${granuleId}`;
73
76
  await (0, S3_1.s3PutObject)({
74
77
  Bucket: bucket,
75
78
  Key: key,
76
79
  Body: '',
77
- ACL,
78
80
  });
79
81
  }
80
- async function removeLock(bucket, providerName, filename) {
81
- await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${filename}`);
82
+ async function removeLock(bucket, providerName, granuleId) {
83
+ await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${granuleId}`);
82
84
  }
83
85
  exports.removeLock = removeLock;
84
86
  /**
85
87
  *
86
- * @param {string} bucket
87
- * @param {Object} provider
88
- * @param {string} provider.id
89
- * @param {number} provider.globalConnectionLimit
90
- * @param {string} filename
91
- * @param {number} counter
92
- * @param {string} ACL
88
+ * @param {string} bucket - system bucket to place the lock files
89
+ * @param {object} provider - provider object
90
+ * @param {string} provider.id - provider id
91
+ * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
92
+ * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
93
+ * @param {string} granuleId - id of downloading granule
94
+ * @param {number} counter - retry counter
93
95
  * @returns {Promise<boolean>}
94
96
  */
95
- async function proceed(bucket, provider, filename, counter = 0, ACL) {
96
- if (provider.globalConnectionLimit === undefined) {
97
+ async function proceed(bucket, provider, granuleId, counter = 0) {
98
+ const { globalConnectionLimit, maxDownloadTime } = provider;
99
+ if (globalConnectionLimit === undefined) {
97
100
  return true;
98
101
  }
99
102
  // Fail if lock is not removed after 270 tries.
100
103
  if (counter > 270) {
104
+ log.debug(`The "${provider.id}" provider has no lock available after ${counter} retries`);
101
105
  return false;
102
106
  }
103
- const globalConnectionLimit = provider.globalConnectionLimit;
104
- const count = await countLock(bucket, provider.id);
107
+ const count = await countLock(bucket, provider.id, maxDownloadTime);
105
108
  if (count >= globalConnectionLimit) {
106
109
  log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
107
110
  // wait for 5 second and try again
108
111
  await (0, common_1.sleep)(5000);
109
- return proceed(bucket, provider, filename, counter + 1, ACL);
112
+ return proceed(bucket, provider, granuleId, counter + 1);
110
113
  }
111
114
  // add the lock
112
- await addLock(bucket, provider.id, filename, ACL);
115
+ await addLock(bucket, provider.id, granuleId);
113
116
  return true;
114
117
  }
115
118
  exports.proceed = proceed;
package/lock.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAIgC;AAChC,4CAAwC;AACxC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,QAAgB,EAAE;IAElB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAEpD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,IAAI,EAAE,EAAE;QACP,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,MAAM,IAAI,SAAS,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAC/E;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC;IACtD,CAAC,CACF,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,IAAI,SAAS,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE;QACD,OAAO,IAAA,mBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAC5C,CAAC;AAvBD,sCAuBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,YAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAe,EAAC;QAClC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,EAAE;KACxC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACtC,CAAC;AAVD,8BAUC;AAED,KAAK,UAAU,OAAO,CACpB,MAAc,EACd,YAAoB,EACpB,QAAgB,EAChB,GAAY;IAEZ,MAAM,GAAG,GAAG,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE,CAAC;IACxD,MAAM,IAAA,gBAAW,EAAC;QAChB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,IAAI,EAAE,EAAE;QACR,GAAG;KACJ,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,YAAoB,EACpB,QAAgB;IAEhB,MAAM,IAAA,mBAAc,EAClB,MAAM,EACN,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE,CAC5C,CAAC;AACJ,CAAC;AATD,gCASC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,QAGC,EACD,QAAgB,EAChB,OAAO,GAAG,CAAC,EACX,GAAW;IAEX,IAAI,QAAQ,CAAC,qBAAqB,KAAK,SAAS,EAAE;QAChD,OAAO,IAAI,CAAC;KACb;IAED,+CAA+C;IAC/C,IAAI,OAAO,GAAG,GAAG,EAAE;QACjB,OAAO,KAAK,CAAC;KACd;IAED,MAAM,qBAAqB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAEnD,IAAI,KAAK,IAAI,qBAAqB,EAAE;QAClC,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,QAAQ,CAAC,qBAAqB,qBAAqB,CAAC,CAAC;QAC5H,kCAAkC;QAClC,MAAM,IAAA,cAAK,EAAC,IAAI,CAAC,CAAC;QAClB,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;KAC9D;IAED,eAAe;IACf,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC;AACd,CAAC;AAjCD,0BAiCC"}
1
+ {"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAIgC;AAChC,4CAAwC;AACxC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B;;;;;;;;GAQG;AACI,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,QAAgB,EAAE,EAClB,yBAAiC,GAAG;IAEpC,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAEzE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,IAAI,EAAE,EAAE;QACP,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,MAAM,IAAI,SAAS,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAC/E;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,mBAAmB,CAAC;IAC3D,CAAC,CACF,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,IAAI,SAAS,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE;QACD,GAAG,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3D,OAAO,IAAA,mBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAC5C,CAAC;AAzBD,sCAyBC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,YAAoB,EACpB,sBAA+B;IAE/B,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAe,EAAC;QAClC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,EAAE;KACxC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,MAAM,EAAE,KAAK,EAAE,sBAAsB,CAAC,CAAC;AAC9D,CAAC;AAXD,8BAWC;AAED,KAAK,UAAU,OAAO,CACpB,MAAc,EACd,YAAoB,EACpB,SAAiB;IAEjB,MAAM,GAAG,GAAG,GAAG,UAAU,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;IACzD,MAAM,IAAA,gBAAW,EAAC;QAChB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,IAAI,EAAE,EAAE;KACT,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,YAAoB,EACpB,SAAiB;IAEjB,MAAM,IAAA,mBAAc,EAClB,MAAM,EACN,GAAG,UAAU,IAAI,YAAY,IAAI,SAAS,EAAE,CAC7C,CAAC;AACJ,CAAC;AATD,gCASC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,QAIC,EACD,SAAiB,EACjB,OAAO,GAAG,CAAC;IAEX,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,QAAQ,CAAC;IAC5D,IAAI,qBAAqB,KAAK,SAAS,EAAE;QACvC,OAAO,IAAI,CAAC;KACb;IAED,+CAA+C;IAC/C,IAAI,OAAO,GAAG,GAAG,EAAE;QACjB,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,OAAO,UAAU,CAAC,CAAC;QAC1F,OAAO,KAAK,CAAC;KACd;IAED,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;IAEpE,IAAI,KAAK,IAAI,qBAAqB,EAAE;QAClC,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,QAAQ,CAAC,qBAAqB,qBAAqB,CAAC,CAAC;QAC5H,kCAAkC;QAClC,MAAM,IAAA,cAAK,EAAC,IAAI,CAAC,CAAC;QAClB,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;KAC1D;IAED,eAAe;IACf,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAjCD,0BAiCC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cumulus/ingest",
3
- "version": "16.0.3-alpha.0",
3
+ "version": "16.1.2",
4
4
  "description": "Ingest utilities",
5
5
  "engines": {
6
6
  "node": ">=16.19.0"
@@ -13,7 +13,8 @@
13
13
  "prepare": "npm run tsc",
14
14
  "tsc": "../../node_modules/.bin/tsc",
15
15
  "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles",
16
- "watch-test": "../../node_modules/.bin/tsc-watch --onsuccess 'npm test'"
16
+ "watch-test": "../../node_modules/.bin/tsc-watch --onsuccess 'npm test'",
17
+ "coverage": "python ../../scripts/coverage_handler/coverage.py"
17
18
  },
18
19
  "publishConfig": {
19
20
  "access": "public"
@@ -38,13 +39,13 @@
38
39
  "author": "Cumulus Authors",
39
40
  "license": "Apache-2.0",
40
41
  "dependencies": {
41
- "@cumulus/aws-client": "16.0.3-alpha.0",
42
- "@cumulus/common": "16.0.3-alpha.0",
43
- "@cumulus/db": "16.0.3-alpha.0",
44
- "@cumulus/errors": "16.0.3-alpha.0",
45
- "@cumulus/logger": "16.0.3-alpha.0",
46
- "@cumulus/message": "16.0.3-alpha.0",
47
- "@cumulus/sftp-client": "16.0.3-alpha.0",
42
+ "@cumulus/aws-client": "16.1.2",
43
+ "@cumulus/common": "16.1.2",
44
+ "@cumulus/db": "16.1.2",
45
+ "@cumulus/errors": "16.1.2",
46
+ "@cumulus/logger": "16.1.2",
47
+ "@cumulus/message": "16.1.2",
48
+ "@cumulus/sftp-client": "16.1.2",
48
49
  "aws-sdk": "^2.585.0",
49
50
  "cksum": "^1.3.0",
50
51
  "encodeurl": "^1.0.2",
@@ -60,10 +61,10 @@
60
61
  "tough-cookie": "~4.0.0"
61
62
  },
62
63
  "devDependencies": {
63
- "@cumulus/checksum": "16.0.3-alpha.0",
64
- "@cumulus/cmrjs": "16.0.3-alpha.0",
65
- "@cumulus/test-data": "16.0.3-alpha.0",
66
- "@cumulus/types": "16.0.3-alpha.0"
64
+ "@cumulus/checksum": "16.1.2",
65
+ "@cumulus/cmrjs": "16.1.2",
66
+ "@cumulus/test-data": "16.1.2",
67
+ "@cumulus/types": "16.1.2"
67
68
  },
68
- "gitHead": "81be18bf42487473dce42432781672da91fbecd0"
69
+ "gitHead": "3abc9c84572564db45073f448fe374ab9fad44e2"
69
70
  }