@cumulus/ingest 18.0.0 → 18.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/lock.d.ts +27 -23
  2. package/lock.d.ts.map +1 -1
  3. package/lock.js +39 -34
  4. package/lock.js.map +1 -1
  5. package/package.json +13 -13
  6. package/src/lock.ts +43 -36
  7. package/tsconfig.tsbuildinfo +1 -1
package/lock.d.ts CHANGED
@@ -3,35 +3,39 @@ export interface Lock {
3
3
  LastModified?: Date;
4
4
  }
5
5
  /**
6
- * Checks all locks and removes those older than five minutes. Returns a count
7
- * of locks that are not older than five minutes.
8
- *
9
- * @param {Object} bucket - The AWS S3 bucket with the locks to check
10
- * @param {Array} locks - The list of locks in the bucket
11
- * @returns {integer} - Number of locks remaining in bucket
12
- **/
13
- export declare function checkOldLocks(bucket: string, locks?: Lock[]): Promise<number>;
6
+ * Checks all locks and removes those older than five minutes. Returns a count
7
+ * of locks that are not older than configured retention period or 5 minutes.
8
+ *
9
+ * @param {object} bucket - The AWS S3 bucket with the locks to check
10
+ * @param {Array} locks - The list of locks in the bucket
11
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
12
+ * @returns {integer} - Number of locks remaining in bucket
13
+ */
14
+ export declare function checkOldLocks(bucket: string, locks?: Lock[], retentionTimeInSeconds?: number): Promise<number>;
14
15
  /**
15
- * Counts the number of locks in a bucket.
16
- *
17
- * @param {Object} bucket - The AWS S3 bucket to check
18
- * @param {string} providerName - The provider name
19
- * @returns {integer} - Number of current locks in the bucket
20
- **/
21
- export declare function countLock(bucket: string, providerName: string): Promise<number>;
22
- export declare function removeLock(bucket: string, providerName: string, filename: string): Promise<void>;
16
+ * Counts the number of locks in a bucket.
17
+ *
18
+ * @param {object} bucket - The AWS S3 bucket to check
19
+ * @param {string} providerName - The provider name
20
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
21
+ * @returns {integer} - Number of current locks in the bucket
22
+ */
23
+ export declare function countLock(bucket: string, providerName: string, retentionTimeInSeconds?: number): Promise<number>;
24
+ export declare function removeLock(bucket: string, providerName: string, granuleId: string): Promise<void>;
23
25
  /**
24
26
  *
25
- * @param {string} bucket
26
- * @param {Object} provider
27
- * @param {string} provider.id
28
- * @param {number} provider.globalConnectionLimit
29
- * @param {string} filename
30
- * @param {number} counter
27
+ * @param {string} bucket - system bucket to place the lock files
28
+ * @param {object} provider - provider object
29
+ * @param {string} provider.id - provider id
30
+ * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
31
+ * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
32
+ * @param {string} granuleId - id of downloading granule
33
+ * @param {number} counter - retry counter
31
34
  * @returns {Promise<boolean>}
32
35
  */
33
36
  export declare function proceed(bucket: string, provider: {
34
37
  id: string;
35
38
  globalConnectionLimit?: number;
36
- }, filename: string, counter?: number): Promise<boolean>;
39
+ maxDownloadTime?: number;
40
+ }, granuleId: string, counter?: number): Promise<boolean>;
37
41
  //# sourceMappingURL=lock.d.ts.map
package/lock.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,IAAI,EAAO,GACjB,OAAO,CAAC,MAAM,CAAC,CAoBjB;AAED;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAOjB;AAeD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;;;;;;;GASG;AACH,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IACR,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAA;CAC/B,EACD,QAAQ,EAAE,MAAM,EAChB,OAAO,SAAI,GACV,OAAO,CAAC,OAAO,CAAC,CAwBlB"}
1
+ {"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,IAAI,EAAO,EAClB,sBAAsB,GAAE,MAAY,GACnC,OAAO,CAAC,MAAM,CAAC,CAqBjB;AAED;;;;;;;GAOG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,sBAAsB,CAAC,EAAE,MAAM,GAC9B,OAAO,CAAC,MAAM,CAAC,CAOjB;AAeD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IACR,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,EACD,SAAS,EAAE,MAAM,EACjB,OAAO,SAAI,GACV,OAAO,CAAC,OAAO,CAAC,CAwBlB"}
package/lock.js CHANGED
@@ -29,85 +29,90 @@ const common_1 = require("@cumulus/common");
29
29
  const log = __importStar(require("@cumulus/common/log"));
30
30
  const lockPrefix = 'lock';
31
31
  /**
32
- * Checks all locks and removes those older than five minutes. Returns a count
33
- * of locks that are not older than five minutes.
34
- *
35
- * @param {Object} bucket - The AWS S3 bucket with the locks to check
36
- * @param {Array} locks - The list of locks in the bucket
37
- * @returns {integer} - Number of locks remaining in bucket
38
- **/
39
- async function checkOldLocks(bucket, locks = []) {
40
- const fiveMinutesAgo = Date.now() - (5 * 60 * 1000);
32
+ * Checks all locks and removes those older than five minutes. Returns a count
33
+ * of locks that are not older than configured retention period or 5 minutes.
34
+ *
35
+ * @param {object} bucket - The AWS S3 bucket with the locks to check
36
+ * @param {Array} locks - The list of locks in the bucket
37
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
38
+ * @returns {integer} - Number of locks remaining in bucket
39
+ */
40
+ async function checkOldLocks(bucket, locks = [], retentionTimeInSeconds = 300) {
41
+ const expirationTimestamp = Date.now() - (retentionTimeInSeconds * 1000);
41
42
  const expiredLocks = locks.filter((lock) => {
42
43
  if (!lock.LastModified) {
43
44
  throw new TypeError(`Could not find LastModified on ${JSON.stringify(lock)}`);
44
45
  }
45
- return lock.LastModified.getTime() < fiveMinutesAgo;
46
+ return lock.LastModified.getTime() < expirationTimestamp;
46
47
  });
47
48
  await Promise.all(expiredLocks.map((lock) => {
48
49
  if (!lock.Key) {
49
50
  throw new TypeError(`Could not find Key on ${JSON.stringify(lock)}`);
50
51
  }
52
+ log.debug(`Removing expired lock ${JSON.stringify(lock)}`);
51
53
  return (0, S3_1.deleteS3Object)(bucket, lock.Key);
52
54
  }));
53
55
  return locks.length - expiredLocks.length;
54
56
  }
55
57
  exports.checkOldLocks = checkOldLocks;
56
58
  /**
57
- * Counts the number of locks in a bucket.
58
- *
59
- * @param {Object} bucket - The AWS S3 bucket to check
60
- * @param {string} providerName - The provider name
61
- * @returns {integer} - Number of current locks in the bucket
62
- **/
63
- async function countLock(bucket, providerName) {
59
+ * Counts the number of locks in a bucket.
60
+ *
61
+ * @param {object} bucket - The AWS S3 bucket to check
62
+ * @param {string} providerName - The provider name
63
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
64
+ * @returns {integer} - Number of current locks in the bucket
65
+ */
66
+ async function countLock(bucket, providerName, retentionTimeInSeconds) {
64
67
  const locks = await (0, S3_1.listS3ObjectsV2)({
65
68
  Bucket: bucket,
66
69
  Prefix: `${lockPrefix}/${providerName}`,
67
70
  });
68
- return checkOldLocks(bucket, locks);
71
+ return checkOldLocks(bucket, locks, retentionTimeInSeconds);
69
72
  }
70
73
  exports.countLock = countLock;
71
- async function addLock(bucket, providerName, filename) {
72
- const key = `${lockPrefix}/${providerName}/${filename}`;
74
+ async function addLock(bucket, providerName, granuleId) {
75
+ const key = `${lockPrefix}/${providerName}/${granuleId}`;
73
76
  await (0, S3_1.s3PutObject)({
74
77
  Bucket: bucket,
75
78
  Key: key,
76
79
  Body: '',
77
80
  });
78
81
  }
79
- async function removeLock(bucket, providerName, filename) {
80
- await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${filename}`);
82
+ async function removeLock(bucket, providerName, granuleId) {
83
+ await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${granuleId}`);
81
84
  }
82
85
  exports.removeLock = removeLock;
83
86
  /**
84
87
  *
85
- * @param {string} bucket
86
- * @param {Object} provider
87
- * @param {string} provider.id
88
- * @param {number} provider.globalConnectionLimit
89
- * @param {string} filename
90
- * @param {number} counter
88
+ * @param {string} bucket - system bucket to place the lock files
89
+ * @param {object} provider - provider object
90
+ * @param {string} provider.id - provider id
91
+ * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
92
+ * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
93
+ * @param {string} granuleId - id of downloading granule
94
+ * @param {number} counter - retry counter
91
95
  * @returns {Promise<boolean>}
92
96
  */
93
- async function proceed(bucket, provider, filename, counter = 0) {
94
- if (provider.globalConnectionLimit === undefined) {
97
+ async function proceed(bucket, provider, granuleId, counter = 0) {
98
+ const { globalConnectionLimit, maxDownloadTime } = provider;
99
+ if (globalConnectionLimit === undefined) {
95
100
  return true;
96
101
  }
97
102
  // Fail if lock is not removed after 270 tries.
98
103
  if (counter > 270) {
104
+ log.debug(`The "${provider.id}" provider has no lock available after ${counter} retries`);
99
105
  return false;
100
106
  }
101
- const globalConnectionLimit = provider.globalConnectionLimit;
102
- const count = await countLock(bucket, provider.id);
107
+ const count = await countLock(bucket, provider.id, maxDownloadTime);
103
108
  if (count >= globalConnectionLimit) {
104
109
  log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
105
110
  // wait for 5 second and try again
106
111
  await (0, common_1.sleep)(5000);
107
- return proceed(bucket, provider, filename, counter + 1);
112
+ return proceed(bucket, provider, granuleId, counter + 1);
108
113
  }
109
114
  // add the lock
110
- await addLock(bucket, provider.id, filename);
115
+ await addLock(bucket, provider.id, granuleId);
111
116
  return true;
112
117
  }
113
118
  exports.proceed = proceed;
package/lock.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAIgC;AAChC,4CAAwC;AACxC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,QAAgB,EAAE;IAElB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAEpD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,IAAI,EAAE,EAAE;QACP,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,MAAM,IAAI,SAAS,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAC/E;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC;IACtD,CAAC,CACF,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,IAAI,SAAS,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE;QACD,OAAO,IAAA,mBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAC5C,CAAC;AAvBD,sCAuBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,YAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAe,EAAC;QAClC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,EAAE;KACxC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACtC,CAAC;AAVD,8BAUC;AAED,KAAK,UAAU,OAAO,CACpB,MAAc,EACd,YAAoB,EACpB,QAAgB;IAEhB,MAAM,GAAG,GAAG,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE,CAAC;IACxD,MAAM,IAAA,gBAAW,EAAC;QAChB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,IAAI,EAAE,EAAE;KACT,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,YAAoB,EACpB,QAAgB;IAEhB,MAAM,IAAA,mBAAc,EAClB,MAAM,EACN,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE,CAC5C,CAAC;AACJ,CAAC;AATD,gCASC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,QAGC,EACD,QAAgB,EAChB,OAAO,GAAG,CAAC;IAEX,IAAI,QAAQ,CAAC,qBAAqB,KAAK,SAAS,EAAE;QAChD,OAAO,IAAI,CAAC;KACb;IAED,+CAA+C;IAC/C,IAAI,OAAO,GAAG,GAAG,EAAE;QACjB,OAAO,KAAK,CAAC;KACd;IAED,MAAM,qBAAqB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAEnD,IAAI,KAAK,IAAI,qBAAqB,EAAE;QAClC,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,QAAQ,CAAC,qBAAqB,qBAAqB,CAAC,CAAC;QAC5H,kCAAkC;QAClC,MAAM,IAAA,cAAK,EAAC,IAAI,CAAC,CAAC;QAClB,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;KACzD;IAED,eAAe;IACf,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC7C,OAAO,IAAI,CAAC;AACd,CAAC;AAhCD,0BAgCC"}
1
+ {"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAIgC;AAChC,4CAAwC;AACxC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B;;;;;;;;GAQG;AACI,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,QAAgB,EAAE,EAClB,yBAAiC,GAAG;IAEpC,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAEzE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,IAAI,EAAE,EAAE;QACP,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,MAAM,IAAI,SAAS,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAC/E;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,mBAAmB,CAAC;IAC3D,CAAC,CACF,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,IAAI,SAAS,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE;QACD,GAAG,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3D,OAAO,IAAA,mBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAC5C,CAAC;AAzBD,sCAyBC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,YAAoB,EACpB,sBAA+B;IAE/B,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAe,EAAC;QAClC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,EAAE;KACxC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,MAAM,EAAE,KAAK,EAAE,sBAAsB,CAAC,CAAC;AAC9D,CAAC;AAXD,8BAWC;AAED,KAAK,UAAU,OAAO,CACpB,MAAc,EACd,YAAoB,EACpB,SAAiB;IAEjB,MAAM,GAAG,GAAG,GAAG,UAAU,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;IACzD,MAAM,IAAA,gBAAW,EAAC;QAChB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,IAAI,EAAE,EAAE;KACT,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,YAAoB,EACpB,SAAiB;IAEjB,MAAM,IAAA,mBAAc,EAClB,MAAM,EACN,GAAG,UAAU,IAAI,YAAY,IAAI,SAAS,EAAE,CAC7C,CAAC;AACJ,CAAC;AATD,gCASC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,QAIC,EACD,SAAiB,EACjB,OAAO,GAAG,CAAC;IAEX,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,QAAQ,CAAC;IAC5D,IAAI,qBAAqB,KAAK,SAAS,EAAE;QACvC,OAAO,IAAI,CAAC;KACb;IAED,+CAA+C;IAC/C,IAAI,OAAO,GAAG,GAAG,EAAE;QACjB,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,OAAO,UAAU,CAAC,CAAC;QAC1F,OAAO,KAAK,CAAC;KACd;IAED,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;IAEpE,IAAI,KAAK,IAAI,qBAAqB,EAAE;QAClC,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,QAAQ,CAAC,qBAAqB,qBAAqB,CAAC,CAAC;QAC5H,kCAAkC;QAClC,MAAM,IAAA,cAAK,EAAC,IAAI,CAAC,CAAC;QAClB,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;KAC1D;IAED,eAAe;IACf,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAjCD,0BAiCC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cumulus/ingest",
3
- "version": "18.0.0",
3
+ "version": "18.1.0",
4
4
  "description": "Ingest utilities",
5
5
  "engines": {
6
6
  "node": ">=16.19.0"
@@ -39,13 +39,13 @@
39
39
  "author": "Cumulus Authors",
40
40
  "license": "Apache-2.0",
41
41
  "dependencies": {
42
- "@cumulus/aws-client": "18.0.0",
43
- "@cumulus/common": "18.0.0",
44
- "@cumulus/db": "18.0.0",
45
- "@cumulus/errors": "18.0.0",
46
- "@cumulus/logger": "18.0.0",
47
- "@cumulus/message": "18.0.0",
48
- "@cumulus/sftp-client": "18.0.0",
42
+ "@cumulus/aws-client": "18.1.0",
43
+ "@cumulus/common": "18.1.0",
44
+ "@cumulus/db": "18.1.0",
45
+ "@cumulus/errors": "18.1.0",
46
+ "@cumulus/logger": "18.1.0",
47
+ "@cumulus/message": "18.1.0",
48
+ "@cumulus/sftp-client": "18.1.0",
49
49
  "aws-sdk": "^2.585.0",
50
50
  "cksum": "^1.3.0",
51
51
  "encodeurl": "^1.0.2",
@@ -61,10 +61,10 @@
61
61
  "tough-cookie": "~4.0.0"
62
62
  },
63
63
  "devDependencies": {
64
- "@cumulus/checksum": "18.0.0",
65
- "@cumulus/cmrjs": "18.0.0",
66
- "@cumulus/test-data": "18.0.0",
67
- "@cumulus/types": "18.0.0"
64
+ "@cumulus/checksum": "18.1.0",
65
+ "@cumulus/cmrjs": "18.1.0",
66
+ "@cumulus/test-data": "18.1.0",
67
+ "@cumulus/types": "18.1.0"
68
68
  },
69
- "gitHead": "43bcef7c0d979a05cd2dd559b811f4ac8f85a9b1"
69
+ "gitHead": "8428a02e0903d8d303c7dd36215aa6a24906e07b"
70
70
  }
package/src/lock.ts CHANGED
@@ -14,25 +14,27 @@ export interface Lock {
14
14
  }
15
15
 
16
16
  /**
17
- * Checks all locks and removes those older than five minutes. Returns a count
18
- * of locks that are not older than five minutes.
19
- *
20
- * @param {Object} bucket - The AWS S3 bucket with the locks to check
21
- * @param {Array} locks - The list of locks in the bucket
22
- * @returns {integer} - Number of locks remaining in bucket
23
- **/
17
+ * Checks all locks and removes those older than five minutes. Returns a count
18
+ * of locks that are not older than configured retention period or 5 minutes.
19
+ *
20
+ * @param {object} bucket - The AWS S3 bucket with the locks to check
21
+ * @param {Array} locks - The list of locks in the bucket
22
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
23
+ * @returns {integer} - Number of locks remaining in bucket
24
+ */
24
25
  export async function checkOldLocks(
25
26
  bucket: string,
26
- locks: Lock[] = []
27
+ locks: Lock[] = [],
28
+ retentionTimeInSeconds: number = 300
27
29
  ): Promise<number> {
28
- const fiveMinutesAgo = Date.now() - (5 * 60 * 1000);
30
+ const expirationTimestamp = Date.now() - (retentionTimeInSeconds * 1000);
29
31
 
30
32
  const expiredLocks = locks.filter(
31
33
  (lock) => {
32
34
  if (!lock.LastModified) {
33
35
  throw new TypeError(`Could not find LastModified on ${JSON.stringify(lock)}`);
34
36
  }
35
- return lock.LastModified.getTime() < fiveMinutesAgo;
37
+ return lock.LastModified.getTime() < expirationTimestamp;
36
38
  }
37
39
  );
38
40
 
@@ -40,6 +42,7 @@ export async function checkOldLocks(
40
42
  if (!lock.Key) {
41
43
  throw new TypeError(`Could not find Key on ${JSON.stringify(lock)}`);
42
44
  }
45
+ log.debug(`Removing expired lock ${JSON.stringify(lock)}`);
43
46
  return deleteS3Object(bucket, lock.Key);
44
47
  }));
45
48
 
@@ -47,30 +50,32 @@ export async function checkOldLocks(
47
50
  }
48
51
 
49
52
  /**
50
- * Counts the number of locks in a bucket.
51
- *
52
- * @param {Object} bucket - The AWS S3 bucket to check
53
- * @param {string} providerName - The provider name
54
- * @returns {integer} - Number of current locks in the bucket
55
- **/
53
+ * Counts the number of locks in a bucket.
54
+ *
55
+ * @param {object} bucket - The AWS S3 bucket to check
56
+ * @param {string} providerName - The provider name
57
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
58
+ * @returns {integer} - Number of current locks in the bucket
59
+ */
56
60
  export async function countLock(
57
61
  bucket: string,
58
- providerName: string
62
+ providerName: string,
63
+ retentionTimeInSeconds?: number
59
64
  ): Promise<number> {
60
65
  const locks = await listS3ObjectsV2({
61
66
  Bucket: bucket,
62
67
  Prefix: `${lockPrefix}/${providerName}`,
63
68
  });
64
69
 
65
- return checkOldLocks(bucket, locks);
70
+ return checkOldLocks(bucket, locks, retentionTimeInSeconds);
66
71
  }
67
72
 
68
73
  async function addLock(
69
74
  bucket: string,
70
75
  providerName: string,
71
- filename: string
76
+ granuleId: string
72
77
  ): Promise<void> {
73
- const key = `${lockPrefix}/${providerName}/${filename}`;
78
+ const key = `${lockPrefix}/${providerName}/${granuleId}`;
74
79
  await s3PutObject({
75
80
  Bucket: bucket,
76
81
  Key: key,
@@ -81,54 +86,56 @@ async function addLock(
81
86
  export async function removeLock(
82
87
  bucket: string,
83
88
  providerName: string,
84
- filename: string
89
+ granuleId: string
85
90
  ): Promise<void> {
86
91
  await deleteS3Object(
87
92
  bucket,
88
- `${lockPrefix}/${providerName}/${filename}`
93
+ `${lockPrefix}/${providerName}/${granuleId}`
89
94
  );
90
95
  }
91
96
 
92
97
  /**
93
98
  *
94
- * @param {string} bucket
95
- * @param {Object} provider
96
- * @param {string} provider.id
97
- * @param {number} provider.globalConnectionLimit
98
- * @param {string} filename
99
- * @param {number} counter
99
+ * @param {string} bucket - system bucket to place the lock files
100
+ * @param {object} provider - provider object
101
+ * @param {string} provider.id - provider id
102
+ * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
103
+ * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
104
+ * @param {string} granuleId - id of downloading granule
105
+ * @param {number} counter - retry counter
100
106
  * @returns {Promise<boolean>}
101
107
  */
102
108
  export async function proceed(
103
109
  bucket: string,
104
110
  provider: {
105
111
  id: string,
106
- globalConnectionLimit?: number
112
+ globalConnectionLimit?: number,
113
+ maxDownloadTime?: number,
107
114
  },
108
- filename: string,
115
+ granuleId: string,
109
116
  counter = 0
110
117
  ): Promise<boolean> {
111
- if (provider.globalConnectionLimit === undefined) {
118
+ const { globalConnectionLimit, maxDownloadTime } = provider;
119
+ if (globalConnectionLimit === undefined) {
112
120
  return true;
113
121
  }
114
122
 
115
123
  // Fail if lock is not removed after 270 tries.
116
124
  if (counter > 270) {
125
+ log.debug(`The "${provider.id}" provider has no lock available after ${counter} retries`);
117
126
  return false;
118
127
  }
119
128
 
120
- const globalConnectionLimit = provider.globalConnectionLimit;
121
-
122
- const count = await countLock(bucket, provider.id);
129
+ const count = await countLock(bucket, provider.id, maxDownloadTime);
123
130
 
124
131
  if (count >= globalConnectionLimit) {
125
132
  log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
126
133
  // wait for 5 second and try again
127
134
  await sleep(5000);
128
- return proceed(bucket, provider, filename, counter + 1);
135
+ return proceed(bucket, provider, granuleId, counter + 1);
129
136
  }
130
137
 
131
138
  // add the lock
132
- await addLock(bucket, provider.id, filename);
139
+ await addLock(bucket, provider.id, granuleId);
133
140
  return true;
134
141
  }