@cumulus/ingest 11.1.5 → 12.0.2

package/lock.d.ts

@@ -20,8 +20,19 @@ export declare function checkOldLocks(bucket: string, locks?: Lock[]): Promise<n
 **/
 export declare function countLock(bucket: string, providerName: string): Promise<number>;
 export declare function removeLock(bucket: string, providerName: string, filename: string): Promise<void>;
+/**
+ *
+ * @param {string} bucket
+ * @param {Object} provider
+ * @param {string} provider.id
+ * @param {number} provider.globalConnectionLimit
+ * @param {string} filename
+ * @param {number} counter
+ * @param {string} ACL
+ * @returns {Promise<boolean>}
+ */
 export declare function proceed(bucket: string, provider: {
     id: string;
     globalConnectionLimit?: number;
-}, filename: string, counter?: number): Promise<boolean>;
+}, filename: string, counter: number | undefined, ACL: string): Promise<boolean>;
 //# sourceMappingURL=lock.d.ts.map

package/lock.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,IAAI,EAAO,GACjB,OAAO,CAAC,MAAM,CAAC,CAoBjB;AAED;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAOjB;AAcD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IACR,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAA;CAC/B,EACD,QAAQ,EAAE,MAAM,EAChB,OAAO,SAAI,GACV,OAAO,CAAC,OAAO,CAAC,CAwBlB"}
+{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,IAAI,EAAO,GACjB,OAAO,CAAC,MAAM,CAAC,CAoBjB;AAED;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,CAAC,CAOjB;AAiBD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IACR,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAA;CAC/B,EACD,QAAQ,EAAE,MAAM,EAChB,OAAO,oBAAI,EACX,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,OAAO,CAAC,CAwBlB"}

package/lock.js

@@ -71,18 +71,31 @@ async function countLock(bucket, providerName) {
     return checkOldLocks(bucket, locks);
 }
 exports.countLock = countLock;
-async function addLock(bucket, providerName, filename) {
+async function addLock(bucket, providerName, filename, ACL) {
+    const key = `${lockPrefix}/${providerName}/${filename}`;
     await (0, S3_1.s3PutObject)({
         Bucket: bucket,
-        Key: `${lockPrefix}/${providerName}/${filename}`,
+        Key: key,
         Body: '',
+        ACL,
     });
 }
 async function removeLock(bucket, providerName, filename) {
     await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${filename}`);
 }
 exports.removeLock = removeLock;
-async function proceed(bucket, provider, filename, counter = 0) {
+/**
+ *
+ * @param {string} bucket
+ * @param {Object} provider
+ * @param {string} provider.id
+ * @param {number} provider.globalConnectionLimit
+ * @param {string} filename
+ * @param {number} counter
+ * @param {string} ACL
+ * @returns {Promise<boolean>}
+ */
+async function proceed(bucket, provider, filename, counter = 0, ACL) {
     if (provider.globalConnectionLimit === undefined) {
         return true;
     }
@@ -96,10 +109,10 @@ async function proceed(bucket, provider, filename, counter = 0) {
         log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
         // wait for 5 second and try again
         await (0, delay_1.default)(5000);
-        return proceed(bucket, provider, filename, counter + 1);
+        return proceed(bucket, provider, filename, counter + 1, ACL);
     }
     // add the lock
-    await addLock(bucket, provider.id, filename);
+    await addLock(bucket, provider.id, filename, ACL);
     return true;
 }
 exports.proceed = proceed;

package/lock.js.map

@@ -1 +1 @@
-{"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kDAA0B;AAC1B,+CAIgC;AAChC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,QAAgB,EAAE;IAElB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAEpD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,IAAI,EAAE,EAAE;QACP,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,MAAM,IAAI,SAAS,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAC/E;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC;IACtD,CAAC,CACF,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,IAAI,SAAS,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE;QACD,OAAO,IAAA,mBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAC5C,CAAC;AAvBD,sCAuBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,YAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAe,EAAC;QAClC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,EAAE;KACxC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACtC,CAAC;AAVD,8BAUC;AAED,KAAK,UAAU,OAAO,CACpB,MAAc,EACd,YAAoB,EACpB,QAAgB;IAEhB,MAAM,IAAA,gBAAW,EAAC;QAChB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE;QAChD,IAAI,EAAE,EAAE;KACT,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,YAAoB,EACpB,QAAgB;IAEhB,MAAM,IAAA,mBAAc,EAClB,MAAM,EACN,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE,CAC5C,CAAC;AACJ,CAAC;AATD,gCASC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,QAGC,EACD,QAAgB,EAChB,OAAO,GAAG,CAAC;IAEX,IAAI,QAAQ,CAAC,qBAAqB,KAAK,SAAS,EAAE;QAChD,OAAO,IAAI,CAAC;KACb;IAED,+CAA+C;IAC/C,IAAI,OAAO,GAAG,GAAG,EAAE;QACjB,OAAO,KAAK,CAAC;KACd;IAED,MAAM,qBAAqB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAEnD,IAAI,KAAK,IAAI,qBAAqB,EAAE;QAClC,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,QAAQ,CAAC,qBAAqB,qBAAqB,CAAC,CAAC;QAC5H,kCAAkC;QAClC,MAAM,IAAA,eAAK,EAAC,IAAI,CAAC,CAAC;QAClB,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;KACzD;IAED,eAAe;IACf,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC7C,OAAO,IAAI,CAAC;AACd,CAAC;AAhCD,0BAgCC"}
+{"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kDAA0B;AAC1B,+CAIgC;AAChC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B;;;;;;;GAOG;AACI,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,QAAgB,EAAE;IAElB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAEpD,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,IAAI,EAAE,EAAE;QACP,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,MAAM,IAAI,SAAS,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAC/E;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC;IACtD,CAAC,CACF,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,IAAI,SAAS,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE;QACD,OAAO,IAAA,mBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAC5C,CAAC;AAvBD,sCAuBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,YAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAe,EAAC;QAClC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,EAAE;KACxC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACtC,CAAC;AAVD,8BAUC;AAED,KAAK,UAAU,OAAO,CACpB,MAAc,EACd,YAAoB,EACpB,QAAgB,EAChB,GAAY;IAEZ,MAAM,GAAG,GAAG,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE,CAAC;IACxD,MAAM,IAAA,gBAAW,EAAC;QAChB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,IAAI,EAAE,EAAE;QACR,GAAG;KACJ,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,YAAoB,EACpB,QAAgB;IAEhB,MAAM,IAAA,mBAAc,EAClB,MAAM,EACN,GAAG,UAAU,IAAI,YAAY,IAAI,QAAQ,EAAE,CAC5C,CAAC;AACJ,CAAC;AATD,gCASC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,QAGC,EACD,QAAgB,EAChB,OAAO,GAAG,CAAC,EACX,GAAW;IAEX,IAAI,QAAQ,CAAC,qBAAqB,KAAK,SAAS,EAAE;QAChD,OAAO,IAAI,CAAC;KACb;IAED,+CAA+C;IAC/C,IAAI,OAAO,GAAG,GAAG,EAAE;QACjB,OAAO,KAAK,CAAC;KACd;IAED,MAAM,qBAAqB,GAAG,QAAQ,CAAC,qBAAqB,CAAC;IAE7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IAEnD,IAAI,KAAK,IAAI,qBAAqB,EAAE;QAClC,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,QAAQ,CAAC,qBAAqB,qBAAqB,CAAC,CAAC;QAC5H,kCAAkC;QAClC,MAAM,IAAA,eAAK,EAAC,IAAI,CAAC,CAAC;QAClB,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;KAC9D;IAED,eAAe;IACf,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC;AACd,CAAC;AAjCD,0BAiCC"}

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@cumulus/ingest",
-  "version": "11.1.5",
+  "version": "12.0.2",
   "description": "Ingest utilities",
   "engines": {
-    "node": ">=12.18.0"
+    "node": ">=14.19.1"
   },
   "scripts": {
     "build": "rm -rf dist && mkdir dist && npm run prepare",
@@ -38,13 +38,13 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "11.1.5",
-    "@cumulus/common": "11.1.5",
-    "@cumulus/db": "11.1.5",
-    "@cumulus/errors": "11.1.5",
-    "@cumulus/logger": "11.1.5",
-    "@cumulus/message": "11.1.5",
-    "@cumulus/sftp-client": "11.1.5",
+    "@cumulus/aws-client": "12.0.2",
+    "@cumulus/common": "12.0.2",
+    "@cumulus/db": "12.0.2",
+    "@cumulus/errors": "12.0.2",
+    "@cumulus/logger": "12.0.2",
+    "@cumulus/message": "12.0.2",
+    "@cumulus/sftp-client": "12.0.2",
     "aws-sdk": "^2.585.0",
     "cksum": "^1.3.0",
     "delay": "^4.3.0",
@@ -61,10 +61,10 @@
     "tough-cookie": "^4.0.0"
   },
   "devDependencies": {
-    "@cumulus/checksum": "11.1.5",
-    "@cumulus/cmrjs": "11.1.5",
-    "@cumulus/test-data": "11.1.5",
-    "@cumulus/types": "11.1.5"
+    "@cumulus/checksum": "12.0.2",
+    "@cumulus/cmrjs": "12.0.2",
+    "@cumulus/test-data": "12.0.2",
+    "@cumulus/types": "12.0.2"
   },
-  "gitHead": "fa0ffeeebc417b7d1194769bceca00d3dd700eae"
+  "gitHead": "a35accbc4444be2e2b09bd30f3626a0a3780ef22"
 }

package/src/lock.ts

@@ -68,12 +68,15 @@ export async function countLock(
 async function addLock(
   bucket: string,
   providerName: string,
-  filename: string
+  filename: string,
+  ACL?: string
 ): Promise<void> {
+  const key = `${lockPrefix}/${providerName}/${filename}`;
   await s3PutObject({
     Bucket: bucket,
-    Key: `${lockPrefix}/${providerName}/${filename}`,
+    Key: key,
     Body: '',
+    ACL,
   });
 }
 
@@ -88,6 +91,17 @@ export async function removeLock(
   );
 }
 
+/**
+ *
+ * @param {string} bucket
+ * @param {Object} provider
+ * @param {string} provider.id
+ * @param {number} provider.globalConnectionLimit
+ * @param {string} filename
+ * @param {number} counter
+ * @param {string} ACL
+ * @returns {Promise<boolean>}
+ */
 export async function proceed(
   bucket: string,
   provider: {
@@ -95,7 +109,8 @@ export async function proceed(
     globalConnectionLimit?: number
   },
   filename: string,
-  counter = 0
+  counter = 0,
+  ACL: string
 ): Promise<boolean> {
   if (provider.globalConnectionLimit === undefined) {
     return true;
@@ -114,10 +129,10 @@ export async function proceed(
     log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
     // wait for 5 second and try again
     await delay(5000);
-    return proceed(bucket, provider, filename, counter + 1);
+    return proceed(bucket, provider, filename, counter + 1, ACL);
   }
 
   // add the lock
-  await addLock(bucket, provider.id, filename);
+  await addLock(bucket, provider.id, filename, ACL);
   return true;
 }

package/src/url-path-template.js

@@ -1,6 +1,5 @@
 const get = require('lodash/get');
 const moment = require('moment');
-const path = require('path');
 
 /**
 * evaluate the operation specified in template
@@ -30,9 +29,6 @@ function evaluateOperation(name, args) {
   case 'substring': {
     return String.prototype.substring.apply(String(valueStr), args.slice(1));
   }
-  case 'extractPath': {
-    return path.dirname(valueStr);
-  }
   default:
     throw new Error(`Could not support operation ${name}`);
   }
@@ -50,21 +46,17 @@ function templateReplacer(context, submatch) {
   const expressionRegex = /([^(]+)\(([^)]+)\)/;
   const matches = submatch.match(expressionRegex);
 
-  // submatch contains operation and args
+  // submatch contains operation
   if (submatch.match(expressionRegex)) {
     const name = matches[1];
     const args = matches[2].split(/\s*,\s*/);
-    // args[0] is either an object path or a constant
-    //   e.g. extractPath(file.path) or extractPath('/a/b/c')
-    // assume args[0] is an object path if it starts with a key of context
-    const isObjectPath = Object.keys(context).includes(args[0].split('.')[0]);
-    const jsonPathValue = get(context, args[0], isObjectPath ? undefined : args[0]);
+    const jsonPathValue = get(context, args[0], null);
     if (!jsonPathValue) throw new Error(`Could not resolve path ${args[0]}`);
     args[0] = jsonPathValue;
     return evaluateOperation(name, args);
   }
 
-  const jsonPathValue = get(context, submatch);
+  const jsonPathValue = get(context, submatch, null);
   if (!jsonPathValue) throw new Error(`Could not resolve path ${submatch}`);
   return jsonPathValue;
 }
@@ -78,17 +70,11 @@ function templateReplacer(context, submatch) {
 * @returns {string} - the url path for the file
 **/
 function urlPathTemplate(pathTemplate, context) {
-  const templateRegex = /{([^{}]+)}/g;
-
+  const templateRegex = /{([^}]+)}/g;
   try {
     // match: The matched substring, submatch: The parenthesized submatch string
-    const replacedPath = pathTemplate.replace(templateRegex, (match, submatch) =>
+    return pathTemplate.replace(templateRegex, (match, submatch) =>
       templateReplacer(context, submatch));
-
-    if (replacedPath.match(templateRegex)) {
-      return urlPathTemplate(replacedPath, context);
-    }
-    return replacedPath;
   } catch (error) {
     throw new Error(
       `Could not resolve path template "${pathTemplate}" with error "${error.toString()}"`