@cumulus/ingest 16.0.3-alpha.0 → 16.1.2

package/queue.d.ts

@@ -29,10 +29,10 @@ export function enqueueParsePdrMessage({ collection, parentExecutionArn, parsePd
  * Enqueue a granule to be ingested
  *
  * @param {Object} params
- * @param {Object} params.granule - the granule to be enqueued for ingest
+ * @param {Object} params.granules - the granules to be enqueued for ingest
  * @param {string} params.queueUrl - the SQS queue to add the message to
- * @param {string} params.granuleIngestMessageTemplateUri - the S3 URI of template for
- * a granule ingest message
+ * @param {Object} params.messageTemplate - Message template for the workflow
+ * @param {Object} params.workflow - workflow name & arn object
  * @param {Object} params.provider - the provider config to be attached to the message
  * @param {Object} params.collection - the collection config to be attached to the
  *   message
@@ -43,10 +43,11 @@ export function enqueueParsePdrMessage({ collection, parentExecutionArn, parsePd
  * @param {Object} [params.additionalCustomMeta] - additional object to merge into meta object
  * @returns {Promise} - resolves when the message has been enqueued
  */
-export function enqueueGranuleIngestMessage({ collection, granules, granuleIngestWorkflow, parentExecutionArn, pdr, provider, stack, systemBucket, queueUrl, executionNamePrefix, additionalCustomMeta, }: {
-    granule: Object;
+export function enqueueGranuleIngestMessage({ collection, granules, parentExecutionArn, pdr, provider, messageTemplate, workflow, queueUrl, executionNamePrefix, additionalCustomMeta, }: {
+    granules: Object;
     queueUrl: string;
-    granuleIngestMessageTemplateUri: string;
+    messageTemplate: Object;
+    workflow: Object;
     provider: Object;
     collection: Object;
     pdr: Object;

package/queue.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;;GAgBG;AACH;IAb0B,GAAG,EAAlB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,0BAA0B,EAAzC,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,kBAAkB,EAAjC,MAAM;IACU,mBAAmB;IAEnB,oBAAoB;iBA+C9C;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH;IAd0B,OAAO,EAAtB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,+BAA+B,EAA9C,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,GAAG,EAAlB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IACU,mBAAmB;IAEnB,oBAAoB;iBAoD9C;AAGD;;;;;;;;;;;;;;GAcG;AACH;IAX0B,QAAQ,EAAvB,MAAM;IACS,aAAa,EAA5B,MAAM;IACU,QAAQ;IACT,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,kBAAkB,EAAjC,MAAM;IACU,mBAAmB;iBAqD7C"}
+{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;;GAgBG;AACH;IAb0B,GAAG,EAAlB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,0BAA0B,EAAzC,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,kBAAkB,EAAjC,MAAM;IACU,mBAAmB;IAEnB,oBAAoB;iBA+C9C;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH;IAd0B,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,eAAe,EAA9B,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,GAAG,EAAlB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IACU,mBAAmB;IAEnB,oBAAoB;iBAkC9C;AAGD;;;;;;;;;;;;;;GAcG;AACH;IAX0B,QAAQ,EAAvB,MAAM;IACS,aAAa,EAA5B,MAAM;IACU,QAAQ;IACT,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,kBAAkB,EAAjC,MAAM;IACU,mBAAmB;iBAqD7C"}

package/queue.js

@@ -50,10 +50,10 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
  * Enqueue a granule to be ingested
  *
  * @param {Object} params
- * @param {Object} params.granule - the granule to be enqueued for ingest
+ * @param {Object} params.granules - the granules to be enqueued for ingest
  * @param {string} params.queueUrl - the SQS queue to add the message to
- * @param {string} params.granuleIngestMessageTemplateUri - the S3 URI of template for
- * a granule ingest message
+ * @param {Object} params.messageTemplate - Message template for the workflow
+ * @param {Object} params.workflow - workflow name & arn object
  * @param {Object} params.provider - the provider config to be attached to the message
  * @param {Object} params.collection - the collection config to be attached to the
  *   message
@@ -64,31 +64,22 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
  * @param {Object} [params.additionalCustomMeta] - additional object to merge into meta object
  * @returns {Promise} - resolves when the message has been enqueued
  */
-async function enqueueGranuleIngestMessage({ collection, granules, granuleIngestWorkflow, parentExecutionArn, pdr, provider, stack, systemBucket, queueUrl, executionNamePrefix, additionalCustomMeta = {}, }) {
-    const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
-    const { arn: ingestGranuleArn } = await getJsonS3Object(systemBucket, getWorkflowFileKey(stack, granuleIngestWorkflow));
-    const payload = { granules };
-    const workflow = {
-        name: granuleIngestWorkflow,
-        arn: ingestGranuleArn,
-    };
+async function enqueueGranuleIngestMessage({ collection, granules, parentExecutionArn, pdr, provider, messageTemplate, workflow, queueUrl, executionNamePrefix, additionalCustomMeta = {}, }) {
     const message = buildQueueMessageFromTemplate({
         messageTemplate,
         parentExecutionArn,
-        payload,
+        payload: { granules },
         workflow,
         customMeta: {
             ...additionalCustomMeta,
+            ...(pdr ? { pdr } : {}),
             collection,
             provider,
         },
         executionNamePrefix,
     });
-    if (pdr)
-        message.meta.pdr = pdr;
-    const arn = buildExecutionArn(message.cumulus_meta.state_machine, message.cumulus_meta.execution_name);
     await sendSQSMessage(queueUrl, message);
-    return arn;
+    return buildExecutionArn(message.cumulus_meta.state_machine, message.cumulus_meta.execution_name);
 }
 exports.enqueueGranuleIngestMessage = enqueueGranuleIngestMessage;
 /**

package/queue.js.map

@@ -1 +1 @@
-{"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAChD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,gBAAgB;QACtB,GAAG,EAAE,WAAW;KACjB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,MAAM,CAAC,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;AAE/D;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,2BAA2B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,qBAAqB,EACrB,kBAAkB,EAClB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,MAAM,eAAe,CACrD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,qBAAqB,CAAC,CACjD,CAAC;IAEF,MAAM,OAAO,GAAG,EAAE,QAAQ,EAAE,CAAC;IAE7B,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,qBAAqB;QAC3B,GAAG,EAAE,gBAAgB;KACtB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,IAAI,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IAEhC,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;AAElE;;;;;;;;;;;;;;GAcG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,kBAAkB,EAClB,KAAK,EACL,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,MAAM,eAAe,CACtD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CACpC,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,GAAG,aAAa;KACjB,CAAC;IAEF,MAAM,wBAAwB,GAAG;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,iBAAiB;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,QAAQ,EAAE,wBAAwB;QAClC,mBAAmB;QACnB,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC"}
+{"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAChD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,gBAAgB;QACtB,GAAG,EAAE,WAAW;KACjB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,MAAM,CAAC,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;AAE/D;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,2BAA2B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,kBAAkB,EAClB,GAAG,EACH,QAAQ,EACR,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO,EAAE,EAAE,QAAQ,EAAE;QACrB,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,iBAAiB,CACtB,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;AACJ,CAAC;AACD,OAAO,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;AAElE;;;;;;;;;;;;;;GAcG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,kBAAkB,EAClB,KAAK,EACL,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,MAAM,eAAe,CACtD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CACpC,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,GAAG,aAAa;KACjB,CAAC;IAEF,MAAM,wBAAwB,GAAG;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,iBAAiB;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,QAAQ,EAAE,wBAAwB;QAClC,mBAAmB;QACnB,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC"}

package/src/HttpProviderClient.js

@@ -9,6 +9,7 @@ const Crawler = require('simplecrawler');
 const got = require('got');
 const { CookieJar } = require('tough-cookie');
 const { promisify } = require('util');
+const stream = require('node:stream');
 
 const {
   buildS3Uri,
@@ -206,7 +207,7 @@ class HttpProviderClient {
   /**
    * Download a remote file to disk
    *
-   * @param {Object} params
+   * @param {object} params
    * @param {string} params.remotePath - the full path to the remote file to be fetched
    * @param {string} params.localPath - the full local destination file path
    * @returns {Promise.<string>} - the path that the file was saved to
@@ -244,7 +245,7 @@ class HttpProviderClient {
   /**
    * Download the remote file to a given s3 location
    *
-   * @param {Object} params
+   * @param {object} params
    * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
    * @param {string} params.destinationBucket - destination s3 bucket of the file
    * @param {string} params.destinationKey - destination s3 key of the file
@@ -291,6 +292,39 @@ class HttpProviderClient {
     return { s3uri, etag };
   }
 
+  /**
+   * Upload a file
+   *
+   * @param {object} params
+   * @param {string} params.localPath - the full local file path
+   * @param {string} params.uploadPath - the full remote file path for uploading file to
+   * @returns {Promise<string>} the uri of the uploaded file
+   */
+  async upload(params) {
+    const { localPath, uploadPath } = params;
+    log.info(params);
+    await this.setUpGotOptions();
+    await this.downloadTLSCertificate();
+    const options = {
+      protocol: this.protocol,
+      host: this.host,
+      port: this.port,
+      path: uploadPath,
+      method: 'POST',
+    };
+
+    const remoteUrl = buildURL(options);
+    got.stream.options = options;
+    await promisify(pipeline)(
+      fs.createReadStream(localPath),
+      await got.stream.post(remoteUrl),
+      new stream.PassThrough()
+    );
+
+    log.info(`Finishing uploading ${localPath} to ${remoteUrl}`);
+    return remoteUrl;
+  }
+
   /* eslint-disable no-empty-function */
   async connect() {}
 

package/src/S3ProviderClient.ts

@@ -15,7 +15,7 @@ class S3ProviderClient implements ProviderClient {
   /**
    * Download a remote file to disk
    *
-   * @param {Object} params
+   * @param {object} params
    * @param {string} params.remotePath - the full path to the remote file to be fetched
    * @param {string} params.localPath - the full local destination file path
    * @param {string} params.remoteAltBucket - alternate per-file bucket override to this.bucket
@@ -79,8 +79,8 @@ class S3ProviderClient implements ProviderClient {
   /**
    * Download the remote file to a given s3 location
    *
-   * @param {Object} params - the full path to the remote file to be fetched
-   * @param {string} params.sourceKey - the full path to the remote file to be fetched
+   * @param {object} params
+   * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
    * @param {string} params.bucket - destination s3 bucket of the file
    * @param {string} params.destinationBucket - destination s3 bucket of the file
    * @param {string} params.destinationKey - destination s3 key of the file
@@ -132,7 +132,6 @@ class S3ProviderClient implements ProviderClient {
         sourceObject,
         destinationBucket,
         destinationKey,
-        ACL: 'private',
         copyTags: true,
       });
       return { s3uri, etag };
@@ -146,6 +145,34 @@ class S3ProviderClient implements ProviderClient {
     }
   }
 
+  /**
+   * Upload a file
+   *
+   * @param {object} params
+   * @param {string} params.localPath - the full local file path
+   * @param {string} params.uploadPath - the full remote file path for uploading file to
+   * @param {string} [params.remoteAltBucket] - alternate per-file bucket override to this.bucket
+   * @returns {Promise<string>} the uri of the uploaded file
+   */
+  async upload(params: {
+    localPath: string,
+    uploadPath: string,
+    remoteAltBucket?: string,
+  }) {
+    const { localPath, uploadPath, remoteAltBucket } = params;
+    log.info(params);
+    const remoteBucket = remoteAltBucket || this.bucket;
+    await S3.putFile(
+      remoteBucket,
+      uploadPath,
+      localPath
+    );
+
+    const s3Uri = S3.buildS3Uri(remoteBucket, uploadPath);
+    log.info(`Finishing uploading ${localPath} to ${s3Uri}`);
+    return s3Uri;
+  }
+
   /* eslint-disable @typescript-eslint/no-empty-function */
   async connect(): Promise<void> {}
 

package/src/consumer.ts

@@ -1,8 +1,10 @@
-import { receiveSQSMessages, deleteSQSMessage, SQSMessage } from '@cumulus/aws-client/SQS';
-import * as log from '@cumulus/common/log';
+import { receiveSQSMessages, SQSMessage } from '@cumulus/aws-client/SQS';
+import * as sqs from '@cumulus/aws-client/SQS';
+import Logger from '@cumulus/logger';
 
 export type MessageConsumerFunction = (queueUrl: string, message: SQSMessage) => Promise<void>;
 
+const log = new Logger({ sender: '@cumulus/ingest/consumer' });
 export interface ConsumerConstructorParams {
   queueUrl: string,
   messageLimit?: number,
@@ -42,9 +44,17 @@ export class Consumer {
   ): Promise<0 | 1> {
     try {
       await fn(this.queueUrl, message);
-      if (this.deleteProcessedMessage) await deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
+      if (this.deleteProcessedMessage) {
+        await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
+      }
       return 1;
     } catch (error) {
+      if (error.code === 'ExecutionAlreadyExists') {
+        log.debug('Deleting message for execution that already exists...');
+        await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
+        log.debug('Completed deleting message.');
+        return 1;
+      }
       log.error(error);
       return 0;
     }

package/src/lock.ts

@@ -14,25 +14,27 @@ export interface Lock {
 }
 
 /**
-* Checks all locks and removes those older than five minutes. Returns a count
-* of locks that are not older than five minutes.
-*
-* @param {Object} bucket - The AWS S3 bucket with the locks to check
-* @param {Array} locks - The list of locks in the bucket
-* @returns {integer} - Number of locks remaining in bucket
-**/
+ * Checks all locks and removes those older than five minutes. Returns a count
+ * of locks that are not older than configured retention period or 5 minutes.
+ *
+ * @param {object} bucket - The AWS S3 bucket with the locks to check
+ * @param {Array} locks - The list of locks in the bucket
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
+ * @returns {integer} - Number of locks remaining in bucket
+ */
 export async function checkOldLocks(
   bucket: string,
-  locks: Lock[] = []
+  locks: Lock[] = [],
+  retentionTimeInSeconds: number = 300
 ): Promise<number> {
-  const fiveMinutesAgo = Date.now() - (5 * 60 * 1000);
+  const expirationTimestamp = Date.now() - (retentionTimeInSeconds * 1000);
 
   const expiredLocks = locks.filter(
     (lock) => {
       if (!lock.LastModified) {
         throw new TypeError(`Could not find LastModified on ${JSON.stringify(lock)}`);
       }
-      return lock.LastModified.getTime() < fiveMinutesAgo;
+      return lock.LastModified.getTime() < expirationTimestamp;
     }
   );
 
@@ -40,6 +42,7 @@ export async function checkOldLocks(
     if (!lock.Key) {
       throw new TypeError(`Could not find Key on ${JSON.stringify(lock)}`);
     }
+    log.debug(`Removing expired lock ${JSON.stringify(lock)}`);
     return deleteS3Object(bucket, lock.Key);
   }));
 
@@ -47,92 +50,92 @@ export async function checkOldLocks(
 }
 
 /**
-* Counts the number of locks in a bucket.
-*
-* @param {Object} bucket - The AWS S3 bucket to check
-* @param {string} providerName - The provider name
-* @returns {integer} - Number of current locks in the bucket
-**/
+ * Counts the number of locks in a bucket.
+ *
+ * @param {object} bucket - The AWS S3 bucket to check
+ * @param {string} providerName - The provider name
+ * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
+ * @returns {integer} - Number of current locks in the bucket
+ */
 export async function countLock(
   bucket: string,
-  providerName: string
+  providerName: string,
+  retentionTimeInSeconds?: number
 ): Promise<number> {
   const locks = await listS3ObjectsV2({
     Bucket: bucket,
     Prefix: `${lockPrefix}/${providerName}`,
   });
 
-  return checkOldLocks(bucket, locks);
+  return checkOldLocks(bucket, locks, retentionTimeInSeconds);
 }
 
 async function addLock(
   bucket: string,
   providerName: string,
-  filename: string,
-  ACL?: string
+  granuleId: string
 ): Promise<void> {
-  const key = `${lockPrefix}/${providerName}/${filename}`;
+  const key = `${lockPrefix}/${providerName}/${granuleId}`;
   await s3PutObject({
     Bucket: bucket,
     Key: key,
     Body: '',
-    ACL,
   });
 }
 
 export async function removeLock(
   bucket: string,
   providerName: string,
-  filename: string
+  granuleId: string
 ): Promise<void> {
   await deleteS3Object(
     bucket,
-    `${lockPrefix}/${providerName}/${filename}`
+    `${lockPrefix}/${providerName}/${granuleId}`
   );
 }
 
 /**
  *
- * @param {string} bucket
- * @param {Object} provider
- * @param {string} provider.id
- * @param {number} provider.globalConnectionLimit
- * @param {string} filename
- * @param {number} counter
- * @param {string} ACL
+ * @param {string} bucket - system bucket to place the lock files
+ * @param {object} provider - provider object
+ * @param {string} provider.id - provider id
+ * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
+ * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
+ * @param {string} granuleId - id of downloading granule
+ * @param {number} counter - retry counter
  * @returns {Promise<boolean>}
  */
 export async function proceed(
   bucket: string,
   provider: {
     id: string,
-    globalConnectionLimit?: number
+    globalConnectionLimit?: number,
+    maxDownloadTime?: number,
   },
-  filename: string,
-  counter = 0,
-  ACL: string
+  granuleId: string,
+  counter = 0
 ): Promise<boolean> {
-  if (provider.globalConnectionLimit === undefined) {
+  const { globalConnectionLimit, maxDownloadTime } = provider;
+  if (globalConnectionLimit === undefined) {
     return true;
   }
 
   // Fail if lock is not removed after 270 tries.
   if (counter > 270) {
+    log.debug(`The "${provider.id}" provider has no lock available after ${counter} retries`);
     return false;
   }
 
-  const globalConnectionLimit = provider.globalConnectionLimit;
-
-  const count = await countLock(bucket, provider.id);
+  const count = await countLock(bucket, provider.id, maxDownloadTime);
 
   if (count >= globalConnectionLimit) {
     log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
     // wait for 5 second and try again
     await sleep(5000);
-    return proceed(bucket, provider, filename, counter + 1, ACL);
+    return proceed(bucket, provider, granuleId, counter + 1);
   }
 
   // add the lock
-  await addLock(bucket, provider.id, filename, ACL);
+  await addLock(bucket, provider.id, granuleId);
   return true;
 }

package/src/queue.js

@@ -79,10 +79,10 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
  * Enqueue a granule to be ingested
  *
  * @param {Object} params
- * @param {Object} params.granule - the granule to be enqueued for ingest
+ * @param {Object} params.granules - the granules to be enqueued for ingest
  * @param {string} params.queueUrl - the SQS queue to add the message to
- * @param {string} params.granuleIngestMessageTemplateUri - the S3 URI of template for
- * a granule ingest message
+ * @param {Object} params.messageTemplate - Message template for the workflow
+ * @param {Object} params.workflow - workflow name & arn object
  * @param {Object} params.provider - the provider config to be attached to the message
  * @param {Object} params.collection - the collection config to be attached to the
  *   message
@@ -96,52 +96,34 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
 async function enqueueGranuleIngestMessage({
   collection,
   granules,
-  granuleIngestWorkflow,
   parentExecutionArn,
   pdr,
   provider,
-  stack,
-  systemBucket,
+  messageTemplate,
+  workflow,
   queueUrl,
   executionNamePrefix,
   additionalCustomMeta = {},
 }) {
-  const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
-  const { arn: ingestGranuleArn } = await getJsonS3Object(
-    systemBucket,
-    getWorkflowFileKey(stack, granuleIngestWorkflow)
-  );
-
-  const payload = { granules };
-
-  const workflow = {
-    name: granuleIngestWorkflow,
-    arn: ingestGranuleArn,
-  };
-
   const message = buildQueueMessageFromTemplate({
     messageTemplate,
     parentExecutionArn,
-    payload,
+    payload: { granules },
     workflow,
     customMeta: {
       ...additionalCustomMeta,
+      ...(pdr ? { pdr } : {}),
       collection,
       provider,
     },
     executionNamePrefix,
   });
 
-  if (pdr) message.meta.pdr = pdr;
-
-  const arn = buildExecutionArn(
+  await sendSQSMessage(queueUrl, message);
+  return buildExecutionArn(
     message.cumulus_meta.state_machine,
     message.cumulus_meta.execution_name
   );
-
-  await sendSQSMessage(queueUrl, message);
-
-  return arn;
 }
 exports.enqueueGranuleIngestMessage = enqueueGranuleIngestMessage;