@cumulus/ingest 21.3.3 → 22.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/package.json +13 -13
  2. package/queue.d.ts +7 -0
  3. package/queue.d.ts.map +1 -1
  4. package/queue.js +45 -0
  5. package/queue.js.map +1 -1
  6. package/src/queue.js +50 -0
  7. package/tsconfig.tsbuildinfo +1 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cumulus/ingest",
3
- "version": "21.3.3",
3
+ "version": "22.0.0",
4
4
  "description": "Ingest utilities",
5
5
  "engines": {
6
6
  "node": ">=22.21.1"
@@ -40,13 +40,13 @@
40
40
  "author": "Cumulus Authors",
41
41
  "license": "Apache-2.0",
42
42
  "dependencies": {
43
- "@cumulus/aws-client": "21.3.3",
44
- "@cumulus/common": "21.3.3",
45
- "@cumulus/db": "21.3.3",
46
- "@cumulus/errors": "21.3.3",
47
- "@cumulus/logger": "21.3.3",
48
- "@cumulus/message": "21.3.3",
49
- "@cumulus/sftp-client": "21.3.3",
43
+ "@cumulus/aws-client": "22.0.0",
44
+ "@cumulus/common": "22.0.0",
45
+ "@cumulus/db": "22.0.0",
46
+ "@cumulus/errors": "22.0.0",
47
+ "@cumulus/logger": "22.0.0",
48
+ "@cumulus/message": "22.0.0",
49
+ "@cumulus/sftp-client": "22.0.0",
50
50
  "cksum": "^1.3.0",
51
51
  "encodeurl": "^1.0.2",
52
52
  "fs-extra": "^5.0.0",
@@ -60,10 +60,10 @@
60
60
  "tough-cookie": "~4.0.0"
61
61
  },
62
62
  "devDependencies": {
63
- "@cumulus/checksum": "21.3.3",
64
- "@cumulus/cmrjs": "21.3.3",
65
- "@cumulus/test-data": "21.3.3",
66
- "@cumulus/types": "21.3.3"
63
+ "@cumulus/checksum": "22.0.0",
64
+ "@cumulus/cmrjs": "22.0.0",
65
+ "@cumulus/test-data": "22.0.0",
66
+ "@cumulus/types": "22.0.0"
67
67
  },
68
- "gitHead": "dbb5ee1d384dc46c37c4371ab8fe40dc2b55d489"
68
+ "gitHead": "9e71ff618aa8f0662309096b94253423c639bad9"
69
69
  }
package/queue.d.ts CHANGED
@@ -1,3 +1,10 @@
1
+ export type CollectionRecord = import('@cumulus/types/api/collections').CollectionRecord;
2
+ /**
3
+ * @typedef { import('@cumulus/types/api/collections').CollectionRecord } CollectionRecord
4
+ * @param {object | undefined} messageTemplate
5
+ * @param {CollectionRecord} collection
6
+ */
7
+ export function joinCollectionProviderToTemplateCmrMeta(messageTemplate: object | undefined, collection: CollectionRecord): any;
1
8
  /**
2
9
  * Enqueue a PDR to be parsed
3
10
  *
package/queue.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;;;;GAkBG;AACH;IAf0B,UAAU,EAAzB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IACS,gBAAgB,EAA/B,MAAM;IACS,GAAG,EAAlB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,KAAK,EAApB,MAAM;IACS,YAAY,EAA3B,MAAM;IACS,QAAQ,EAAvB,MAAM;IACU,mBAAmB;IAEnB,oBAAoB;IAElC,QAAQ,MAAM,CAAC,CA+C3B;AAGD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;IAjB0B,UAAU,EAAzB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,eAAe,EAA9B,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACsB,GAAG,GAA/B,MAAM,GAAG,SAAS;IAEF,mBAAmB;IAEnB,oBAAoB;IAElC,QAAQ,MAAM,CAAC,CAkC3B;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH;IAd0B,kBAAkB,EAAjC,MAAM;IAES,KAAK,EAApB,MAAM;IACS,YAAY,EAA3B,MAAM;IACS,UAAU,EAAzB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,aAAa,EAA5B,MAAM;IACU,QAAQ;IACR,mBAAmB;IAEjC,QAAQ,MAAM,CAAC,CAoD3B"}
1
+ {"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"+BA6Bc,OAAO,gCAAgC,EAAE,gBAAgB;AADvE;;;;GAIG;AACH,yEAHW,MAAM,GAAG,SAAS,cAClB,gBAAgB,OAc1B;AAGD;;;;;;;;;;;;;;;;;;GAkBG;AACH;IAf0B,UAAU,EAAzB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IACS,gBAAgB,EAA/B,MAAM;IACS,GAAG,EAAlB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,KAAK,EAApB,MAAM;IACS,YAAY,EAA3B,MAAM;IACS,QAAQ,EAAvB,MAAM;IACU,mBAAmB;IAEnB,oBAAoB;IAElC,QAAQ,MAAM,CAAC,CAoD3B;AAGD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;IAjB0B,UAAU,EAAzB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,eAAe,EAA9B,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACsB,GAAG,GAA/B,MAAM,GAAG,SAAS;IAEF,mBAAmB;IAEnB,oBAAoB;IAElC,QAAQ,MAAM,CAAC,CAuC3B;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH;IAd0B,kBAAkB,EAAjC,MAAM;IAES,KAAK,EAApB,MAAM;IACS,YAAY,EAA3B,MAAM;IACS,UAAU,EAAzB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,aAAa,EAA5B,MAAM;IACU,QAAQ;IACR,mBAAmB;IAEjC,QAAQ,MAAM,CAAC,CAyD3B"}
package/queue.js CHANGED
@@ -4,6 +4,36 @@ const { sendSQSMessage } = require('@cumulus/aws-client/SQS');
4
4
  const { buildQueueMessageFromTemplate } = require('@cumulus/message/Build');
5
5
  const { buildExecutionArn } = require('@cumulus/message/Executions');
6
6
  const { getWorkflowFileKey, templateKey, } = require('@cumulus/common/workflows');
7
+ const isNil = require('lodash/isNil');
8
+ const CmrProviderNotConfiguredMessage = 'all collections must configure a cmr_provider for sf to be scheduled';
9
+ class CMRProviderNotConfiguredError extends Error {
10
+ /**
11
+ *
12
+ * @param {string} message
13
+ */
14
+ constructor(message) {
15
+ super(`${message} ${CmrProviderNotConfiguredMessage}`);
16
+ this.name = 'CMRProviderNotConfiguredError';
17
+ Error.captureStackTrace(this, CMRProviderNotConfiguredError);
18
+ }
19
+ }
20
+ /**
21
+ * @typedef { import('@cumulus/types/api/collections').CollectionRecord } CollectionRecord
22
+ * @param {object | undefined} messageTemplate
23
+ * @param {CollectionRecord} collection
24
+ */
25
+ function joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection) {
26
+ const { cmrProvider } = collection;
27
+ if (isNil(cmrProvider)) {
28
+ throw new CMRProviderNotConfiguredError(`no cmr_provider found for collection ${collection.name}___${collection.version}`);
29
+ }
30
+ const templateCmr = messageTemplate?.meta?.cmr || {};
31
+ return {
32
+ ...templateCmr,
33
+ provider: cmrProvider,
34
+ };
35
+ }
36
+ exports.joinCollectionProviderToTemplateCmrMeta = joinCollectionProviderToTemplateCmrMeta;
7
37
  /**
8
38
  * Enqueue a PDR to be parsed
9
39
  *
@@ -25,6 +55,10 @@ const { getWorkflowFileKey, templateKey, } = require('@cumulus/common/workflows'
25
55
  */
26
56
  async function enqueueParsePdrMessage({ collection, parentExecutionArn, parsePdrWorkflow, pdr, provider, stack, systemBucket, queueUrl, executionNamePrefix, additionalCustomMeta = {}, }) {
27
57
  const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
58
+ let cmrMeta = {};
59
+ if (collection) {
60
+ cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
61
+ }
28
62
  const { arn: parsePdrArn } = await getJsonS3Object(systemBucket, getWorkflowFileKey(stack, parsePdrWorkflow));
29
63
  const payload = { pdr };
30
64
  const workflow = {
@@ -40,6 +74,7 @@ async function enqueueParsePdrMessage({ collection, parentExecutionArn, parsePdr
40
74
  ...additionalCustomMeta,
41
75
  collection,
42
76
  provider,
77
+ cmr: cmrMeta,
43
78
  },
44
79
  executionNamePrefix,
45
80
  });
@@ -70,6 +105,10 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
70
105
  * - resolves when the message has been enqueued
71
106
  */
72
107
  async function enqueueGranuleIngestMessage({ collection, granules, parentExecutionArn, pdr, provider, messageTemplate, workflow, queueUrl, executionNamePrefix, additionalCustomMeta = {}, }) {
108
+ let cmrMeta = {};
109
+ if (collection) {
110
+ cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
111
+ }
73
112
  const message = buildQueueMessageFromTemplate({
74
113
  messageTemplate,
75
114
  parentExecutionArn,
@@ -80,6 +119,7 @@ async function enqueueGranuleIngestMessage({ collection, granules, parentExecuti
80
119
  ...(pdr ? { pdr } : {}),
81
120
  collection,
82
121
  provider,
122
+ cmr: cmrMeta,
83
123
  },
84
124
  executionNamePrefix,
85
125
  });
@@ -107,6 +147,10 @@ exports.enqueueGranuleIngestMessage = enqueueGranuleIngestMessage;
107
147
  */
108
148
  async function enqueueWorkflowMessage({ parentExecutionArn, stack, systemBucket, collection, provider, queueUrl, workflow, workflowInput, executionNamePrefix, additionalCustomMeta = {}, }) {
109
149
  const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
150
+ let cmrMeta = {};
151
+ if (collection) {
152
+ cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
153
+ }
110
154
  const { arn: queuedWorkflowArn } = await getJsonS3Object(systemBucket, getWorkflowFileKey(stack, workflow));
111
155
  const payload = {
112
156
  ...workflowInput,
@@ -126,6 +170,7 @@ async function enqueueWorkflowMessage({ parentExecutionArn, stack, systemBucket,
126
170
  ...additionalCustomMeta,
127
171
  collection,
128
172
  provider,
173
+ cmr: cmrMeta,
129
174
  },
130
175
  });
131
176
  const arn = buildExecutionArn(message.cumulus_meta.state_machine, message.cumulus_meta.execution_name);
package/queue.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;;;GAkBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAChD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,gBAAgB;QACtB,GAAG,EAAE,WAAW;KACjB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,MAAM,CAAC,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,KAAK,UAAU,2BAA2B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,kBAAkB,EAClB,GAAG,EACH,QAAQ,EACR,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO,EAAE,EAAE,QAAQ,EAAE;QACrB,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,iBAAiB,CACtB,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;AACJ,CAAC;AACD,OAAO,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;AAElE;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,kBAAkB,EAClB,KAAK,EACL,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,MAAM,eAAe,CACtD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CACpC,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,GAAG,aAAa;KACjB,CAAC;IAEF,MAAM,wBAAwB,GAAG;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,iBAAiB;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,QAAQ,EAAE,wBAAwB;QAClC,mBAAmB;QACnB,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC"}
1
+ {"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AACzC,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AAEtC,MAAM,+BAA+B,GAAG,sEAAsE,CAAC;AAC/G,MAAM,6BAA8B,SAAQ,KAAK;IAC/C;;;OAGG;IACH,YAAY,OAAO;QACjB,KAAK,CAAC,GAAG,OAAO,IAAI,+BAA+B,EAAE,CAAC,CAAC;QAEvD,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;QAE5C,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,6BAA6B,CAAC,CAAC;IAC/D,CAAC;CACF;AACD;;;;GAIG;AACH,SAAS,uCAAuC,CAAC,eAAe,EAAE,UAAU;IAC1E,MAAM,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;IACnC,IAAI,KAAK,CAAC,WAAW,CAAC,EAAE;QACtB,MAAM,IAAI,6BAA6B,CACrC,wCAAwC,UAAU,CAAC,IAAI,MAAM,UAAU,CAAC,OAAO,EAAE,CAClF,CAAC;KACH;IACD,MAAM,WAAW,GAAG,eAAe,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;IACrD,OAAO;QACL,GAAG,WAAW;QACd,QAAQ,EAAE,WAAW;KACtB,CAAC;AACJ,CAAC;AACD,OAAO,CAAC,uCAAuC,GAAG,uCAAuC,CAAC;AAE1F;;;;;;;;;;;;;;;;;;GAkBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,UAAU,EAAE;QACd,OAAO,GAAG,uCAAuC,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;KAChF;IACD,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAChD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,gBAAgB;QACtB,GAAG,EAAE,WAAW;KACjB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;YACR,GAAG,EAAE,OAAO;SACb;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,MAAM,CAAC,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,KAAK,UAAU,2BAA2B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,kBAAkB,EAClB,GAAG,EACH,QAAQ,EACR,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,UAAU,EAAE;QACd,OAAO,GAAG,uCAAuC,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;KAChF;IACD,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO,EAAE,EAAE,QAAQ,EAAE;QACrB,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,UAAU;YACV,QAAQ;YACR,GAAG,EAAE,OAAO;SACb;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,iBAAiB,CACtB,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;AACJ,CAAC;AACD,OAAO,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;AAElE;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,kBAAkB,EAClB,KAAK,EACL,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,UAAU,EAAE;QACd,OAAO,GAAG,uCAAuC,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;KAChF;IACD,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,MAAM,eAAe,CACtD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CACpC,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,GAAG,aAAa;KACjB,CAAC;IAEF,MAAM,wBAAwB,GAAG;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,iBAAiB;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,QAAQ,EAAE,wBAAwB;QAClC,mBAAmB;QACnB,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;YACR,GAAG,EAAE,OAAO;SACb;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC"}
package/src/queue.js CHANGED
@@ -10,6 +10,41 @@ const {
10
10
  getWorkflowFileKey,
11
11
  templateKey,
12
12
  } = require('@cumulus/common/workflows');
13
+ const isNil = require('lodash/isNil');
14
+
15
+ const CmrProviderNotConfiguredMessage = 'all collections must configure a cmr_provider for sf to be scheduled';
16
+ class CMRProviderNotConfiguredError extends Error {
17
+ /**
18
+ *
19
+ * @param {string} message
20
+ */
21
+ constructor(message) {
22
+ super(`${message} ${CmrProviderNotConfiguredMessage}`);
23
+
24
+ this.name = 'CMRProviderNotConfiguredError';
25
+
26
+ Error.captureStackTrace(this, CMRProviderNotConfiguredError);
27
+ }
28
+ }
29
+ /**
30
+ * @typedef { import('@cumulus/types/api/collections').CollectionRecord } CollectionRecord
31
+ * @param {object | undefined} messageTemplate
32
+ * @param {CollectionRecord} collection
33
+ */
34
+ function joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection) {
35
+ const { cmrProvider } = collection;
36
+ if (isNil(cmrProvider)) {
37
+ throw new CMRProviderNotConfiguredError(
38
+ `no cmr_provider found for collection ${collection.name}___${collection.version}`
39
+ );
40
+ }
41
+ const templateCmr = messageTemplate?.meta?.cmr || {};
42
+ return {
43
+ ...templateCmr,
44
+ provider: cmrProvider,
45
+ };
46
+ }
47
+ exports.joinCollectionProviderToTemplateCmrMeta = joinCollectionProviderToTemplateCmrMeta;
13
48
 
14
49
  /**
15
50
  * Enqueue a PDR to be parsed
@@ -43,6 +78,10 @@ async function enqueueParsePdrMessage({
43
78
  additionalCustomMeta = {},
44
79
  }) {
45
80
  const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
81
+ let cmrMeta = {};
82
+ if (collection) {
83
+ cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
84
+ }
46
85
  const { arn: parsePdrArn } = await getJsonS3Object(
47
86
  systemBucket,
48
87
  getWorkflowFileKey(stack, parsePdrWorkflow)
@@ -62,6 +101,7 @@ async function enqueueParsePdrMessage({
62
101
  ...additionalCustomMeta,
63
102
  collection,
64
103
  provider,
104
+ cmr: cmrMeta,
65
105
  },
66
106
  executionNamePrefix,
67
107
  });
@@ -110,6 +150,10 @@ async function enqueueGranuleIngestMessage({
110
150
  executionNamePrefix,
111
151
  additionalCustomMeta = {},
112
152
  }) {
153
+ let cmrMeta = {};
154
+ if (collection) {
155
+ cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
156
+ }
113
157
  const message = buildQueueMessageFromTemplate({
114
158
  messageTemplate,
115
159
  parentExecutionArn,
@@ -120,6 +164,7 @@ async function enqueueGranuleIngestMessage({
120
164
  ...(pdr ? { pdr } : {}),
121
165
  collection,
122
166
  provider,
167
+ cmr: cmrMeta,
123
168
  },
124
169
  executionNamePrefix,
125
170
  });
@@ -163,6 +208,10 @@ async function enqueueWorkflowMessage({
163
208
  additionalCustomMeta = {},
164
209
  }) {
165
210
  const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
211
+ let cmrMeta = {};
212
+ if (collection) {
213
+ cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
214
+ }
166
215
  const { arn: queuedWorkflowArn } = await getJsonS3Object(
167
216
  systemBucket,
168
217
  getWorkflowFileKey(stack, workflow)
@@ -188,6 +237,7 @@ async function enqueueWorkflowMessage({
188
237
  ...additionalCustomMeta,
189
238
  collection,
190
239
  provider,
240
+ cmr: cmrMeta,
191
241
  },
192
242
  });
193
243