@cumulus/cmrjs 18.5.5 → 18.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nycrc.json +4 -4
- package/cmr-utils.d.ts +89 -31
- package/cmr-utils.d.ts.map +1 -1
- package/cmr-utils.js +182 -45
- package/cmr-utils.js.map +1 -1
- package/index.d.ts +2 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/package.json +10 -9
- package/src/cmr-utils.js +229 -55
- package/src/index.js +2 -0
- package/tsconfig.tsbuildinfo +1 -1
package/.nycrc.json
CHANGED
package/cmr-utils.d.ts
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
export type CmrCredentials = {
|
|
2
|
+
provider: string;
|
|
3
|
+
clientId: string;
|
|
4
|
+
username?: string;
|
|
5
|
+
password?: string;
|
|
6
|
+
token?: string;
|
|
7
|
+
};
|
|
8
|
+
export type CMRConstructorParams = import('@cumulus/cmr-client/CMR').CMRConstructorParams;
|
|
9
|
+
export type DistributionBucketMap = import('@cumulus/distribution-utils/dist/types').DistributionBucketMap;
|
|
10
|
+
export type ApiFile = any;
|
|
1
11
|
/**
|
|
2
12
|
* Add ETags to file objects as some downstream functions expect this structure.
|
|
3
13
|
*
|
|
@@ -10,31 +20,22 @@ export function constructCmrConceptLink(conceptId: any, extension: any): string;
|
|
|
10
20
|
/**
|
|
11
21
|
* Construct online access url for a given file and a url type.
|
|
12
22
|
*
|
|
13
|
-
* @param {Object} params - input parameters
|
|
14
23
|
* @param {Object} params.file - file object
|
|
15
24
|
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
16
|
-
* @param {
|
|
25
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket name to bucket type
|
|
17
26
|
* @param {Object} params.urlType - url type, distribution or s3
|
|
18
27
|
* @param {distributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
19
28
|
* for all distribution bucketss
|
|
20
29
|
* @param {boolean} params.useDirectS3Type - indicate if direct s3 access type is used
|
|
21
30
|
* @returns {(Object | undefined)} online access url object, undefined if no URL exists
|
|
22
31
|
*/
|
|
23
|
-
export function constructOnlineAccessUrl({ file, distEndpoint, bucketTypes, urlType, distributionBucketMap, useDirectS3Type, }:
|
|
24
|
-
file: Object;
|
|
25
|
-
distEndpoint: string;
|
|
26
|
-
bucketTypes: Object;
|
|
27
|
-
urlType: Object;
|
|
28
|
-
distributionBucketMap: any;
|
|
29
|
-
useDirectS3Type: boolean;
|
|
30
|
-
}): (Object | undefined);
|
|
32
|
+
export function constructOnlineAccessUrl({ file, distEndpoint, bucketTypes, urlType, distributionBucketMap, useDirectS3Type, }: Object): (Object | undefined);
|
|
31
33
|
/**
|
|
32
34
|
* Construct a list of online access urls grouped by link type.
|
|
33
35
|
*
|
|
34
|
-
* @param {Object} params - input parameters
|
|
35
36
|
* @param {Array<Object>} params.files - array of file objects
|
|
36
37
|
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
37
|
-
* @param {
|
|
38
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket name to bucket type
|
|
38
39
|
* @param {string} params.cmrGranuleUrlType - cmrGranuleUrlType from config
|
|
39
40
|
* @param {distributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
40
41
|
* for all distribution bucketss
|
|
@@ -42,14 +43,7 @@ export function constructOnlineAccessUrl({ file, distEndpoint, bucketTypes, urlT
|
|
|
42
43
|
* @returns {Promise<[{URL: string, URLDescription: string}]>} an array of
|
|
43
44
|
* online access url objects grouped by link type.
|
|
44
45
|
*/
|
|
45
|
-
export function constructOnlineAccessUrls({ files, distEndpoint, bucketTypes, cmrGranuleUrlType, distributionBucketMap, useDirectS3Type, }: {
|
|
46
|
-
files: Array<Object>;
|
|
47
|
-
distEndpoint: string;
|
|
48
|
-
bucketTypes: Object;
|
|
49
|
-
cmrGranuleUrlType: string;
|
|
50
|
-
distributionBucketMap: any;
|
|
51
|
-
useDirectS3Type: boolean;
|
|
52
|
-
}): Promise<[{
|
|
46
|
+
export function constructOnlineAccessUrls({ files, distEndpoint, bucketTypes, cmrGranuleUrlType, distributionBucketMap, useDirectS3Type, }: Array<Object>): Promise<[{
|
|
53
47
|
URL: string;
|
|
54
48
|
URLDescription: string;
|
|
55
49
|
}]>;
|
|
@@ -99,7 +93,7 @@ export function granuleToCmrFileObject({ granuleId, files }: {
|
|
|
99
93
|
* @param {string} cmrConfig.certificate - Launchpad certificate
|
|
100
94
|
* @param {string} cmrConfig.username - EDL username
|
|
101
95
|
* @param {string} cmrConfig.passwordSecretName - CMR password secret name
|
|
102
|
-
* @returns {Promise<
|
|
96
|
+
* @returns {Promise<CMRConstructorParams>} object to create CMR instance - contains the
|
|
103
97
|
* provider, clientId, and either launchpad token or EDL username and
|
|
104
98
|
* password
|
|
105
99
|
*/
|
|
@@ -112,7 +106,14 @@ export function getCmrSettings(cmrConfig?: {
|
|
|
112
106
|
certificate: string;
|
|
113
107
|
username: string;
|
|
114
108
|
passwordSecretName: string;
|
|
115
|
-
}): Promise<
|
|
109
|
+
}): Promise<CMRConstructorParams>;
|
|
110
|
+
/**
|
|
111
|
+
* Get collectionId from cmr metadata
|
|
112
|
+
* @param {object} cmrObject - cmr metadata object
|
|
113
|
+
* @param {string} cmrFileName - used to know how to parse this object
|
|
114
|
+
* @returns {string}
|
|
115
|
+
*/
|
|
116
|
+
export function getCMRCollectionId(cmrObject: object, cmrFileName: string): string;
|
|
116
117
|
export function getFileDescription(file: any, urlType?: string): string;
|
|
117
118
|
export function getFilename(file: any): any;
|
|
118
119
|
/**
|
|
@@ -219,7 +220,7 @@ export function publish2CMR(cmrPublishObject: {
|
|
|
219
220
|
username: string;
|
|
220
221
|
password: string;
|
|
221
222
|
token: string;
|
|
222
|
-
}, cmrRevisionId: string):
|
|
223
|
+
}, cmrRevisionId: string): Object;
|
|
223
224
|
/**
|
|
224
225
|
* Update CMR Metadata record with the information contained in updatedFiles
|
|
225
226
|
* @param {Object} params - parameter object
|
|
@@ -229,7 +230,7 @@ export function publish2CMR(cmrPublishObject: {
|
|
|
229
230
|
* @param {string} params.distEndpoint - distribution endpoint URL
|
|
230
231
|
* @param {boolean} params.published - boolean true if the data should be published to
|
|
231
232
|
* the CMR service.
|
|
232
|
-
* @param {
|
|
233
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
233
234
|
* @param {string} params.cmrGranuleUrlType - type of granule CMR url
|
|
234
235
|
* @param {distributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
235
236
|
* for all distribution buckets
|
|
@@ -239,7 +240,9 @@ export function reconcileCMRMetadata({ granuleId, updatedFiles, distEndpoint, pu
|
|
|
239
240
|
updatedFiles: Object;
|
|
240
241
|
distEndpoint: string;
|
|
241
242
|
published: boolean;
|
|
242
|
-
bucketTypes:
|
|
243
|
+
bucketTypes: {
|
|
244
|
+
[key: string]: string;
|
|
245
|
+
};
|
|
243
246
|
cmrGranuleUrlType: string;
|
|
244
247
|
distributionBucketMap: any;
|
|
245
248
|
}): Promise<void | Object>;
|
|
@@ -250,18 +253,25 @@ export function reconcileCMRMetadata({ granuleId, updatedFiles, distEndpoint, pu
|
|
|
250
253
|
* @returns {undefined}
|
|
251
254
|
*/
|
|
252
255
|
export function removeEtagsFromFileObjects(granule: Object): undefined;
|
|
256
|
+
/**
|
|
257
|
+
* Remove granule from CMR.
|
|
258
|
+
*
|
|
259
|
+
* @param {string} granuleUR - the granuleUR
|
|
260
|
+
* @param {CmrCredentials} creds - credentials needed to post to CMR service
|
|
261
|
+
*/
|
|
262
|
+
export function removeFromCMR(granuleUR: string, creds: CmrCredentials): Promise<unknown>;
|
|
253
263
|
/**
|
|
254
264
|
* Modifies cmr metadata file with file's URLs updated to their new locations.
|
|
255
265
|
*
|
|
256
266
|
* @param {Object} params - parameter object
|
|
257
267
|
* @param {string} params.granuleId - granuleId
|
|
258
268
|
* @param {Object} params.cmrFile - cmr xml file to be updated
|
|
259
|
-
* @param {Array<
|
|
269
|
+
* @param {Array<ApiFile>} params.files - array of file objects
|
|
260
270
|
* @param {string} params.distEndpoint - distribution enpoint from config
|
|
261
271
|
* @param {boolean} params.published - indicate if publish is needed
|
|
262
|
-
* @param {
|
|
272
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
263
273
|
* @param {string} params.cmrGranuleUrlType - type of granule CMR url
|
|
264
|
-
* @param {
|
|
274
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
265
275
|
* mapping for all distribution buckets
|
|
266
276
|
* @returns {Promise<Object>} CMR file object with the `etag` of the newly
|
|
267
277
|
* updated metadata file
|
|
@@ -269,13 +279,61 @@ export function removeEtagsFromFileObjects(granule: Object): undefined;
|
|
|
269
279
|
export function updateCMRMetadata({ granuleId, cmrFile, files, distEndpoint, published, bucketTypes, cmrGranuleUrlType, distributionBucketMap, }: {
|
|
270
280
|
granuleId: string;
|
|
271
281
|
cmrFile: Object;
|
|
272
|
-
files: Array<
|
|
282
|
+
files: Array<ApiFile>;
|
|
273
283
|
distEndpoint: string;
|
|
274
284
|
published: boolean;
|
|
275
|
-
bucketTypes:
|
|
285
|
+
bucketTypes: {
|
|
286
|
+
[key: string]: string;
|
|
287
|
+
};
|
|
276
288
|
cmrGranuleUrlType: string;
|
|
277
|
-
distributionBucketMap:
|
|
289
|
+
distributionBucketMap: DistributionBucketMap;
|
|
278
290
|
}): Promise<Object>;
|
|
291
|
+
/**
|
|
292
|
+
* Update the Echo10 cmr metadata object to have corrected urls
|
|
293
|
+
*
|
|
294
|
+
* @param {Object} params.metadataObject - xml cmr metadata object
|
|
295
|
+
* @param {Array<Object>} params.files - files with which to update the cmr metadata
|
|
296
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
297
|
+
* @param {string} params.cmrGranuleUrlType
|
|
298
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
299
|
+
* mapping for all distribution buckets
|
|
300
|
+
* @returns {Object}
|
|
301
|
+
*/
|
|
302
|
+
export function updateEcho10XMLMetadataObject({ metadataObject, files, distEndpoint, bucketTypes, cmrGranuleUrlType, distributionBucketMap, }: Object): Object;
|
|
303
|
+
/**
|
|
304
|
+
* Update the UMMG cmr metadata object to have corrected urls
|
|
305
|
+
*
|
|
306
|
+
* @param {Object} params.metadataObject - ummg cmr metadata object
|
|
307
|
+
* @param {Array<ApiFile>} params.files - files with which to update the cmr metadata
|
|
308
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
309
|
+
* @param {string} params.cmrGranuleUrlType
|
|
310
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
311
|
+
* mapping for all distribution buckets
|
|
312
|
+
* @returns {Object}
|
|
313
|
+
*/
|
|
314
|
+
export function updateUMMGMetadataObject({ metadataObject, files, distEndpoint, bucketTypes, cmrGranuleUrlType, distributionBucketMap, }: Object): Object;
|
|
315
|
+
/**
|
|
316
|
+
* Update collection in an ECHO10 cmr metadata object
|
|
317
|
+
*
|
|
318
|
+
* @param {object} cmrObject - CMR metadata object
|
|
319
|
+
* @param {{ name: string, version: string }} collection - collection name and version to update to
|
|
320
|
+
* @returns {object}
|
|
321
|
+
*/
|
|
322
|
+
export function setECHO10Collection(cmrObject: object, collection: {
|
|
323
|
+
name: string;
|
|
324
|
+
version: string;
|
|
325
|
+
}): object;
|
|
326
|
+
/**
|
|
327
|
+
* Update collection in an UMMG cmr metadata object
|
|
328
|
+
*
|
|
329
|
+
* @param {object} cmrObject - CMR metadata object
|
|
330
|
+
* @param {{ name: string, version: string }} collection - collection name and version to update to
|
|
331
|
+
* @returns {object}
|
|
332
|
+
*/
|
|
333
|
+
export function setUMMGCollection(cmrObject: object, collection: {
|
|
334
|
+
name: string;
|
|
335
|
+
version: string;
|
|
336
|
+
}): object;
|
|
279
337
|
/**
|
|
280
338
|
* Updates CMR xml file with 'xml' string
|
|
281
339
|
*
|
package/cmr-utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cmr-utils.d.ts","sourceRoot":"","sources":["src/cmr-utils.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cmr-utils.d.ts","sourceRoot":"","sources":["src/cmr-utils.js"],"names":[],"mappings":"6BAkDa;IACZ,QAAY,EAAE,MAAM,CAAC;IACrB,QAAY,EAAE,MAAM,CAAC;IACrB,QAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAS,CAAC,EAAE,MAAM,CAAA;CACf;mCAhBS,OAAO,yBAAyB,EAAE,oBAAoB;oCACtD,OAAO,wCAAwC,EAAE,qBAAqB;;AAwWnF;;;;;;GAMG;AACH,+CAJW,MAAM,SACN,MAAM,GACJ,MAAM,CASlB;AA7TD,gFAA+G;AA+X/G;;;;;;;;;;;GAWG;AACH,gIATW,MAAM,GAOJ,CAAC,MAAM,GAAG,SAAS,CAAC,CAyBhC;AAED;;;;;;;;;;;;GAYG;AACH,4IAVW,MAAM,MAAM,CAAC,GAOX,QAAQ,CAAC;IAAC,KAAK,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAC,CAAC,CAAC,CA0C5D;AA6RD,8DAWC;AApaD;;;;;;;;;;GAUG;AACH;IAP0B,IAAI,EAAnB,MAAM;IACS,YAAY,EAA3B,MAAM;IACS,OAAO,EAAtB,MAAM;IACwB,qBAAqB;IAEjD,CAAC,MAAM,GAAG,SAAS,CAAC,CAwBhC;AArWD;;;;;;;;;;;GAWG;AACH;IAPkC,KAAK,EAA5B,MAAM,MAAM,CAAC;IACG,SAAS,EAAzB,MAAM;2BAGJ,MAAM,MAAM,CAAC,CAgBzB;AAyoBD;;;;;;;;;;;;;;;EAeE;AACF;IAZ6B,aAAa,EAA/B,MAAM;IACY,QAAQ,EAA1B,MAAM;IACY,QAAQ,EAA1B,MAAM;IACY,oBAAoB,EAAtC,MAAM;IACY,GAAG,EAArB,MAAM;IACY,WAAW,EAA7B,MAAM;IACY,QAAQ,EAA1B,MAAM;IACY,kBAAkB,EAApC,MAAM;IACJ,QAAQ,oBAAoB,CAAC,CA6CzC;AAofD;;;;;GAKG;AACH,8CAJW,MAAM,eACN,MAAM,GACJ,MAAM,CAgClB;AA3xCD,wEAMC;AAfD,4CAOC;AAikCD;;;;;;;GAOG;AACH,gDALW,MAAM,GACJ,QAAQ,MAAM,CAAC,CAuI3B;AAlLD;;;;;;GAMG;AACH,6DAJW,MAAM,MAAM,CAAC,GACX,QAAQ,MAAM,CAAC,CAoB3B;AAhkCD,+CAKC;AA6jCD;;;;;;GAMG;AACH,kDAJW,MAAM,gBACN,MAAM,GACJ,MAAM,CAMlB;AA1/BD;;;;;;;;GAQG;AACH,mDANW,MAAM,MAAM,CAAC,0BAIX,MAAM,MAAM,CAAC,CAIzB;AA5DD;;;;;GAKG;AACH,sCAHW,MAAM,GACJ,OAAO,CAKnB;AAfD,kDAE+B;AAH/B,qDAAuE;AAHvE,qDAAmE;AAqBnE;;;;;GAKG;AACH,sCAHW,MAAM,GACJ,OAAO,CAKnB;AA5BD,kDAAiE;AADjE,mDAAkE;AAmVlE;;;;;;GAMG;AACH,oCAJW,MAAM,EAAE,GAEN,MAAM,CASlB;AA9GD;;;;;;;;;;;;GAYG;AACH,uDATW,MAAM,8BAGJ,QAAQ,MAAM,CAAC,CAgB3B;AApID;;;;;;;;;;;;;;;;GAgBG;AACH;IAZoC,QAAQ,EAAjC,MAAM;IACmB,cAAc,EAAvC,MAAM;IACmB,SAAS,EAAlC,MAAM;;IAEQ,QAAQ,EAAtB,MAAM;IACQ,QAAQ,EAAtB,MAAM;IACQ,QAAQ,EAAtB,MAAM;IACQ,QAAQ,EAAtB,MAAM;IACQ,KAAK,EAAnB,MAAM;kBACN,MAAM,UAchB;AA4yBD;;;;;;;;;;;;;GAaG;AACH;eAXW,MAAM;kBACN,MAAM;kBAEN,MAAM;eACN,OAAO;;;;uBAGP,MAAM;;2BA8BhB;AAhrBD;;;;;GAKG;AACH,oDAHW,MAAM,GACJ,SAAS,CAOrB;AA7KD;;;;;GAKG;AACH,yCAHW,MAAM,SACN,cAAc,oBAKxB;AAguBD;;;;;;;;;;;;;;;GAeG;AACH;eAZW,MAAM;aACN,MAAM;WACN,MAAM,OAAO,CAAC;kBACd,MAAM;eACN,OAAO;;;;uBAEP,MAAM;2BACN,qBAAqB;IAEnB,QAAQ,MAAM,CAAC,CAkD3B;AA7JD;;;;;;;;;;GAUG;AACH,+IARW,MAAM,GAMJ,MAAM,CA4ClB;AA/OD;;;;;;;;;;GAUG;AACH,0IARW,MAAM,GAMJ,MAAM,CA2BlB;AA0iBD;;;;;;GAMG;AACH,+CAJW,MAAM,cACN;IAAE,MAAM,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC/B,MAAM,CAiBlB;AAED;;;;;;GAMG;AACH,6CAJW,MAAM,cACN;IAAE,MAAM,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC/B,MAAM,CAYlB;AAneD;;;;;;GAMG;AACH,yCAJY,MAAM,WACN,MAAM,gBAejB;AA7MD;;;;;;GAMG;AACH,sDAJW,MAAM,WACN,MAAM,gBAehB"}
|
package/cmr-utils.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
+
//@ts-check
|
|
2
3
|
const got = require('got');
|
|
3
4
|
const get = require('lodash/get');
|
|
4
5
|
const pick = require('lodash/pick');
|
|
5
6
|
const set = require('lodash/set');
|
|
7
|
+
const cloneDeep = require('lodash/cloneDeep');
|
|
6
8
|
const { promisify } = require('util');
|
|
7
9
|
const js2xmlParser = require('js2xmlparser');
|
|
8
10
|
const path = require('path');
|
|
@@ -18,8 +20,26 @@ const errors = require('@cumulus/errors');
|
|
|
18
20
|
const { CMR, getSearchUrl, ummVersion } = require('@cumulus/cmr-client');
|
|
19
21
|
const { constructDistributionUrl } = require('@cumulus/distribution-utils');
|
|
20
22
|
const { getBucketAccessUrl } = require('@cumulus/cmr-client/getUrl');
|
|
23
|
+
const { constructCollectionId } = require('@cumulus/message/Collections');
|
|
21
24
|
const { xmlParseOptions, ummVersionToMetadataFormat, } = require('./utils');
|
|
25
|
+
/* eslint-disable max-len */
|
|
26
|
+
/**
|
|
27
|
+
* @typedef {import('@cumulus/cmr-client/CMR').CMRConstructorParams} CMRConstructorParams
|
|
28
|
+
* @typedef {import('@cumulus/distribution-utils/dist/types').DistributionBucketMap} DistributionBucketMap
|
|
29
|
+
* @typedef {import('@cumulus/types').ApiFile} ApiFile
|
|
30
|
+
*/
|
|
31
|
+
/* eslint-enable max-len */
|
|
22
32
|
const log = new Logger({ sender: '@cumulus/cmrjs/src/cmr-utils' });
|
|
33
|
+
const s3CredsEndpoint = 's3credentials';
|
|
34
|
+
/**
|
|
35
|
+
* @typedef {{
|
|
36
|
+
* provider: string,
|
|
37
|
+
* clientId: string,
|
|
38
|
+
* username?: string,
|
|
39
|
+
* password?: string,
|
|
40
|
+
* token?: string
|
|
41
|
+
* }} CmrCredentials
|
|
42
|
+
*/
|
|
23
43
|
function getS3KeyOfFile(file) {
|
|
24
44
|
if (file.filename)
|
|
25
45
|
return parseS3Uri(file.filename).Key;
|
|
@@ -199,18 +219,28 @@ async function publishUMMGJSON2CMR(cmrFile, cmrClient, revisionId) {
|
|
|
199
219
|
* @param {string} cmrRevisionId - Optional CMR Revision ID
|
|
200
220
|
* if not provided, CMR username and password are used to get a cmr token
|
|
201
221
|
*/
|
|
202
|
-
|
|
222
|
+
function publish2CMR(cmrPublishObject, creds, cmrRevisionId) {
|
|
203
223
|
const cmrClient = new CMR(creds);
|
|
204
224
|
const cmrFileName = getFilename(cmrPublishObject);
|
|
205
225
|
// choose xml or json and do the things.
|
|
206
226
|
if (isECHO10Filename(cmrFileName)) {
|
|
207
|
-
return
|
|
227
|
+
return publishECHO10XML2CMR(cmrPublishObject, cmrClient, cmrRevisionId);
|
|
208
228
|
}
|
|
209
229
|
if (isUMMGFilename(cmrFileName)) {
|
|
210
|
-
return
|
|
230
|
+
return publishUMMGJSON2CMR(cmrPublishObject, cmrClient, cmrRevisionId);
|
|
211
231
|
}
|
|
212
232
|
throw new Error(`invalid cmrPublishObject passed to publis2CMR ${JSON.stringify(cmrPublishObject)}`);
|
|
213
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* Remove granule from CMR.
|
|
236
|
+
*
|
|
237
|
+
* @param {string} granuleUR - the granuleUR
|
|
238
|
+
* @param {CmrCredentials} creds - credentials needed to post to CMR service
|
|
239
|
+
*/
|
|
240
|
+
async function removeFromCMR(granuleUR, creds) {
|
|
241
|
+
const cmrClient = new CMR(creds);
|
|
242
|
+
return await cmrClient.deleteGranule(granuleUR);
|
|
243
|
+
}
|
|
214
244
|
/**
|
|
215
245
|
* Returns the S3 object identified by the specified S3 URI and (optional)
|
|
216
246
|
* entity tag, retrying up to 5 times, if necessary.
|
|
@@ -405,10 +435,9 @@ function generateFileUrl({ file, distEndpoint, urlType = 'distribution', distrib
|
|
|
405
435
|
/**
|
|
406
436
|
* Construct online access url for a given file and a url type.
|
|
407
437
|
*
|
|
408
|
-
* @param {Object} params - input parameters
|
|
409
438
|
* @param {Object} params.file - file object
|
|
410
439
|
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
411
|
-
* @param {
|
|
440
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket name to bucket type
|
|
412
441
|
* @param {Object} params.urlType - url type, distribution or s3
|
|
413
442
|
* @param {distributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
414
443
|
* for all distribution bucketss
|
|
@@ -435,10 +464,9 @@ function constructOnlineAccessUrl({ file, distEndpoint, bucketTypes, urlType = '
|
|
|
435
464
|
/**
|
|
436
465
|
* Construct a list of online access urls grouped by link type.
|
|
437
466
|
*
|
|
438
|
-
* @param {Object} params - input parameters
|
|
439
467
|
* @param {Array<Object>} params.files - array of file objects
|
|
440
468
|
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
441
|
-
* @param {
|
|
469
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket name to bucket type
|
|
442
470
|
* @param {string} params.cmrGranuleUrlType - cmrGranuleUrlType from config
|
|
443
471
|
* @param {distributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
444
472
|
* for all distribution bucketss
|
|
@@ -484,16 +512,15 @@ function constructOnlineAccessUrls({ files, distEndpoint, bucketTypes, cmrGranul
|
|
|
484
512
|
* @param {Object} params - input parameters
|
|
485
513
|
* @param {Array<Object>} params.files - array of file objects
|
|
486
514
|
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
487
|
-
* @param {
|
|
488
|
-
* @param {string} params.s3CredsEndpoint - Optional endpoint for acquiring temporary s3 creds
|
|
515
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
489
516
|
* @param {string} params.cmrGranuleUrlType - cmrGranuleUrlType from config
|
|
490
|
-
* @param {
|
|
517
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
491
518
|
* mapping for all distribution buckets
|
|
492
519
|
* @param {boolean} params.useDirectS3Type - indicate if direct s3 access type is used
|
|
493
520
|
* @returns {Promise<[{URL: string, string, Description: string, Type: string}]>}
|
|
494
521
|
* an array of online access url objects
|
|
495
522
|
*/
|
|
496
|
-
function constructRelatedUrls({ files, distEndpoint, bucketTypes,
|
|
523
|
+
function constructRelatedUrls({ files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, useDirectS3Type = false, }) {
|
|
497
524
|
const credsUrl = urljoin(distEndpoint, s3CredsEndpoint);
|
|
498
525
|
const s3CredentialsObject = getS3CredentialsObject(credsUrl);
|
|
499
526
|
const cmrUrlObjects = constructOnlineAccessUrls({
|
|
@@ -510,7 +537,7 @@ function constructRelatedUrls({ files, distEndpoint, bucketTypes, s3CredsEndpoin
|
|
|
510
537
|
/**
|
|
511
538
|
* Create a list of URL objects that should not appear under onlineAccess in the CMR metadata.
|
|
512
539
|
* @param {Array<Object>} files - array of updated file objects
|
|
513
|
-
* @param {
|
|
540
|
+
* @param {{ [key: string]: string }} bucketTypes - map of buckets name to bucket types
|
|
514
541
|
* @returns {Array<Object>} array of files to be omitted in cmr's OnlineAccessURLs
|
|
515
542
|
*/
|
|
516
543
|
function onlineAccessURLsToRemove(files, bucketTypes) {
|
|
@@ -601,6 +628,34 @@ function shouldUseDirectS3Type(metadataObject) {
|
|
|
601
628
|
}
|
|
602
629
|
return false;
|
|
603
630
|
}
|
|
631
|
+
/**
|
|
632
|
+
* Update the UMMG cmr metadata object to have corrected urls
|
|
633
|
+
*
|
|
634
|
+
* @param {Object} params.metadataObject - ummg cmr metadata object
|
|
635
|
+
* @param {Array<ApiFile>} params.files - files with which to update the cmr metadata
|
|
636
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
637
|
+
* @param {string} params.cmrGranuleUrlType
|
|
638
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
639
|
+
* mapping for all distribution buckets
|
|
640
|
+
* @returns {Object}
|
|
641
|
+
*/
|
|
642
|
+
function updateUMMGMetadataObject({ metadataObject, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
643
|
+
const updatedMetadataObject = cloneDeep(metadataObject);
|
|
644
|
+
const useDirectS3Type = shouldUseDirectS3Type(updatedMetadataObject);
|
|
645
|
+
const newURLs = constructRelatedUrls({
|
|
646
|
+
files,
|
|
647
|
+
distEndpoint,
|
|
648
|
+
bucketTypes,
|
|
649
|
+
cmrGranuleUrlType,
|
|
650
|
+
distributionBucketMap,
|
|
651
|
+
useDirectS3Type,
|
|
652
|
+
});
|
|
653
|
+
const removedURLs = onlineAccessURLsToRemove(files, bucketTypes);
|
|
654
|
+
const originalURLs = get(updatedMetadataObject, 'RelatedUrls', []);
|
|
655
|
+
const mergedURLs = mergeURLs(originalURLs, newURLs, removedURLs);
|
|
656
|
+
set(updatedMetadataObject, 'RelatedUrls', mergedURLs);
|
|
657
|
+
return updatedMetadataObject;
|
|
658
|
+
}
|
|
604
659
|
/**
|
|
605
660
|
* After files are moved, create new online access URLs and then update the S3
|
|
606
661
|
* UMMG cmr.json file with this information.
|
|
@@ -609,9 +664,9 @@ function shouldUseDirectS3Type(metadataObject) {
|
|
|
609
664
|
* @param {Object} params.cmrFile - cmr.json file whose contents will be updated.
|
|
610
665
|
* @param {Array<Object>} params.files - array of moved file objects.
|
|
611
666
|
* @param {string} params.distEndpoint - distribution endpoint form config.
|
|
612
|
-
* @param {
|
|
667
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
613
668
|
* @param {string} params.cmrGranuleUrlType - cmrGranuleUrlType from config
|
|
614
|
-
* @param {
|
|
669
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
615
670
|
* mapping for all distribution buckets
|
|
616
671
|
* @returns {Promise<{ metadataObject: Object, etag: string}>} an object
|
|
617
672
|
* containing a `metadataObject` (the updated UMMG metadata object) and the
|
|
@@ -620,21 +675,16 @@ function shouldUseDirectS3Type(metadataObject) {
|
|
|
620
675
|
async function updateUMMGMetadata({ cmrFile, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
621
676
|
const filename = getS3UrlOfFile(cmrFile);
|
|
622
677
|
const metadataObject = await metadataObjectFromCMRJSONFile(filename);
|
|
623
|
-
const
|
|
624
|
-
|
|
678
|
+
const updatedMetadataObject = updateUMMGMetadataObject({
|
|
679
|
+
metadataObject,
|
|
625
680
|
files,
|
|
626
681
|
distEndpoint,
|
|
627
682
|
bucketTypes,
|
|
628
683
|
cmrGranuleUrlType,
|
|
629
684
|
distributionBucketMap,
|
|
630
|
-
useDirectS3Type,
|
|
631
685
|
});
|
|
632
|
-
const
|
|
633
|
-
|
|
634
|
-
const mergedURLs = mergeURLs(originalURLs, newURLs, removedURLs);
|
|
635
|
-
set(metadataObject, 'RelatedUrls', mergedURLs);
|
|
636
|
-
const { ETag: etag } = await uploadUMMGJSONCMRFile(metadataObject, cmrFile);
|
|
637
|
-
return { metadataObject, etag };
|
|
686
|
+
const { ETag: etag } = await uploadUMMGJSONCMRFile(updatedMetadataObject, cmrFile);
|
|
687
|
+
return { metadataObject: updatedMetadataObject, etag };
|
|
638
688
|
}
|
|
639
689
|
/**
|
|
640
690
|
* Helper to build an CMR settings object, used to initialize CMR.
|
|
@@ -648,7 +698,7 @@ async function updateUMMGMetadata({ cmrFile, files, distEndpoint, bucketTypes, c
|
|
|
648
698
|
* @param {string} cmrConfig.certificate - Launchpad certificate
|
|
649
699
|
* @param {string} cmrConfig.username - EDL username
|
|
650
700
|
* @param {string} cmrConfig.passwordSecretName - CMR password secret name
|
|
651
|
-
* @returns {Promise<
|
|
701
|
+
* @returns {Promise<CMRConstructorParams>} object to create CMR instance - contains the
|
|
652
702
|
* provider, clientId, and either launchpad token or EDL username and
|
|
653
703
|
* password
|
|
654
704
|
*/
|
|
@@ -734,23 +784,17 @@ function buildMergedEchoURLObject(URLlist = [], originalURLlist = [], removedURL
|
|
|
734
784
|
return mergeURLs(originalURLlist, filteredURLObjectList, removedURLs);
|
|
735
785
|
}
|
|
736
786
|
/**
|
|
737
|
-
*
|
|
738
|
-
* the S3 ECHO10 CMR XML file with this information.
|
|
787
|
+
* Update the Echo10 cmr metadata object to have corrected urls
|
|
739
788
|
*
|
|
740
|
-
* @param {Object} params -
|
|
741
|
-
* @param {Object} params.
|
|
742
|
-
* @param {
|
|
743
|
-
* @param {string} params.
|
|
744
|
-
* @param {
|
|
745
|
-
* @param {Object} params.distributionBucketMap - Object with bucket:tea-path
|
|
789
|
+
* @param {Object} params.metadataObject - xml cmr metadata object
|
|
790
|
+
* @param {Array<Object>} params.files - files with which to update the cmr metadata
|
|
791
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
792
|
+
* @param {string} params.cmrGranuleUrlType
|
|
793
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
746
794
|
* mapping for all distribution buckets
|
|
747
|
-
* @returns {
|
|
748
|
-
* containing a `metadataObject` and the `etag` of the uploaded CMR file
|
|
795
|
+
* @returns {Object}
|
|
749
796
|
*/
|
|
750
|
-
|
|
751
|
-
// add/replace the OnlineAccessUrls
|
|
752
|
-
const filename = getS3UrlOfFile(cmrFile);
|
|
753
|
-
const metadataObject = await metadataObjectFromCMRXMLFile(filename);
|
|
797
|
+
function updateEcho10XMLMetadataObject({ metadataObject, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
754
798
|
const metadataGranule = metadataObject.Granule;
|
|
755
799
|
const updatedGranule = { ...metadataGranule };
|
|
756
800
|
const originalOnlineAccessURLs = [].concat(get(metadataGranule, 'OnlineAccessURLs.OnlineAccessURL', []));
|
|
@@ -772,10 +816,40 @@ async function updateEcho10XMLMetadata({ cmrFile, files, distEndpoint, bucketTyp
|
|
|
772
816
|
set(updatedGranule, 'OnlineAccessURLs.OnlineAccessURL', mergedOnlineAccessURLs);
|
|
773
817
|
set(updatedGranule, 'OnlineResources.OnlineResource', mergedOnlineResources);
|
|
774
818
|
set(updatedGranule, 'AssociatedBrowseImageUrls.ProviderBrowseUrl', mergedAssociatedBrowse);
|
|
775
|
-
|
|
776
|
-
|
|
819
|
+
return {
|
|
820
|
+
...metadataObject,
|
|
821
|
+
Granule: updatedGranule,
|
|
822
|
+
};
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* After files are moved, creates new online access URLs and then updates
|
|
826
|
+
* the S3 ECHO10 CMR XML file with this information.
|
|
827
|
+
*
|
|
828
|
+
* @param {Object} params - parameter object
|
|
829
|
+
* @param {Object} params.cmrFile - cmr xml file object to be updated
|
|
830
|
+
* @param {Array<Object>} params.files - array of file objects
|
|
831
|
+
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
832
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
833
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
834
|
+
* mapping for all distribution buckets
|
|
835
|
+
* @returns {Promise<{ metadataObject: Object, etag: string}>} an object
|
|
836
|
+
* containing a `metadataObject` and the `etag` of the uploaded CMR file
|
|
837
|
+
*/
|
|
838
|
+
async function updateEcho10XMLMetadata({ cmrFile, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
839
|
+
// add/replace the OnlineAccessUrls
|
|
840
|
+
const filename = getS3UrlOfFile(cmrFile);
|
|
841
|
+
const metadataObject = await metadataObjectFromCMRXMLFile(filename);
|
|
842
|
+
const updatedMetadataObject = updateEcho10XMLMetadataObject({
|
|
843
|
+
metadataObject,
|
|
844
|
+
files,
|
|
845
|
+
distEndpoint,
|
|
846
|
+
bucketTypes,
|
|
847
|
+
cmrGranuleUrlType,
|
|
848
|
+
distributionBucketMap,
|
|
849
|
+
});
|
|
850
|
+
const xml = generateEcho10XMLString(updatedMetadataObject.Granule);
|
|
777
851
|
const { ETag: etag } = await uploadEcho10CMRFile(xml, cmrFile);
|
|
778
|
-
return { metadataObject, etag };
|
|
852
|
+
return { metadataObject: updatedMetadataObject, etag };
|
|
779
853
|
}
|
|
780
854
|
/**
|
|
781
855
|
* Modifies cmr metadata file with file's URLs updated to their new locations.
|
|
@@ -783,12 +857,12 @@ async function updateEcho10XMLMetadata({ cmrFile, files, distEndpoint, bucketTyp
|
|
|
783
857
|
* @param {Object} params - parameter object
|
|
784
858
|
* @param {string} params.granuleId - granuleId
|
|
785
859
|
* @param {Object} params.cmrFile - cmr xml file to be updated
|
|
786
|
-
* @param {Array<
|
|
860
|
+
* @param {Array<ApiFile>} params.files - array of file objects
|
|
787
861
|
* @param {string} params.distEndpoint - distribution enpoint from config
|
|
788
862
|
* @param {boolean} params.published - indicate if publish is needed
|
|
789
|
-
* @param {
|
|
863
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
790
864
|
* @param {string} params.cmrGranuleUrlType - type of granule CMR url
|
|
791
|
-
* @param {
|
|
865
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
792
866
|
* mapping for all distribution buckets
|
|
793
867
|
* @returns {Promise<Object>} CMR file object with the `etag` of the newly
|
|
794
868
|
* updated metadata file
|
|
@@ -836,7 +910,7 @@ async function updateCMRMetadata({ granuleId, cmrFile, files, distEndpoint, publ
|
|
|
836
910
|
* @param {string} params.distEndpoint - distribution endpoint URL
|
|
837
911
|
* @param {boolean} params.published - boolean true if the data should be published to
|
|
838
912
|
* the CMR service.
|
|
839
|
-
* @param {
|
|
913
|
+
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
840
914
|
* @param {string} params.cmrGranuleUrlType - type of granule CMR url
|
|
841
915
|
* @param {distributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
842
916
|
* for all distribution buckets
|
|
@@ -1003,6 +1077,63 @@ async function getGranuleTemporalInfo(granule) {
|
|
|
1003
1077
|
}
|
|
1004
1078
|
return {};
|
|
1005
1079
|
}
|
|
1080
|
+
/**
|
|
1081
|
+
* Update collection in an ECHO10 cmr metadata object
|
|
1082
|
+
*
|
|
1083
|
+
* @param {object} cmrObject - CMR metadata object
|
|
1084
|
+
* @param {{ name: string, version: string }} collection - collection name and version to update to
|
|
1085
|
+
* @returns {object}
|
|
1086
|
+
*/
|
|
1087
|
+
const setECHO10Collection = (cmrObject, collection) => {
|
|
1088
|
+
const cmrObjectCopy = cloneDeep(cmrObject);
|
|
1089
|
+
if (!cmrObjectCopy.Granule) {
|
|
1090
|
+
throw new errors.ValidationError(`invalid ECHO10 cmr metadata ${JSON.stringify(cmrObjectCopy)}, must have granule tag`);
|
|
1091
|
+
}
|
|
1092
|
+
cmrObjectCopy.Granule.Collection = {
|
|
1093
|
+
ShortName: collection.name,
|
|
1094
|
+
VersionId: collection.version,
|
|
1095
|
+
};
|
|
1096
|
+
return cmrObjectCopy;
|
|
1097
|
+
};
|
|
1098
|
+
/**
|
|
1099
|
+
* Update collection in an UMMG cmr metadata object
|
|
1100
|
+
*
|
|
1101
|
+
* @param {object} cmrObject - CMR metadata object
|
|
1102
|
+
* @param {{ name: string, version: string }} collection - collection name and version to update to
|
|
1103
|
+
* @returns {object}
|
|
1104
|
+
*/
|
|
1105
|
+
const setUMMGCollection = (cmrObject, collection) => {
|
|
1106
|
+
const cmrObjectCopy = cloneDeep(cmrObject);
|
|
1107
|
+
set(cmrObjectCopy, 'CollectionReference', {
|
|
1108
|
+
ShortName: collection.name,
|
|
1109
|
+
Version: collection.version,
|
|
1110
|
+
});
|
|
1111
|
+
return cmrObjectCopy;
|
|
1112
|
+
};
|
|
1113
|
+
/**
|
|
1114
|
+
* Get collectionId from cmr metadata
|
|
1115
|
+
* @param {object} cmrObject - cmr metadata object
|
|
1116
|
+
* @param {string} cmrFileName - used to know how to parse this object
|
|
1117
|
+
* @returns {string}
|
|
1118
|
+
*/
|
|
1119
|
+
const getCMRCollectionId = (cmrObject, cmrFileName) => {
|
|
1120
|
+
if (isUMMGFilename(cmrFileName)) {
|
|
1121
|
+
const collectionName = cmrObject.CollectionReference?.ShortName;
|
|
1122
|
+
const collectionVersion = cmrObject.CollectionReference?.Version;
|
|
1123
|
+
if (!(collectionName && collectionVersion)) {
|
|
1124
|
+
throw new errors.ValidationError('UMMG metadata file has invalid collection configuration' +
|
|
1125
|
+
` ${JSON.stringify(cmrObject.CollectionReference)}`);
|
|
1126
|
+
}
|
|
1127
|
+
return constructCollectionId(collectionName, collectionVersion);
|
|
1128
|
+
}
|
|
1129
|
+
const collectionName = cmrObject.Granule?.Collection?.ShortName;
|
|
1130
|
+
const collectionVersion = cmrObject.Granule?.Collection?.VersionId;
|
|
1131
|
+
if (!(collectionName && collectionVersion)) {
|
|
1132
|
+
throw new errors.ValidationError('ECHO10 metadata file has invalid collection configuration' +
|
|
1133
|
+
` ${JSON.stringify(cmrObject.Granule?.Collection)}`);
|
|
1134
|
+
}
|
|
1135
|
+
return constructCollectionId(collectionName, collectionVersion);
|
|
1136
|
+
};
|
|
1006
1137
|
module.exports = {
|
|
1007
1138
|
addEtagsToFileObjects,
|
|
1008
1139
|
constructCmrConceptLink,
|
|
@@ -1012,6 +1143,7 @@ module.exports = {
|
|
|
1012
1143
|
generateFileUrl,
|
|
1013
1144
|
granuleToCmrFileObject,
|
|
1014
1145
|
getCmrSettings,
|
|
1146
|
+
getCMRCollectionId,
|
|
1015
1147
|
getFileDescription,
|
|
1016
1148
|
getFilename,
|
|
1017
1149
|
getGranuleTemporalInfo,
|
|
@@ -1031,7 +1163,12 @@ module.exports = {
|
|
|
1031
1163
|
publish2CMR,
|
|
1032
1164
|
reconcileCMRMetadata,
|
|
1033
1165
|
removeEtagsFromFileObjects,
|
|
1166
|
+
removeFromCMR,
|
|
1034
1167
|
updateCMRMetadata,
|
|
1168
|
+
updateEcho10XMLMetadataObject,
|
|
1169
|
+
updateUMMGMetadataObject,
|
|
1170
|
+
setECHO10Collection,
|
|
1171
|
+
setUMMGCollection,
|
|
1035
1172
|
uploadEcho10CMRFile,
|
|
1036
1173
|
uploadUMMGJSONCMRFile,
|
|
1037
1174
|
};
|