@cumulus/cmrjs 20.3.1 → 21.0.0-echo10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cmr-utils.d.ts +371 -108
- package/cmr-utils.d.ts.map +1 -1
- package/cmr-utils.js +263 -101
- package/cmr-utils.js.map +1 -1
- package/echo10Modifiers.d.ts +23 -0
- package/echo10Modifiers.d.ts.map +1 -0
- package/echo10Modifiers.js +69 -0
- package/echo10Modifiers.js.map +1 -0
- package/package.json +12 -12
- package/src/cmr-utils.js +333 -125
- package/src/echo10Modifiers.ts +91 -0
- package/src/ummgModifiers.ts +71 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/ummgModifiers.d.ts +35 -0
- package/ummgModifiers.d.ts.map +1 -0
- package/ummgModifiers.js +45 -0
- package/ummgModifiers.js.map +1 -0
package/cmr-utils.js
CHANGED
|
@@ -22,15 +22,24 @@ const { constructDistributionUrl } = require('@cumulus/distribution-utils');
|
|
|
22
22
|
const { getBucketAccessUrl } = require('@cumulus/cmr-client/getUrl');
|
|
23
23
|
const { constructCollectionId } = require('@cumulus/message/Collections');
|
|
24
24
|
const { xmlParseOptions, ummVersionToMetadataFormat, } = require('./utils');
|
|
25
|
+
const { updateEcho10XMLGranuleUrAndGranuleIdentifier } = require('./echo10Modifiers');
|
|
26
|
+
const { updateUMMGGranuleURAndGranuleIdentifier } = require('./ummgModifiers');
|
|
25
27
|
/* eslint-disable max-len */
|
|
26
28
|
/**
|
|
27
29
|
* @typedef {import('@cumulus/cmr-client/CMR').CMRConstructorParams} CMRConstructorParams
|
|
28
30
|
* @typedef {import('@cumulus/distribution-utils/dist/types').DistributionBucketMap} DistributionBucketMap
|
|
29
|
-
* @typedef {import('@cumulus/types').
|
|
31
|
+
* @typedef {import('@cumulus/types').ApiFileGranuleIdOptional} ApiFileGranuleIdOptional
|
|
32
|
+
* @typedef { ApiFileGranuleIdOptional & { filepath?: string }} ApiFileWithFilePath
|
|
30
33
|
*/
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
+
/**
|
|
35
|
+
* @typedef {Object} CmrFile
|
|
36
|
+
* @property {string} bucket - The S3 bucket name
|
|
37
|
+
* @property {string} key - The S3 key for the metadata file
|
|
38
|
+
* @property {string} granuleId - The granule ID associated with the file
|
|
39
|
+
* @property {string} [etag] - Optional entity tag for file versioning
|
|
40
|
+
*/
|
|
41
|
+
/**
|
|
42
|
+
|
|
34
43
|
/**
|
|
35
44
|
* @typedef {{
|
|
36
45
|
* provider: string,
|
|
@@ -40,6 +49,30 @@ const s3CredsEndpoint = 's3credentials';
|
|
|
40
49
|
* token?: string
|
|
41
50
|
* }} CmrCredentials
|
|
42
51
|
*/
|
|
52
|
+
/**
|
|
53
|
+
* @typedef {Object} Echo10URLObject
|
|
54
|
+
* @property {string} URL
|
|
55
|
+
* @property {string} [Type]
|
|
56
|
+
* @property {string} [Description]
|
|
57
|
+
* @property {string} [URLDescription]
|
|
58
|
+
*/
|
|
59
|
+
/**
|
|
60
|
+
* @typedef {Object} Echo10MetadataObject
|
|
61
|
+
* @property {Object} Granule - The root ECHO10 granule object
|
|
62
|
+
* @property {{ OnlineAccessURL?: Echo10URLObject[] }} [Granule.OnlineAccessURLs]
|
|
63
|
+
* @property {{ OnlineResource?: Echo10URLObject[] }} [Granule.OnlineResources]
|
|
64
|
+
* @property {{ ProviderBrowseUrl?: Echo10URLObject[] }} [Granule.AssociatedBrowseImageUrls]
|
|
65
|
+
*/
|
|
66
|
+
/**
|
|
67
|
+
* @typedef {Object} getS3UrlOfFileFile
|
|
68
|
+
* @property {string} [filename] - Full S3 URI (e.g., s3://bucket/key)
|
|
69
|
+
* @property {string} [bucket] - Bucket name (used with `key` or `filepath`)
|
|
70
|
+
* @property {string} [key] - S3 key (used with `bucket`)
|
|
71
|
+
* @property {string} [filepath] - Alternate key for the file within the bucket
|
|
72
|
+
*/
|
|
73
|
+
/* eslint-enable max-len */
|
|
74
|
+
const log = new Logger({ sender: '@cumulus/cmrjs/src/cmr-utils' });
|
|
75
|
+
const s3CredsEndpoint = 's3credentials';
|
|
43
76
|
function getS3KeyOfFile(file) {
|
|
44
77
|
if (file.filename)
|
|
45
78
|
return parseS3Uri(file.filename).Key;
|
|
@@ -49,6 +82,34 @@ function getS3KeyOfFile(file) {
|
|
|
49
82
|
return file.key;
|
|
50
83
|
throw new Error(`Unable to determine s3 key of file: ${JSON.stringify(file)}`);
|
|
51
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* Validates that required granule metadata parameters are provided.
|
|
87
|
+
* Throws an error if either parameter is missing or falsy.
|
|
88
|
+
*
|
|
89
|
+
* @param {Object} params - Parameter object
|
|
90
|
+
* @param {string} params.producerGranuleId - The original granule identifier (must be non-empty)
|
|
91
|
+
* @param {string} params.granuleId - The updated granule identifier (must be non-empty)
|
|
92
|
+
*
|
|
93
|
+
* @throws {Error} if either `producerGranuleId` or `granuleId` is not provided
|
|
94
|
+
*/
|
|
95
|
+
function checkRequiredMetadataParms({ producerGranuleId, granuleId }) {
|
|
96
|
+
if (!producerGranuleId) {
|
|
97
|
+
throw new Error('No producerGranuleId was provided when required for CMR metadata update');
|
|
98
|
+
}
|
|
99
|
+
if (!granuleId) {
|
|
100
|
+
throw new Error('No granuleId was provided when required for CMR Metadata update');
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Returns the S3 URI for a given file object.
|
|
105
|
+
*
|
|
106
|
+
* Accepts multiple file shapes commonly used throughout Cumulus and resolves
|
|
107
|
+
* them to a valid `s3://bucket/key` URI.
|
|
108
|
+
*
|
|
109
|
+
* @param {getS3UrlOfFileFile} file - File object containing filename or bucket/key data
|
|
110
|
+
* @returns {string} - A string representing the S3 URI (e.g., `s3://bucket/key`)
|
|
111
|
+
* @throws {Error} if the file does not contain enough information to construct the URI
|
|
112
|
+
*/
|
|
52
113
|
function getS3UrlOfFile(file) {
|
|
53
114
|
if (file.filename)
|
|
54
115
|
return file.filename;
|
|
@@ -58,6 +119,15 @@ function getS3UrlOfFile(file) {
|
|
|
58
119
|
return buildS3Uri(file.bucket, file.key);
|
|
59
120
|
throw new Error(`Unable to determine location of file: ${JSON.stringify(file)}`);
|
|
60
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* Returns the file 'name' of a given object.
|
|
124
|
+
*
|
|
125
|
+
* Accepts multiple file shapes commonly used throughout Cumulus and resolves
|
|
126
|
+
* them to a valid `s3://bucket/key` URI.
|
|
127
|
+
*
|
|
128
|
+
* @param {ApiFileWithFilePath} file - API File
|
|
129
|
+
* @returns {string | undefined} - The file name, or undefined if not found
|
|
130
|
+
*/
|
|
61
131
|
function getFilename(file) {
|
|
62
132
|
if (file.fileName)
|
|
63
133
|
return file.fileName;
|
|
@@ -115,7 +185,7 @@ function isISOFile(fileobject) {
|
|
|
115
185
|
* @param {string} granule.granuleId - granule ID
|
|
116
186
|
* @param {Function} filterFunc - function to determine if the given file object is a
|
|
117
187
|
CMR file; defaults to `isCMRFile`
|
|
118
|
-
* @returns {Array<
|
|
188
|
+
* @returns {Array<CmrFile>} an array of CMR file objects, each with properties
|
|
119
189
|
* `granuleId`, `bucket`, `key`, and possibly `etag` (if present on input)
|
|
120
190
|
*/
|
|
121
191
|
function granuleToCmrFileObject({ granuleId, files = [] }, filterFunc = isCMRFile) {
|
|
@@ -139,7 +209,7 @@ function granuleToCmrFileObject({ granuleId, files = [] }, filterFunc = isCMRFil
|
|
|
139
209
|
* @param {Function} filterFunc - function to determine if the given file object is a
|
|
140
210
|
CMR file; defaults to `isCMRFile`
|
|
141
211
|
*
|
|
142
|
-
* @returns {Array<
|
|
212
|
+
* @returns {Array<CmrFile>} - CMR file object array: { etag, bucket, key, granuleId }
|
|
143
213
|
*/
|
|
144
214
|
function granulesToCmrFileObjects(granules, filterFunc = isCMRFile) {
|
|
145
215
|
return granules.flatMap((granule) => granuleToCmrFileObject(granule, filterFunc));
|
|
@@ -153,7 +223,7 @@ function granulesToCmrFileObjects(granules, filterFunc = isCMRFile) {
|
|
|
153
223
|
* @param {string} cmrFile.metadata - granule xml document
|
|
154
224
|
* @param {Object} cmrClient - a CMR instance
|
|
155
225
|
* @param {string} revisionId - Optional CMR Revision ID
|
|
156
|
-
* @returns {Object} CMR's success response which includes the concept-id
|
|
226
|
+
* @returns {Promise<Object>} CMR's success response which includes the concept-id
|
|
157
227
|
*/
|
|
158
228
|
async function publishECHO10XML2CMR(cmrFile, cmrClient, revisionId) {
|
|
159
229
|
const builder = new xml2js.Builder();
|
|
@@ -181,7 +251,7 @@ async function publishECHO10XML2CMR(cmrFile, cmrClient, revisionId) {
|
|
|
181
251
|
* @param {Object} cmrFile.granuleId - the metadata's granuleId
|
|
182
252
|
* @param {Object} cmrClient - a CMR instance
|
|
183
253
|
* @param {string} revisionId - Optional CMR Revision ID
|
|
184
|
-
* @returns {Object} CMR's success response which includes the concept-id
|
|
254
|
+
* @returns {Promise<Object>} CMR's success response which includes the concept-id
|
|
185
255
|
*/
|
|
186
256
|
async function publishUMMGJSON2CMR(cmrFile, cmrClient, revisionId) {
|
|
187
257
|
const granuleId = cmrFile.metadataObject.GranuleUR;
|
|
@@ -327,7 +397,7 @@ function metadataObjectFromCMRFile(cmrFilename, etag) {
|
|
|
327
397
|
* Build and return an S3 Credentials Object for adding to CMR onlineAccessUrls
|
|
328
398
|
*
|
|
329
399
|
* @param {string} s3CredsUrl - full url pointing to the s3 credential distribution api
|
|
330
|
-
* @returns {
|
|
400
|
+
* @returns {Echo10URLObject} Object with attributes required for adding an onlineAccessUrl
|
|
331
401
|
*/
|
|
332
402
|
function getS3CredentialsObject(s3CredsUrl) {
|
|
333
403
|
return {
|
|
@@ -341,11 +411,12 @@ function getS3CredentialsObject(s3CredsUrl) {
|
|
|
341
411
|
* Returns UMM/ECHO10 resource type mapping for CNM file type
|
|
342
412
|
*
|
|
343
413
|
* @param {string} type - CNM resource type to convert to UMM/ECHO10 type
|
|
344
|
-
* @param {string} urlType - url type, distribution or s3
|
|
345
|
-
* @param {boolean} useDirectS3Type - indicate if direct s3 access type is used
|
|
346
|
-
* @returns {
|
|
414
|
+
* @param {string} [urlType = distribution] - url type, distribution or s3
|
|
415
|
+
* @param {boolean} [useDirectS3Type = false] - indicate if direct s3 access type is used
|
|
416
|
+
* @returns {string} - UMM/ECHO10 resource type
|
|
347
417
|
*/
|
|
348
418
|
function mapCNMTypeToCMRType(type, urlType = 'distribution', useDirectS3Type = false) {
|
|
419
|
+
/** @type {Record<string, string>} */
|
|
349
420
|
const mapping = {
|
|
350
421
|
ancillary: 'VIEW RELATED INFORMATION',
|
|
351
422
|
data: 'GET DATA',
|
|
@@ -354,7 +425,10 @@ function mapCNMTypeToCMRType(type, urlType = 'distribution', useDirectS3Type = f
|
|
|
354
425
|
metadata: 'EXTENDED METADATA',
|
|
355
426
|
qa: 'EXTENDED METADATA',
|
|
356
427
|
};
|
|
357
|
-
|
|
428
|
+
let mappedType = 'GET DATA';
|
|
429
|
+
if (type && type in mapping) {
|
|
430
|
+
mappedType = mapping[type];
|
|
431
|
+
}
|
|
358
432
|
// The CMR Type for the s3 link of science file is "GET DATA VIA DIRECT ACCESS".
|
|
359
433
|
// For non-science file, the Type for the s3 link is the same as its Type for the HTTPS URL.
|
|
360
434
|
if (urlType === 's3' && mappedType === 'GET DATA' && useDirectS3Type) {
|
|
@@ -411,10 +485,10 @@ function mapFileEtags(files) {
|
|
|
411
485
|
* @param {Object} params - input parameters
|
|
412
486
|
* @param {Object} params.file - file object
|
|
413
487
|
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
414
|
-
* @param {
|
|
415
|
-
* @param {
|
|
416
|
-
* for all distribution
|
|
417
|
-
* @returns {(
|
|
488
|
+
* @param {string} [params.urlType = 'distribution'] - url type, distribution or s3
|
|
489
|
+
* @param {Object} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
490
|
+
* for all distribution buckets
|
|
491
|
+
* @returns {(string | undefined)} online access url object, undefined if no URL exists
|
|
418
492
|
*/
|
|
419
493
|
function generateFileUrl({ file, distEndpoint, urlType = 'distribution', distributionBucketMap, }) {
|
|
420
494
|
if (urlType === 'distribution') {
|
|
@@ -442,19 +516,19 @@ function generateFileUrl({ file, distEndpoint, urlType = 'distribution', distrib
|
|
|
442
516
|
/**
|
|
443
517
|
* Construct online access url for a given file and a url type.
|
|
444
518
|
*
|
|
445
|
-
* @param {Object} params
|
|
446
|
-
* @param {
|
|
447
|
-
* @param {
|
|
448
|
-
* @param {
|
|
449
|
-
* @param {
|
|
450
|
-
*
|
|
451
|
-
* @param {boolean} [params.useDirectS3Type] -
|
|
452
|
-
* @returns {
|
|
519
|
+
* @param {Object} params
|
|
520
|
+
* @param {ApiFileWithFilePath} params.file - File object
|
|
521
|
+
* @param {string} params.distEndpoint - Distribution endpoint from config
|
|
522
|
+
* @param {{ [key: string]: string }} params.bucketTypes - Map of bucket names to bucket types
|
|
523
|
+
* @param {'distribution' | 's3'} params.urlType - URL type: 'distribution' or 's3'
|
|
524
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Map of bucket to distribution path
|
|
525
|
+
* @param {boolean} [params.useDirectS3Type=false] - Whether to use direct S3 Type
|
|
526
|
+
* @returns {Echo10URLObject | undefined} - Online access URL object, or undefined if not applicable
|
|
453
527
|
*/
|
|
454
528
|
function constructOnlineAccessUrl({ file, distEndpoint, bucketTypes, urlType = 'distribution', distributionBucketMap, useDirectS3Type = false, }) {
|
|
455
|
-
const bucketType = bucketTypes[file.bucket];
|
|
529
|
+
const bucketType = file.bucket ? bucketTypes[file.bucket] : undefined;
|
|
456
530
|
const distributionApiBuckets = ['protected', 'public'];
|
|
457
|
-
if (distributionApiBuckets.includes(bucketType)) {
|
|
531
|
+
if (bucketType && distributionApiBuckets.includes(bucketType)) {
|
|
458
532
|
const fileUrl = generateFileUrl({ file, distEndpoint, urlType, distributionBucketMap });
|
|
459
533
|
if (fileUrl) {
|
|
460
534
|
const fileDescription = getFileDescription(file, urlType);
|
|
@@ -471,21 +545,23 @@ function constructOnlineAccessUrl({ file, distEndpoint, bucketTypes, urlType = '
|
|
|
471
545
|
/**
|
|
472
546
|
* Construct a list of online access urls grouped by link type.
|
|
473
547
|
*
|
|
474
|
-
* @param {
|
|
475
|
-
* @param {
|
|
476
|
-
* @param {
|
|
477
|
-
* @param {string} params.
|
|
478
|
-
* @param {
|
|
479
|
-
*
|
|
480
|
-
* @param {
|
|
481
|
-
*
|
|
482
|
-
*
|
|
548
|
+
* @param {Object} params
|
|
549
|
+
* @param {ApiFileWithFilePath[]} params.files - Array of file objects
|
|
550
|
+
* @param {string} params.distEndpoint - Distribution endpoint from config
|
|
551
|
+
* @param {{ [key: string]: string }} params.bucketTypes - Map of bucket name to bucket type
|
|
552
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Mapping of bucket to
|
|
553
|
+
* distribution path
|
|
554
|
+
* @param {string} [params.cmrGranuleUrlType=both] - Granule URL type: 's3',
|
|
555
|
+
* 'distribution', or 'both'
|
|
556
|
+
* @param {boolean} [params.useDirectS3Type=false] - Whether direct S3 URL types are used
|
|
557
|
+
* @returns {Echo10URLObject[]} Array of online access URL objects
|
|
483
558
|
*/
|
|
484
|
-
function constructOnlineAccessUrls({
|
|
559
|
+
function constructOnlineAccessUrls({ bucketTypes, cmrGranuleUrlType = 'both', distEndpoint, distributionBucketMap, files, useDirectS3Type = false, }) {
|
|
485
560
|
if (['distribution', 'both'].includes(cmrGranuleUrlType) && !distEndpoint) {
|
|
486
561
|
throw new Error(`cmrGranuleUrlType is ${cmrGranuleUrlType}, but no distribution endpoint is configured.`);
|
|
487
562
|
}
|
|
488
|
-
const [distributionUrls, s3Urls] = files.reduce((
|
|
563
|
+
const [distributionUrls, s3Urls] = files.reduce((
|
|
564
|
+
/** @type {[Echo10URLObject[], Echo10URLObject[]]} */ [distributionAcc, s3Acc], file) => {
|
|
489
565
|
if (['both', 'distribution'].includes(cmrGranuleUrlType)) {
|
|
490
566
|
const url = constructOnlineAccessUrl({
|
|
491
567
|
file,
|
|
@@ -495,7 +571,8 @@ function constructOnlineAccessUrls({ files, distEndpoint, bucketTypes, cmrGranul
|
|
|
495
571
|
distributionBucketMap,
|
|
496
572
|
useDirectS3Type,
|
|
497
573
|
});
|
|
498
|
-
|
|
574
|
+
if (url)
|
|
575
|
+
distributionAcc.push(url);
|
|
499
576
|
}
|
|
500
577
|
if (['both', 's3'].includes(cmrGranuleUrlType)) {
|
|
501
578
|
const url = constructOnlineAccessUrl({
|
|
@@ -506,7 +583,8 @@ function constructOnlineAccessUrls({ files, distEndpoint, bucketTypes, cmrGranul
|
|
|
506
583
|
distributionBucketMap,
|
|
507
584
|
useDirectS3Type,
|
|
508
585
|
});
|
|
509
|
-
|
|
586
|
+
if (url)
|
|
587
|
+
s3Acc.push(url);
|
|
510
588
|
}
|
|
511
589
|
return [distributionAcc, s3Acc];
|
|
512
590
|
}, [[], []]);
|
|
@@ -524,7 +602,7 @@ function constructOnlineAccessUrls({ files, distEndpoint, bucketTypes, cmrGranul
|
|
|
524
602
|
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
525
603
|
* mapping for all distribution buckets
|
|
526
604
|
* @param {boolean} params.useDirectS3Type - indicate if direct s3 access type is used
|
|
527
|
-
* @returns {
|
|
605
|
+
* @returns {[{URL: string, string, Description: string, Type: string}]}
|
|
528
606
|
* an array of online access url objects
|
|
529
607
|
*/
|
|
530
608
|
function constructRelatedUrls({ files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, useDirectS3Type = false, }) {
|
|
@@ -603,8 +681,9 @@ function mergeURLs(original, updated = [], removed = []) {
|
|
|
603
681
|
* Updates CMR JSON file with stringified 'metadataObject'
|
|
604
682
|
*
|
|
605
683
|
* @param {Object} metadataObject - JSON Object to stringify
|
|
606
|
-
* @param {
|
|
607
|
-
* @returns {Promise} returns promised
|
|
684
|
+
* @param {CmrFile} cmrFile - cmr file object to write body to
|
|
685
|
+
* @returns {Promise<{[key: string]: any, ETag?: string | undefined }>} returns promised
|
|
686
|
+
* promiseS3Upload response
|
|
608
687
|
*/
|
|
609
688
|
async function uploadUMMGJSONCMRFile(metadataObject, cmrFile) {
|
|
610
689
|
const tags = await s3GetObjectTagging(cmrFile.bucket, getS3KeyOfFile(cmrFile));
|
|
@@ -638,13 +717,19 @@ function shouldUseDirectS3Type(metadataObject) {
|
|
|
638
717
|
/**
|
|
639
718
|
* Update the UMMG cmr metadata object to have corrected urls
|
|
640
719
|
*
|
|
641
|
-
* @param {Object} params
|
|
642
|
-
* @param {
|
|
643
|
-
* @param {
|
|
644
|
-
*
|
|
645
|
-
* @param {
|
|
646
|
-
*
|
|
647
|
-
*
|
|
720
|
+
* @param {Object} params - Parameters for updating the metadata object
|
|
721
|
+
* @param {Object} params.metadataObject - The existing UMMG CMR metadata object to update
|
|
722
|
+
* @param {ApiFileWithFilePath[]} params.files - Array of file
|
|
723
|
+
* objects used to generate URLs
|
|
724
|
+
* @param {string} params.distEndpoint - Base URL for distribution endpoints (e.g., CloudFront)
|
|
725
|
+
* @param {{ [bucket: string]: string }} params.bucketTypes - Map of bucket names
|
|
726
|
+
* to types (e.g., public, protected)
|
|
727
|
+
* @param {string} [params.cmrGranuleUrlType='both'] - Type of URLs to generate: 'distribution',
|
|
728
|
+
* 's3', or 'both'
|
|
729
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Mapping of bucket names to
|
|
730
|
+
* distribution paths
|
|
731
|
+
*
|
|
732
|
+
* @returns {Object} - A deep clone of the original metadata object with updated RelatedUrls
|
|
648
733
|
*/
|
|
649
734
|
function updateUMMGMetadataObject({ metadataObject, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
650
735
|
const updatedMetadataObject = cloneDeep(metadataObject);
|
|
@@ -658,6 +743,7 @@ function updateUMMGMetadataObject({ metadataObject, files, distEndpoint, bucketT
|
|
|
658
743
|
useDirectS3Type,
|
|
659
744
|
});
|
|
660
745
|
const removedURLs = onlineAccessURLsToRemove(files, bucketTypes);
|
|
746
|
+
/** @type {Array<{ URL: string, Description?: string, Type?: string }>} */
|
|
661
747
|
const originalURLs = get(updatedMetadataObject, 'RelatedUrls', []);
|
|
662
748
|
const mergedURLs = mergeURLs(originalURLs, newURLs, removedURLs);
|
|
663
749
|
set(updatedMetadataObject, 'RelatedUrls', mergedURLs);
|
|
@@ -668,21 +754,27 @@ function updateUMMGMetadataObject({ metadataObject, files, distEndpoint, bucketT
|
|
|
668
754
|
* UMMG cmr.json file with this information.
|
|
669
755
|
*
|
|
670
756
|
* @param {Object} params - parameter object
|
|
671
|
-
* @param {
|
|
672
|
-
* @param {
|
|
757
|
+
* @param {CmrFile} params.cmrFile - cmr.json file whose contents will be updated.
|
|
758
|
+
* @param {ApiFileWithFilePath[]} params.files - array of moved file objects.
|
|
673
759
|
* @param {string} params.distEndpoint - distribution endpoint form config.
|
|
674
|
-
* @param {{ [
|
|
760
|
+
* @param {{ [bucket: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
675
761
|
* @param {string} params.cmrGranuleUrlType - cmrGranuleUrlType from config
|
|
676
762
|
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
677
763
|
* mapping for all distribution buckets
|
|
678
|
-
* @
|
|
764
|
+
* @param {string} params.producerGranuleId - producer granule id
|
|
765
|
+
* @param {string} params.granuleId - granule id
|
|
766
|
+
* @param {boolean} [params.updateGranuleIdentifiers=false] - whether to update the granule UR/add
|
|
767
|
+
* producerGranuleID to the CMR metadata object
|
|
768
|
+
* @param {any} [params.testOverrides] - overrides for testing
|
|
769
|
+
* @returns {Promise<{ metadataObject: Object, etag: string | undefined}>} an object
|
|
679
770
|
* containing a `metadataObject` (the updated UMMG metadata object) and the
|
|
680
771
|
* `etag` of the uploaded CMR file
|
|
681
772
|
*/
|
|
682
|
-
async function updateUMMGMetadata({ cmrFile, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
773
|
+
async function updateUMMGMetadata({ cmrFile, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, producerGranuleId, granuleId, updateGranuleIdentifiers = false, testOverrides = {}, }) {
|
|
774
|
+
const { uploadUMMGJSONCMRFileMethod = uploadUMMGJSONCMRFile, metadataObjectFromCMRJSONFileMethod = metadataObjectFromCMRJSONFile, } = testOverrides;
|
|
683
775
|
const filename = getS3UrlOfFile(cmrFile);
|
|
684
|
-
const metadataObject = await
|
|
685
|
-
|
|
776
|
+
const metadataObject = await metadataObjectFromCMRJSONFileMethod(filename);
|
|
777
|
+
let updatedMetadataObject = updateUMMGMetadataObject({
|
|
686
778
|
metadataObject,
|
|
687
779
|
files,
|
|
688
780
|
distEndpoint,
|
|
@@ -690,7 +782,16 @@ async function updateUMMGMetadata({ cmrFile, files, distEndpoint, bucketTypes, c
|
|
|
690
782
|
cmrGranuleUrlType,
|
|
691
783
|
distributionBucketMap,
|
|
692
784
|
});
|
|
693
|
-
|
|
785
|
+
if (updateGranuleIdentifiers) {
|
|
786
|
+
// Type checks are needed as this callers/API are not all typed/ts converted yet
|
|
787
|
+
checkRequiredMetadataParms({ producerGranuleId, granuleId });
|
|
788
|
+
updatedMetadataObject = updateUMMGGranuleURAndGranuleIdentifier({
|
|
789
|
+
granuleUr: granuleId,
|
|
790
|
+
producerGranuleId,
|
|
791
|
+
metadataObject: updatedMetadataObject,
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
const { ETag: etag } = await uploadUMMGJSONCMRFileMethod(updatedMetadataObject, cmrFile);
|
|
694
795
|
return { metadataObject: updatedMetadataObject, etag };
|
|
695
796
|
}
|
|
696
797
|
/**
|
|
@@ -791,22 +892,45 @@ function buildMergedEchoURLObject(URLlist = [], originalURLlist = [], removedURL
|
|
|
791
892
|
return mergeURLs(originalURLlist, filteredURLObjectList, removedURLs);
|
|
792
893
|
}
|
|
793
894
|
/**
|
|
794
|
-
*
|
|
895
|
+
* Updates the OnlineAccessURLs, OnlineResources, and AssociatedBrowseImageUrls
|
|
896
|
+
* fields of an ECHO10 CMR metadata object with newly constructed URLs.
|
|
795
897
|
*
|
|
796
|
-
*
|
|
797
|
-
*
|
|
798
|
-
*
|
|
799
|
-
*
|
|
800
|
-
*
|
|
801
|
-
*
|
|
802
|
-
* @
|
|
898
|
+
* This function:
|
|
899
|
+
* - Extracts the original URL sets from the ECHO10 XML metadata.
|
|
900
|
+
* - Constructs new URL entries based on the provided file list and configuration.
|
|
901
|
+
* - Merges new URLs with original ones, removing outdated or irrelevant URLs.
|
|
902
|
+
* - Returns a new metadata object with an updated `Granule` field.
|
|
903
|
+
*
|
|
904
|
+
* @param {Object} params - Input parameters
|
|
905
|
+
* @param {Echo10MetadataObject} params.metadataObject - The parsed ECHO10 metadata XML
|
|
906
|
+
* object (as a JavaScript object), expected to include a `Granule` key
|
|
907
|
+
* @param {ApiFileWithFilePath[]} params.files - Granule files to generate
|
|
908
|
+
* updated URLs from
|
|
909
|
+
* @param {string} params.distEndpoint - The base distribution endpoint URL
|
|
910
|
+
* (e.g., CloudFront origin)
|
|
911
|
+
* @param {{ [bucketName: string]: string }} params.bucketTypes - Mapping of bucket names
|
|
912
|
+
* to access types ('public', 'protected', etc.)
|
|
913
|
+
* @param {string} [params.cmrGranuleUrlType='both'] - Type of URLs to generate
|
|
914
|
+
* for CMR: 'distribution', 's3', or 'both'
|
|
915
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Maps S3 buckets to their
|
|
916
|
+
* distribution URL paths
|
|
917
|
+
*
|
|
918
|
+
* @returns {Echo10MetadataObject} A new ECHO10 metadata object with updated
|
|
919
|
+
* `Granule.OnlineAccessURLs`, `Granule.OnlineResources`, and `Granule.AssociatedBrowseImageUrls`
|
|
920
|
+
* fields
|
|
803
921
|
*/
|
|
804
|
-
function
|
|
922
|
+
function updateEcho10XMLMetadataObjectUrls({ metadataObject, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
805
923
|
const metadataGranule = metadataObject.Granule;
|
|
806
924
|
const updatedGranule = { ...metadataGranule };
|
|
807
|
-
|
|
808
|
-
const
|
|
809
|
-
|
|
925
|
+
/** @type {Echo10URLObject[]} */
|
|
926
|
+
const originalOnlineAccessURLs = /** @type {Echo10URLObject[]} */ (
|
|
927
|
+
/** @type {Echo10URLObject[]} */ ([]).concat(get(metadataGranule, 'OnlineAccessURLs.OnlineAccessURL') ?? []));
|
|
928
|
+
/** @type {Echo10URLObject[]} */
|
|
929
|
+
const originalOnlineResourceURLs = /** @type {Echo10URLObject[]} */ (
|
|
930
|
+
/** @type {Echo10URLObject[]} */ ([]).concat(get(metadataGranule, 'OnlineResources.OnlineResource') ?? []));
|
|
931
|
+
/** @type {Echo10URLObject[]} */
|
|
932
|
+
const originalAssociatedBrowseURLs = /** @type {Echo10URLObject[]} */ (
|
|
933
|
+
/** @type {Echo10URLObject[]} */ ([]).concat(get(metadataGranule, 'AssociatedBrowseImageUrls.ProviderBrowseUrl') ?? []));
|
|
810
934
|
const removedURLs = onlineAccessURLsToRemove(files, bucketTypes);
|
|
811
935
|
const newURLs = constructOnlineAccessUrls({
|
|
812
936
|
files,
|
|
@@ -829,24 +953,34 @@ function updateEcho10XMLMetadataObject({ metadataObject, files, distEndpoint, bu
|
|
|
829
953
|
};
|
|
830
954
|
}
|
|
831
955
|
/**
|
|
832
|
-
*
|
|
833
|
-
*
|
|
956
|
+
* Updates an ECHO10 CMR XML metadata file on S3 to reflect new URLs and optionally
|
|
957
|
+
* a new GranuleUR and ProducerGranuleId.
|
|
834
958
|
*
|
|
835
|
-
* @param {Object} params
|
|
836
|
-
* @param {
|
|
837
|
-
* @param {
|
|
838
|
-
* @param {
|
|
839
|
-
* @param {
|
|
840
|
-
*
|
|
841
|
-
*
|
|
842
|
-
* @
|
|
843
|
-
*
|
|
959
|
+
* @param {Object} params
|
|
960
|
+
* @param {string} params.granuleId - New GranuleUR to set in metadata
|
|
961
|
+
* @param {string} params.producerGranuleId - Original ProducerGranuleId to record
|
|
962
|
+
* @param {CmrFile} params.cmrFile - The cmr xml file to be updated
|
|
963
|
+
* @param {ApiFileWithFilePath[]} params.files - List of granule files used
|
|
964
|
+
* to generate OnlineAccess URLs
|
|
965
|
+
* @param {string} params.distEndpoint - Distribution endpoint for download URLs
|
|
966
|
+
* @param {{ [bucket: string]: string }} params.bucketTypes - Mapping of bucket names to their types
|
|
967
|
+
* @param {string} [params.cmrGranuleUrlType]
|
|
968
|
+
* - Type of URLs to generate ('distribution' | 's3' | 'both')
|
|
969
|
+
* @param {DistributionBucketMap} params.distributionBucketMap
|
|
970
|
+
* - Maps buckets to distribution paths
|
|
971
|
+
* @param {boolean} [params.updateGranuleIdentifiers]
|
|
972
|
+
* - If true, update the GranuleUR and ProducerGranuleId in metadata
|
|
973
|
+
* @param {any} [params.testOverrides]
|
|
974
|
+
* - Optional test overrides for internal functions
|
|
975
|
+
* @returns {Promise<{ metadataObject: any, etag: string }>}
|
|
976
|
+
* The updated metadata object and resulting ETag
|
|
844
977
|
*/
|
|
845
|
-
async function updateEcho10XMLMetadata({ cmrFile, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
978
|
+
async function updateEcho10XMLMetadata({ granuleId, producerGranuleId, cmrFile, files, distEndpoint, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, updateGranuleIdentifiers = false, testOverrides = {}, }) {
|
|
979
|
+
const { generateEcho10XMLStringMethod = generateEcho10XMLString, uploadEcho10CMRFileMethod = uploadEcho10CMRFile, metadataObjectFromCMRXMLFileMethod = metadataObjectFromCMRXMLFile, } = testOverrides;
|
|
846
980
|
// add/replace the OnlineAccessUrls
|
|
847
981
|
const filename = getS3UrlOfFile(cmrFile);
|
|
848
|
-
const metadataObject = await
|
|
849
|
-
|
|
982
|
+
const metadataObject = await metadataObjectFromCMRXMLFileMethod(filename);
|
|
983
|
+
let updatedMetadataObject = updateEcho10XMLMetadataObjectUrls({
|
|
850
984
|
metadataObject,
|
|
851
985
|
files,
|
|
852
986
|
distEndpoint,
|
|
@@ -854,8 +988,22 @@ async function updateEcho10XMLMetadata({ cmrFile, files, distEndpoint, bucketTyp
|
|
|
854
988
|
cmrGranuleUrlType,
|
|
855
989
|
distributionBucketMap,
|
|
856
990
|
});
|
|
857
|
-
|
|
858
|
-
|
|
991
|
+
if (updateGranuleIdentifiers) {
|
|
992
|
+
// Type checks are needed as this callers/API are not all typed/ts converted yet
|
|
993
|
+
try {
|
|
994
|
+
checkRequiredMetadataParms({ producerGranuleId, granuleId });
|
|
995
|
+
}
|
|
996
|
+
catch (error) {
|
|
997
|
+
throw new Error(`updateGranuleIdentifiers was set, but producerGranuleId ${producerGranuleId} or granuleId ${granuleId} is not set.`, { cause: error });
|
|
998
|
+
}
|
|
999
|
+
updatedMetadataObject = updateEcho10XMLGranuleUrAndGranuleIdentifier({
|
|
1000
|
+
granuleUr: granuleId,
|
|
1001
|
+
producerGranuleId,
|
|
1002
|
+
xml: updatedMetadataObject,
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
const xml = generateEcho10XMLStringMethod(updatedMetadataObject.Granule);
|
|
1006
|
+
const { ETag: etag } = await uploadEcho10CMRFileMethod(xml, cmrFile);
|
|
859
1007
|
return { metadataObject: updatedMetadataObject, etag };
|
|
860
1008
|
}
|
|
861
1009
|
/**
|
|
@@ -863,28 +1011,37 @@ async function updateEcho10XMLMetadata({ cmrFile, files, distEndpoint, bucketTyp
|
|
|
863
1011
|
*
|
|
864
1012
|
* @param {Object} params - parameter object
|
|
865
1013
|
* @param {string} params.granuleId - granuleId
|
|
866
|
-
* @param {
|
|
867
|
-
* @param {
|
|
1014
|
+
* @param {string} [params.producerGranuleId] - producer granuleId
|
|
1015
|
+
* @param {CmrFile} params.cmrFile - cmr file to be updated
|
|
1016
|
+
* @param {ApiFileWithFilePath[]} params.files - array of file objects
|
|
868
1017
|
* @param {string} params.distEndpoint - distribution enpoint from config
|
|
869
1018
|
* @param {boolean} params.published - indicate if publish is needed
|
|
870
1019
|
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
871
1020
|
* @param {string} params.cmrGranuleUrlType - type of granule CMR url
|
|
1021
|
+
* @param {boolean} [params.updateGranuleIdentifiers]
|
|
1022
|
+
* - If true, update the GranuleUR and ProducerGranuleId in metadata
|
|
1023
|
+
* @param {any} [params.testOverrides]
|
|
1024
|
+
* - Optional test overrides for internal functions
|
|
872
1025
|
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
873
1026
|
* mapping for all distribution buckets
|
|
874
1027
|
* @returns {Promise<Object>} CMR file object with the `etag` of the newly
|
|
875
1028
|
* updated metadata file
|
|
876
1029
|
*/
|
|
877
|
-
async function updateCMRMetadata({ granuleId, cmrFile, files, distEndpoint, published, bucketTypes, cmrGranuleUrlType = 'both', distributionBucketMap, }) {
|
|
1030
|
+
async function updateCMRMetadata({ granuleId, producerGranuleId, cmrFile, files, distEndpoint, published, bucketTypes, cmrGranuleUrlType = 'both', updateGranuleIdentifiers = false, distributionBucketMap, testOverrides = {}, }) {
|
|
1031
|
+
const { publish2CMRMethod = publish2CMR, getCmrSettingsMethod = getCmrSettings, } = testOverrides;
|
|
878
1032
|
const filename = getS3UrlOfFile(cmrFile);
|
|
879
|
-
log.debug(`cmrjs.updateCMRMetadata granuleId ${granuleId}
|
|
880
|
-
const cmrCredentials = (published) ? await
|
|
1033
|
+
log.debug(`cmrjs.updateCMRMetadata granuleId ${granuleId} cmrMetadata file ${filename}`);
|
|
1034
|
+
const cmrCredentials = (published) ? await getCmrSettingsMethod() : {};
|
|
881
1035
|
const params = {
|
|
882
|
-
cmrFile,
|
|
883
|
-
files,
|
|
884
|
-
distEndpoint,
|
|
885
1036
|
bucketTypes,
|
|
1037
|
+
cmrFile,
|
|
886
1038
|
cmrGranuleUrlType,
|
|
1039
|
+
distEndpoint,
|
|
887
1040
|
distributionBucketMap,
|
|
1041
|
+
files,
|
|
1042
|
+
granuleId,
|
|
1043
|
+
producerGranuleId: producerGranuleId || granuleId,
|
|
1044
|
+
updateGranuleIdentifiers,
|
|
888
1045
|
};
|
|
889
1046
|
let metadataObject;
|
|
890
1047
|
let etag;
|
|
@@ -904,7 +1061,7 @@ async function updateCMRMetadata({ granuleId, cmrFile, files, distEndpoint, publ
|
|
|
904
1061
|
metadataObject,
|
|
905
1062
|
granuleId,
|
|
906
1063
|
};
|
|
907
|
-
return { ...await
|
|
1064
|
+
return { ...await publish2CMRMethod(cmrPublishObject, cmrCredentials), etag };
|
|
908
1065
|
}
|
|
909
1066
|
return { ...cmrFile, etag };
|
|
910
1067
|
}
|
|
@@ -1144,21 +1301,23 @@ const getCMRCollectionId = (cmrObject, cmrFileName) => {
|
|
|
1144
1301
|
};
|
|
1145
1302
|
module.exports = {
|
|
1146
1303
|
addEtagsToFileObjects,
|
|
1304
|
+
buildCMRQuery,
|
|
1147
1305
|
constructCmrConceptLink,
|
|
1148
1306
|
constructOnlineAccessUrl,
|
|
1149
1307
|
constructOnlineAccessUrls,
|
|
1150
1308
|
generateEcho10XMLString,
|
|
1151
1309
|
generateFileUrl,
|
|
1152
|
-
granuleToCmrFileObject,
|
|
1153
|
-
getCmrSettings,
|
|
1154
1310
|
getCMRCollectionId,
|
|
1311
|
+
getCmrSettings,
|
|
1312
|
+
getCollectionsByShortNameAndVersion,
|
|
1155
1313
|
getFileDescription,
|
|
1156
1314
|
getFilename,
|
|
1157
1315
|
getGranuleTemporalInfo,
|
|
1158
|
-
getCollectionsByShortNameAndVersion,
|
|
1159
1316
|
getS3UrlOfFile,
|
|
1160
1317
|
getUserAccessibleBuckets,
|
|
1318
|
+
getXMLMetadataAsString,
|
|
1161
1319
|
granulesToCmrFileObjects,
|
|
1320
|
+
granuleToCmrFileObject,
|
|
1162
1321
|
isCMRFile,
|
|
1163
1322
|
isCMRFilename,
|
|
1164
1323
|
isCMRISOFilename,
|
|
@@ -1168,15 +1327,18 @@ module.exports = {
|
|
|
1168
1327
|
isUMMGFilename,
|
|
1169
1328
|
mapFileEtags,
|
|
1170
1329
|
metadataObjectFromCMRFile,
|
|
1330
|
+
parseXmlString,
|
|
1171
1331
|
publish2CMR,
|
|
1172
1332
|
reconcileCMRMetadata,
|
|
1173
1333
|
removeEtagsFromFileObjects,
|
|
1174
1334
|
removeFromCMR,
|
|
1175
|
-
updateCMRMetadata,
|
|
1176
|
-
updateEcho10XMLMetadataObject,
|
|
1177
|
-
updateUMMGMetadataObject,
|
|
1178
1335
|
setECHO10Collection,
|
|
1179
1336
|
setUMMGCollection,
|
|
1337
|
+
updateCMRMetadata,
|
|
1338
|
+
updateEcho10XMLMetadata,
|
|
1339
|
+
updateEcho10XMLMetadataObjectUrls,
|
|
1340
|
+
updateUMMGMetadata,
|
|
1341
|
+
updateUMMGMetadataObject,
|
|
1180
1342
|
uploadEcho10CMRFile,
|
|
1181
1343
|
uploadUMMGJSONCMRFile,
|
|
1182
1344
|
};
|