@cumulus/cmrjs 20.3.1 → 21.0.0-echo10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cmr-utils.d.ts +371 -108
- package/cmr-utils.d.ts.map +1 -1
- package/cmr-utils.js +263 -101
- package/cmr-utils.js.map +1 -1
- package/echo10Modifiers.d.ts +23 -0
- package/echo10Modifiers.d.ts.map +1 -0
- package/echo10Modifiers.js +69 -0
- package/echo10Modifiers.js.map +1 -0
- package/package.json +12 -12
- package/src/cmr-utils.js +333 -125
- package/src/echo10Modifiers.ts +91 -0
- package/src/ummgModifiers.ts +71 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/ummgModifiers.d.ts +35 -0
- package/ummgModifiers.d.ts.map +1 -0
- package/ummgModifiers.js +45 -0
- package/ummgModifiers.js.map +1 -0
package/src/cmr-utils.js
CHANGED
|
@@ -36,16 +36,25 @@ const {
|
|
|
36
36
|
xmlParseOptions,
|
|
37
37
|
ummVersionToMetadataFormat,
|
|
38
38
|
} = require('./utils');
|
|
39
|
+
const { updateEcho10XMLGranuleUrAndGranuleIdentifier } = require('./echo10Modifiers');
|
|
40
|
+
const { updateUMMGGranuleURAndGranuleIdentifier } = require('./ummgModifiers');
|
|
41
|
+
|
|
39
42
|
/* eslint-disable max-len */
|
|
40
43
|
/**
|
|
41
44
|
* @typedef {import('@cumulus/cmr-client/CMR').CMRConstructorParams} CMRConstructorParams
|
|
42
45
|
* @typedef {import('@cumulus/distribution-utils/dist/types').DistributionBucketMap} DistributionBucketMap
|
|
43
|
-
* @typedef {import('@cumulus/types').
|
|
46
|
+
* @typedef {import('@cumulus/types').ApiFileGranuleIdOptional} ApiFileGranuleIdOptional
|
|
47
|
+
* @typedef { ApiFileGranuleIdOptional & { filepath?: string }} ApiFileWithFilePath
|
|
44
48
|
*/
|
|
45
|
-
/* eslint-enable max-len */
|
|
46
|
-
const log = new Logger({ sender: '@cumulus/cmrjs/src/cmr-utils' });
|
|
47
49
|
|
|
48
|
-
|
|
50
|
+
/**
|
|
51
|
+
* @typedef {Object} CmrFile
|
|
52
|
+
* @property {string} bucket - The S3 bucket name
|
|
53
|
+
* @property {string} key - The S3 key for the metadata file
|
|
54
|
+
* @property {string} granuleId - The granule ID associated with the file
|
|
55
|
+
* @property {string} [etag] - Optional entity tag for file versioning
|
|
56
|
+
*/
|
|
57
|
+
/**
|
|
49
58
|
|
|
50
59
|
/**
|
|
51
60
|
* @typedef {{
|
|
@@ -57,6 +66,35 @@ const s3CredsEndpoint = 's3credentials';
|
|
|
57
66
|
* }} CmrCredentials
|
|
58
67
|
*/
|
|
59
68
|
|
|
69
|
+
/**
|
|
70
|
+
* @typedef {Object} Echo10URLObject
|
|
71
|
+
* @property {string} URL
|
|
72
|
+
* @property {string} [Type]
|
|
73
|
+
* @property {string} [Description]
|
|
74
|
+
* @property {string} [URLDescription]
|
|
75
|
+
*/
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* @typedef {Object} Echo10MetadataObject
|
|
79
|
+
* @property {Object} Granule - The root ECHO10 granule object
|
|
80
|
+
* @property {{ OnlineAccessURL?: Echo10URLObject[] }} [Granule.OnlineAccessURLs]
|
|
81
|
+
* @property {{ OnlineResource?: Echo10URLObject[] }} [Granule.OnlineResources]
|
|
82
|
+
* @property {{ ProviderBrowseUrl?: Echo10URLObject[] }} [Granule.AssociatedBrowseImageUrls]
|
|
83
|
+
*/
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* @typedef {Object} getS3UrlOfFileFile
|
|
87
|
+
* @property {string} [filename] - Full S3 URI (e.g., s3://bucket/key)
|
|
88
|
+
* @property {string} [bucket] - Bucket name (used with `key` or `filepath`)
|
|
89
|
+
* @property {string} [key] - S3 key (used with `bucket`)
|
|
90
|
+
* @property {string} [filepath] - Alternate key for the file within the bucket
|
|
91
|
+
*/
|
|
92
|
+
/* eslint-enable max-len */
|
|
93
|
+
|
|
94
|
+
const log = new Logger({ sender: '@cumulus/cmrjs/src/cmr-utils' });
|
|
95
|
+
|
|
96
|
+
const s3CredsEndpoint = 's3credentials';
|
|
97
|
+
|
|
60
98
|
function getS3KeyOfFile(file) {
|
|
61
99
|
if (file.filename) return parseS3Uri(file.filename).Key;
|
|
62
100
|
if (file.filepath) return file.filepath;
|
|
@@ -64,6 +102,37 @@ function getS3KeyOfFile(file) {
|
|
|
64
102
|
throw new Error(`Unable to determine s3 key of file: ${JSON.stringify(file)}`);
|
|
65
103
|
}
|
|
66
104
|
|
|
105
|
+
/**
|
|
106
|
+
* Validates that required granule metadata parameters are provided.
|
|
107
|
+
* Throws an error if either parameter is missing or falsy.
|
|
108
|
+
*
|
|
109
|
+
* @param {Object} params - Parameter object
|
|
110
|
+
* @param {string} params.producerGranuleId - The original granule identifier (must be non-empty)
|
|
111
|
+
* @param {string} params.granuleId - The updated granule identifier (must be non-empty)
|
|
112
|
+
*
|
|
113
|
+
* @throws {Error} if either `producerGranuleId` or `granuleId` is not provided
|
|
114
|
+
*/
|
|
115
|
+
function checkRequiredMetadataParms({ producerGranuleId, granuleId }) {
|
|
116
|
+
if (!producerGranuleId) {
|
|
117
|
+
throw new Error(
|
|
118
|
+
'No producerGranuleId was provided when required for CMR metadata update'
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
if (!granuleId) {
|
|
122
|
+
throw new Error('No granuleId was provided when required for CMR Metadata update');
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Returns the S3 URI for a given file object.
|
|
128
|
+
*
|
|
129
|
+
* Accepts multiple file shapes commonly used throughout Cumulus and resolves
|
|
130
|
+
* them to a valid `s3://bucket/key` URI.
|
|
131
|
+
*
|
|
132
|
+
* @param {getS3UrlOfFileFile} file - File object containing filename or bucket/key data
|
|
133
|
+
* @returns {string} - A string representing the S3 URI (e.g., `s3://bucket/key`)
|
|
134
|
+
* @throws {Error} if the file does not contain enough information to construct the URI
|
|
135
|
+
*/
|
|
67
136
|
function getS3UrlOfFile(file) {
|
|
68
137
|
if (file.filename) return file.filename;
|
|
69
138
|
if (file.bucket && file.filepath) return buildS3Uri(file.bucket, file.filepath);
|
|
@@ -71,6 +140,15 @@ function getS3UrlOfFile(file) {
|
|
|
71
140
|
throw new Error(`Unable to determine location of file: ${JSON.stringify(file)}`);
|
|
72
141
|
}
|
|
73
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Returns the file 'name' of a given object.
|
|
145
|
+
*
|
|
146
|
+
* Accepts multiple file shapes commonly used throughout Cumulus and resolves
|
|
147
|
+
* them to a valid `s3://bucket/key` URI.
|
|
148
|
+
*
|
|
149
|
+
* @param {ApiFileWithFilePath} file - API File
|
|
150
|
+
* @returns {string | undefined} - The file name, or undefined if not found
|
|
151
|
+
*/
|
|
74
152
|
function getFilename(file) {
|
|
75
153
|
if (file.fileName) return file.fileName;
|
|
76
154
|
if (file.name) return file.name;
|
|
@@ -129,7 +207,7 @@ function isISOFile(fileobject) {
|
|
|
129
207
|
* @param {string} granule.granuleId - granule ID
|
|
130
208
|
* @param {Function} filterFunc - function to determine if the given file object is a
|
|
131
209
|
CMR file; defaults to `isCMRFile`
|
|
132
|
-
* @returns {Array<
|
|
210
|
+
* @returns {Array<CmrFile>} an array of CMR file objects, each with properties
|
|
133
211
|
* `granuleId`, `bucket`, `key`, and possibly `etag` (if present on input)
|
|
134
212
|
*/
|
|
135
213
|
function granuleToCmrFileObject({ granuleId, files = [] }, filterFunc = isCMRFile) {
|
|
@@ -154,7 +232,7 @@ function granuleToCmrFileObject({ granuleId, files = [] }, filterFunc = isCMRFil
|
|
|
154
232
|
* @param {Function} filterFunc - function to determine if the given file object is a
|
|
155
233
|
CMR file; defaults to `isCMRFile`
|
|
156
234
|
*
|
|
157
|
-
* @returns {Array<
|
|
235
|
+
* @returns {Array<CmrFile>} - CMR file object array: { etag, bucket, key, granuleId }
|
|
158
236
|
*/
|
|
159
237
|
function granulesToCmrFileObjects(granules, filterFunc = isCMRFile) {
|
|
160
238
|
return granules.flatMap((granule) => granuleToCmrFileObject(granule, filterFunc));
|
|
@@ -169,7 +247,7 @@ function granulesToCmrFileObjects(granules, filterFunc = isCMRFile) {
|
|
|
169
247
|
* @param {string} cmrFile.metadata - granule xml document
|
|
170
248
|
* @param {Object} cmrClient - a CMR instance
|
|
171
249
|
* @param {string} revisionId - Optional CMR Revision ID
|
|
172
|
-
* @returns {Object} CMR's success response which includes the concept-id
|
|
250
|
+
* @returns {Promise<Object>} CMR's success response which includes the concept-id
|
|
173
251
|
*/
|
|
174
252
|
async function publishECHO10XML2CMR(cmrFile, cmrClient, revisionId) {
|
|
175
253
|
const builder = new xml2js.Builder();
|
|
@@ -199,7 +277,7 @@ async function publishECHO10XML2CMR(cmrFile, cmrClient, revisionId) {
|
|
|
199
277
|
* @param {Object} cmrFile.granuleId - the metadata's granuleId
|
|
200
278
|
* @param {Object} cmrClient - a CMR instance
|
|
201
279
|
* @param {string} revisionId - Optional CMR Revision ID
|
|
202
|
-
* @returns {Object} CMR's success response which includes the concept-id
|
|
280
|
+
* @returns {Promise<Object>} CMR's success response which includes the concept-id
|
|
203
281
|
*/
|
|
204
282
|
async function publishUMMGJSON2CMR(cmrFile, cmrClient, revisionId) {
|
|
205
283
|
const granuleId = cmrFile.metadataObject.GranuleUR;
|
|
@@ -361,7 +439,7 @@ function metadataObjectFromCMRFile(cmrFilename, etag) {
|
|
|
361
439
|
* Build and return an S3 Credentials Object for adding to CMR onlineAccessUrls
|
|
362
440
|
*
|
|
363
441
|
* @param {string} s3CredsUrl - full url pointing to the s3 credential distribution api
|
|
364
|
-
* @returns {
|
|
442
|
+
* @returns {Echo10URLObject} Object with attributes required for adding an onlineAccessUrl
|
|
365
443
|
*/
|
|
366
444
|
function getS3CredentialsObject(s3CredsUrl) {
|
|
367
445
|
return {
|
|
@@ -376,11 +454,12 @@ function getS3CredentialsObject(s3CredsUrl) {
|
|
|
376
454
|
* Returns UMM/ECHO10 resource type mapping for CNM file type
|
|
377
455
|
*
|
|
378
456
|
* @param {string} type - CNM resource type to convert to UMM/ECHO10 type
|
|
379
|
-
* @param {string} urlType - url type, distribution or s3
|
|
380
|
-
* @param {boolean} useDirectS3Type - indicate if direct s3 access type is used
|
|
381
|
-
* @returns {
|
|
457
|
+
* @param {string} [urlType = distribution] - url type, distribution or s3
|
|
458
|
+
* @param {boolean} [useDirectS3Type = false] - indicate if direct s3 access type is used
|
|
459
|
+
* @returns {string} - UMM/ECHO10 resource type
|
|
382
460
|
*/
|
|
383
461
|
function mapCNMTypeToCMRType(type, urlType = 'distribution', useDirectS3Type = false) {
|
|
462
|
+
/** @type {Record<string, string>} */
|
|
384
463
|
const mapping = {
|
|
385
464
|
ancillary: 'VIEW RELATED INFORMATION',
|
|
386
465
|
data: 'GET DATA',
|
|
@@ -389,8 +468,11 @@ function mapCNMTypeToCMRType(type, urlType = 'distribution', useDirectS3Type = f
|
|
|
389
468
|
metadata: 'EXTENDED METADATA',
|
|
390
469
|
qa: 'EXTENDED METADATA',
|
|
391
470
|
};
|
|
392
|
-
const mappedType = mapping[type] || 'GET DATA';
|
|
393
471
|
|
|
472
|
+
let mappedType = 'GET DATA';
|
|
473
|
+
if (type && type in mapping) {
|
|
474
|
+
mappedType = mapping[type];
|
|
475
|
+
}
|
|
394
476
|
// The CMR Type for the s3 link of science file is "GET DATA VIA DIRECT ACCESS".
|
|
395
477
|
// For non-science file, the Type for the s3 link is the same as its Type for the HTTPS URL.
|
|
396
478
|
if (urlType === 's3' && mappedType === 'GET DATA' && useDirectS3Type) {
|
|
@@ -450,10 +532,10 @@ function mapFileEtags(files) {
|
|
|
450
532
|
* @param {Object} params - input parameters
|
|
451
533
|
* @param {Object} params.file - file object
|
|
452
534
|
* @param {string} params.distEndpoint - distribution endpoint from config
|
|
453
|
-
* @param {
|
|
454
|
-
* @param {
|
|
455
|
-
* for all distribution
|
|
456
|
-
* @returns {(
|
|
535
|
+
* @param {string} [params.urlType = 'distribution'] - url type, distribution or s3
|
|
536
|
+
* @param {Object} params.distributionBucketMap - Object with bucket:tea-path mapping
|
|
537
|
+
* for all distribution buckets
|
|
538
|
+
* @returns {(string | undefined)} online access url object, undefined if no URL exists
|
|
457
539
|
*/
|
|
458
540
|
function generateFileUrl({
|
|
459
541
|
file,
|
|
@@ -490,14 +572,14 @@ function generateFileUrl({
|
|
|
490
572
|
/**
|
|
491
573
|
* Construct online access url for a given file and a url type.
|
|
492
574
|
*
|
|
493
|
-
* @param {Object} params
|
|
494
|
-
* @param {
|
|
495
|
-
* @param {
|
|
496
|
-
* @param {
|
|
497
|
-
* @param {
|
|
498
|
-
*
|
|
499
|
-
* @param {boolean} [params.useDirectS3Type] -
|
|
500
|
-
* @returns {
|
|
575
|
+
* @param {Object} params
|
|
576
|
+
* @param {ApiFileWithFilePath} params.file - File object
|
|
577
|
+
* @param {string} params.distEndpoint - Distribution endpoint from config
|
|
578
|
+
* @param {{ [key: string]: string }} params.bucketTypes - Map of bucket names to bucket types
|
|
579
|
+
* @param {'distribution' | 's3'} params.urlType - URL type: 'distribution' or 's3'
|
|
580
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Map of bucket to distribution path
|
|
581
|
+
* @param {boolean} [params.useDirectS3Type=false] - Whether to use direct S3 Type
|
|
582
|
+
* @returns {Echo10URLObject | undefined} - Online access URL object, or undefined if not applicable
|
|
501
583
|
*/
|
|
502
584
|
function constructOnlineAccessUrl({
|
|
503
585
|
file,
|
|
@@ -507,9 +589,9 @@ function constructOnlineAccessUrl({
|
|
|
507
589
|
distributionBucketMap,
|
|
508
590
|
useDirectS3Type = false,
|
|
509
591
|
}) {
|
|
510
|
-
const bucketType = bucketTypes[file.bucket];
|
|
592
|
+
const bucketType = file.bucket ? bucketTypes[file.bucket] : undefined;
|
|
511
593
|
const distributionApiBuckets = ['protected', 'public'];
|
|
512
|
-
if (distributionApiBuckets.includes(bucketType)) {
|
|
594
|
+
if (bucketType && distributionApiBuckets.includes(bucketType)) {
|
|
513
595
|
const fileUrl = generateFileUrl({ file, distEndpoint, urlType, distributionBucketMap });
|
|
514
596
|
if (fileUrl) {
|
|
515
597
|
const fileDescription = getFileDescription(file, urlType);
|
|
@@ -527,53 +609,58 @@ function constructOnlineAccessUrl({
|
|
|
527
609
|
/**
|
|
528
610
|
* Construct a list of online access urls grouped by link type.
|
|
529
611
|
*
|
|
530
|
-
* @param {
|
|
531
|
-
* @param {
|
|
532
|
-
* @param {
|
|
533
|
-
* @param {string} params.
|
|
534
|
-
* @param {
|
|
535
|
-
*
|
|
536
|
-
* @param {
|
|
537
|
-
*
|
|
538
|
-
*
|
|
612
|
+
* @param {Object} params
|
|
613
|
+
* @param {ApiFileWithFilePath[]} params.files - Array of file objects
|
|
614
|
+
* @param {string} params.distEndpoint - Distribution endpoint from config
|
|
615
|
+
* @param {{ [key: string]: string }} params.bucketTypes - Map of bucket name to bucket type
|
|
616
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Mapping of bucket to
|
|
617
|
+
* distribution path
|
|
618
|
+
* @param {string} [params.cmrGranuleUrlType=both] - Granule URL type: 's3',
|
|
619
|
+
* 'distribution', or 'both'
|
|
620
|
+
* @param {boolean} [params.useDirectS3Type=false] - Whether direct S3 URL types are used
|
|
621
|
+
* @returns {Echo10URLObject[]} Array of online access URL objects
|
|
539
622
|
*/
|
|
540
623
|
function constructOnlineAccessUrls({
|
|
541
|
-
files,
|
|
542
|
-
distEndpoint,
|
|
543
624
|
bucketTypes,
|
|
544
625
|
cmrGranuleUrlType = 'both',
|
|
626
|
+
distEndpoint,
|
|
545
627
|
distributionBucketMap,
|
|
628
|
+
files,
|
|
546
629
|
useDirectS3Type = false,
|
|
547
630
|
}) {
|
|
548
631
|
if (['distribution', 'both'].includes(cmrGranuleUrlType) && !distEndpoint) {
|
|
549
632
|
throw new Error(`cmrGranuleUrlType is ${cmrGranuleUrlType}, but no distribution endpoint is configured.`);
|
|
550
633
|
}
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
634
|
+
const [distributionUrls, s3Urls] = files.reduce(
|
|
635
|
+
(
|
|
636
|
+
/** @type {[Echo10URLObject[], Echo10URLObject[]]} */ [distributionAcc, s3Acc],
|
|
637
|
+
file
|
|
638
|
+
) => {
|
|
639
|
+
if (['both', 'distribution'].includes(cmrGranuleUrlType)) {
|
|
640
|
+
const url = constructOnlineAccessUrl({
|
|
641
|
+
file,
|
|
642
|
+
distEndpoint,
|
|
643
|
+
bucketTypes,
|
|
644
|
+
urlType: 'distribution',
|
|
645
|
+
distributionBucketMap,
|
|
646
|
+
useDirectS3Type,
|
|
647
|
+
});
|
|
648
|
+
if (url) distributionAcc.push(url);
|
|
649
|
+
}
|
|
650
|
+
if (['both', 's3'].includes(cmrGranuleUrlType)) {
|
|
651
|
+
const url = constructOnlineAccessUrl({
|
|
652
|
+
file,
|
|
653
|
+
distEndpoint,
|
|
654
|
+
bucketTypes,
|
|
655
|
+
urlType: 's3',
|
|
656
|
+
distributionBucketMap,
|
|
657
|
+
useDirectS3Type,
|
|
658
|
+
});
|
|
659
|
+
if (url) s3Acc.push(url);
|
|
660
|
+
}
|
|
661
|
+
return [distributionAcc, s3Acc];
|
|
662
|
+
}, [[], []]
|
|
663
|
+
);
|
|
577
664
|
const urlList = distributionUrls.concat(s3Urls);
|
|
578
665
|
return urlList.filter((urlObj) => urlObj);
|
|
579
666
|
}
|
|
@@ -589,7 +676,7 @@ function constructOnlineAccessUrls({
|
|
|
589
676
|
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
590
677
|
* mapping for all distribution buckets
|
|
591
678
|
* @param {boolean} params.useDirectS3Type - indicate if direct s3 access type is used
|
|
592
|
-
* @returns {
|
|
679
|
+
* @returns {[{URL: string, string, Description: string, Type: string}]}
|
|
593
680
|
* an array of online access url objects
|
|
594
681
|
*/
|
|
595
682
|
function constructRelatedUrls({
|
|
@@ -692,8 +779,9 @@ function mergeURLs(original, updated = [], removed = []) {
|
|
|
692
779
|
* Updates CMR JSON file with stringified 'metadataObject'
|
|
693
780
|
*
|
|
694
781
|
* @param {Object} metadataObject - JSON Object to stringify
|
|
695
|
-
* @param {
|
|
696
|
-
* @returns {Promise} returns promised
|
|
782
|
+
* @param {CmrFile} cmrFile - cmr file object to write body to
|
|
783
|
+
* @returns {Promise<{[key: string]: any, ETag?: string | undefined }>} returns promised
|
|
784
|
+
* promiseS3Upload response
|
|
697
785
|
*/
|
|
698
786
|
async function uploadUMMGJSONCMRFile(metadataObject, cmrFile) {
|
|
699
787
|
const tags = await s3GetObjectTagging(cmrFile.bucket, getS3KeyOfFile(cmrFile));
|
|
@@ -729,13 +817,19 @@ function shouldUseDirectS3Type(metadataObject) {
|
|
|
729
817
|
/**
|
|
730
818
|
* Update the UMMG cmr metadata object to have corrected urls
|
|
731
819
|
*
|
|
732
|
-
* @param {Object} params
|
|
733
|
-
* @param {
|
|
734
|
-
* @param {
|
|
735
|
-
*
|
|
736
|
-
* @param {
|
|
737
|
-
*
|
|
738
|
-
*
|
|
820
|
+
* @param {Object} params - Parameters for updating the metadata object
|
|
821
|
+
* @param {Object} params.metadataObject - The existing UMMG CMR metadata object to update
|
|
822
|
+
* @param {ApiFileWithFilePath[]} params.files - Array of file
|
|
823
|
+
* objects used to generate URLs
|
|
824
|
+
* @param {string} params.distEndpoint - Base URL for distribution endpoints (e.g., CloudFront)
|
|
825
|
+
* @param {{ [bucket: string]: string }} params.bucketTypes - Map of bucket names
|
|
826
|
+
* to types (e.g., public, protected)
|
|
827
|
+
* @param {string} [params.cmrGranuleUrlType='both'] - Type of URLs to generate: 'distribution',
|
|
828
|
+
* 's3', or 'both'
|
|
829
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Mapping of bucket names to
|
|
830
|
+
* distribution paths
|
|
831
|
+
*
|
|
832
|
+
* @returns {Object} - A deep clone of the original metadata object with updated RelatedUrls
|
|
739
833
|
*/
|
|
740
834
|
function updateUMMGMetadataObject({
|
|
741
835
|
metadataObject,
|
|
@@ -758,6 +852,7 @@ function updateUMMGMetadataObject({
|
|
|
758
852
|
});
|
|
759
853
|
|
|
760
854
|
const removedURLs = onlineAccessURLsToRemove(files, bucketTypes);
|
|
855
|
+
/** @type {Array<{ URL: string, Description?: string, Type?: string }>} */
|
|
761
856
|
const originalURLs = get(updatedMetadataObject, 'RelatedUrls', []);
|
|
762
857
|
const mergedURLs = mergeURLs(originalURLs, newURLs, removedURLs);
|
|
763
858
|
set(updatedMetadataObject, 'RelatedUrls', mergedURLs);
|
|
@@ -769,14 +864,19 @@ function updateUMMGMetadataObject({
|
|
|
769
864
|
* UMMG cmr.json file with this information.
|
|
770
865
|
*
|
|
771
866
|
* @param {Object} params - parameter object
|
|
772
|
-
* @param {
|
|
773
|
-
* @param {
|
|
867
|
+
* @param {CmrFile} params.cmrFile - cmr.json file whose contents will be updated.
|
|
868
|
+
* @param {ApiFileWithFilePath[]} params.files - array of moved file objects.
|
|
774
869
|
* @param {string} params.distEndpoint - distribution endpoint form config.
|
|
775
|
-
* @param {{ [
|
|
870
|
+
* @param {{ [bucket: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
776
871
|
* @param {string} params.cmrGranuleUrlType - cmrGranuleUrlType from config
|
|
777
872
|
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
778
873
|
* mapping for all distribution buckets
|
|
779
|
-
* @
|
|
874
|
+
* @param {string} params.producerGranuleId - producer granule id
|
|
875
|
+
* @param {string} params.granuleId - granule id
|
|
876
|
+
* @param {boolean} [params.updateGranuleIdentifiers=false] - whether to update the granule UR/add
|
|
877
|
+
* producerGranuleID to the CMR metadata object
|
|
878
|
+
* @param {any} [params.testOverrides] - overrides for testing
|
|
879
|
+
* @returns {Promise<{ metadataObject: Object, etag: string | undefined}>} an object
|
|
780
880
|
* containing a `metadataObject` (the updated UMMG metadata object) and the
|
|
781
881
|
* `etag` of the uploaded CMR file
|
|
782
882
|
*/
|
|
@@ -787,10 +887,18 @@ async function updateUMMGMetadata({
|
|
|
787
887
|
bucketTypes,
|
|
788
888
|
cmrGranuleUrlType = 'both',
|
|
789
889
|
distributionBucketMap,
|
|
890
|
+
producerGranuleId,
|
|
891
|
+
granuleId,
|
|
892
|
+
updateGranuleIdentifiers = false,
|
|
893
|
+
testOverrides = {},
|
|
790
894
|
}) {
|
|
895
|
+
const {
|
|
896
|
+
uploadUMMGJSONCMRFileMethod = uploadUMMGJSONCMRFile,
|
|
897
|
+
metadataObjectFromCMRJSONFileMethod = metadataObjectFromCMRJSONFile,
|
|
898
|
+
} = testOverrides;
|
|
791
899
|
const filename = getS3UrlOfFile(cmrFile);
|
|
792
|
-
const metadataObject = await
|
|
793
|
-
|
|
900
|
+
const metadataObject = await metadataObjectFromCMRJSONFileMethod(filename);
|
|
901
|
+
let updatedMetadataObject = updateUMMGMetadataObject({
|
|
794
902
|
metadataObject,
|
|
795
903
|
files,
|
|
796
904
|
distEndpoint,
|
|
@@ -798,7 +906,19 @@ async function updateUMMGMetadata({
|
|
|
798
906
|
cmrGranuleUrlType,
|
|
799
907
|
distributionBucketMap,
|
|
800
908
|
});
|
|
801
|
-
|
|
909
|
+
if (updateGranuleIdentifiers) {
|
|
910
|
+
// Type checks are needed as this callers/API are not all typed/ts converted yet
|
|
911
|
+
checkRequiredMetadataParms({ producerGranuleId, granuleId });
|
|
912
|
+
updatedMetadataObject = updateUMMGGranuleURAndGranuleIdentifier({
|
|
913
|
+
granuleUr: granuleId,
|
|
914
|
+
producerGranuleId,
|
|
915
|
+
metadataObject: updatedMetadataObject,
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
const { ETag: etag } = await uploadUMMGJSONCMRFileMethod(
|
|
919
|
+
updatedMetadataObject,
|
|
920
|
+
cmrFile
|
|
921
|
+
);
|
|
802
922
|
return { metadataObject: updatedMetadataObject, etag };
|
|
803
923
|
}
|
|
804
924
|
|
|
@@ -913,17 +1033,34 @@ function buildMergedEchoURLObject(URLlist = [], originalURLlist = [], removedURL
|
|
|
913
1033
|
}
|
|
914
1034
|
|
|
915
1035
|
/**
|
|
916
|
-
*
|
|
1036
|
+
* Updates the OnlineAccessURLs, OnlineResources, and AssociatedBrowseImageUrls
|
|
1037
|
+
* fields of an ECHO10 CMR metadata object with newly constructed URLs.
|
|
917
1038
|
*
|
|
918
|
-
*
|
|
919
|
-
*
|
|
920
|
-
*
|
|
921
|
-
*
|
|
922
|
-
*
|
|
923
|
-
*
|
|
924
|
-
* @
|
|
1039
|
+
* This function:
|
|
1040
|
+
* - Extracts the original URL sets from the ECHO10 XML metadata.
|
|
1041
|
+
* - Constructs new URL entries based on the provided file list and configuration.
|
|
1042
|
+
* - Merges new URLs with original ones, removing outdated or irrelevant URLs.
|
|
1043
|
+
* - Returns a new metadata object with an updated `Granule` field.
|
|
1044
|
+
*
|
|
1045
|
+
* @param {Object} params - Input parameters
|
|
1046
|
+
* @param {Echo10MetadataObject} params.metadataObject - The parsed ECHO10 metadata XML
|
|
1047
|
+
* object (as a JavaScript object), expected to include a `Granule` key
|
|
1048
|
+
* @param {ApiFileWithFilePath[]} params.files - Granule files to generate
|
|
1049
|
+
* updated URLs from
|
|
1050
|
+
* @param {string} params.distEndpoint - The base distribution endpoint URL
|
|
1051
|
+
* (e.g., CloudFront origin)
|
|
1052
|
+
* @param {{ [bucketName: string]: string }} params.bucketTypes - Mapping of bucket names
|
|
1053
|
+
* to access types ('public', 'protected', etc.)
|
|
1054
|
+
* @param {string} [params.cmrGranuleUrlType='both'] - Type of URLs to generate
|
|
1055
|
+
* for CMR: 'distribution', 's3', or 'both'
|
|
1056
|
+
* @param {DistributionBucketMap} params.distributionBucketMap - Maps S3 buckets to their
|
|
1057
|
+
* distribution URL paths
|
|
1058
|
+
*
|
|
1059
|
+
* @returns {Echo10MetadataObject} A new ECHO10 metadata object with updated
|
|
1060
|
+
* `Granule.OnlineAccessURLs`, `Granule.OnlineResources`, and `Granule.AssociatedBrowseImageUrls`
|
|
1061
|
+
* fields
|
|
925
1062
|
*/
|
|
926
|
-
function
|
|
1063
|
+
function updateEcho10XMLMetadataObjectUrls({
|
|
927
1064
|
metadataObject,
|
|
928
1065
|
files,
|
|
929
1066
|
distEndpoint,
|
|
@@ -934,12 +1071,25 @@ function updateEcho10XMLMetadataObject({
|
|
|
934
1071
|
const metadataGranule = metadataObject.Granule;
|
|
935
1072
|
const updatedGranule = { ...metadataGranule };
|
|
936
1073
|
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1074
|
+
/** @type {Echo10URLObject[]} */
|
|
1075
|
+
const originalOnlineAccessURLs = /** @type {Echo10URLObject[]} */ (
|
|
1076
|
+
/** @type {Echo10URLObject[]} */ ([]).concat(
|
|
1077
|
+
get(metadataGranule, 'OnlineAccessURLs.OnlineAccessURL') ?? []
|
|
1078
|
+
)
|
|
1079
|
+
);
|
|
1080
|
+
/** @type {Echo10URLObject[]} */
|
|
1081
|
+
const originalOnlineResourceURLs = /** @type {Echo10URLObject[]} */ (
|
|
1082
|
+
/** @type {Echo10URLObject[]} */ ([]).concat(
|
|
1083
|
+
get(metadataGranule, 'OnlineResources.OnlineResource') ?? []
|
|
1084
|
+
)
|
|
1085
|
+
);
|
|
1086
|
+
|
|
1087
|
+
/** @type {Echo10URLObject[]} */
|
|
1088
|
+
const originalAssociatedBrowseURLs = /** @type {Echo10URLObject[]} */ (
|
|
1089
|
+
/** @type {Echo10URLObject[]} */ ([]).concat(
|
|
1090
|
+
get(metadataGranule, 'AssociatedBrowseImageUrls.ProviderBrowseUrl') ?? []
|
|
1091
|
+
)
|
|
1092
|
+
);
|
|
943
1093
|
|
|
944
1094
|
const removedURLs = onlineAccessURLsToRemove(files, bucketTypes);
|
|
945
1095
|
const newURLs = constructOnlineAccessUrls({
|
|
@@ -968,32 +1118,52 @@ function updateEcho10XMLMetadataObject({
|
|
|
968
1118
|
}
|
|
969
1119
|
|
|
970
1120
|
/**
|
|
971
|
-
*
|
|
972
|
-
*
|
|
1121
|
+
* Updates an ECHO10 CMR XML metadata file on S3 to reflect new URLs and optionally
|
|
1122
|
+
* a new GranuleUR and ProducerGranuleId.
|
|
973
1123
|
*
|
|
974
|
-
* @param {Object} params
|
|
975
|
-
* @param {
|
|
976
|
-
* @param {
|
|
977
|
-
* @param {
|
|
978
|
-
* @param {
|
|
979
|
-
*
|
|
980
|
-
*
|
|
981
|
-
* @
|
|
982
|
-
*
|
|
1124
|
+
* @param {Object} params
|
|
1125
|
+
* @param {string} params.granuleId - New GranuleUR to set in metadata
|
|
1126
|
+
* @param {string} params.producerGranuleId - Original ProducerGranuleId to record
|
|
1127
|
+
* @param {CmrFile} params.cmrFile - The cmr xml file to be updated
|
|
1128
|
+
* @param {ApiFileWithFilePath[]} params.files - List of granule files used
|
|
1129
|
+
* to generate OnlineAccess URLs
|
|
1130
|
+
* @param {string} params.distEndpoint - Distribution endpoint for download URLs
|
|
1131
|
+
* @param {{ [bucket: string]: string }} params.bucketTypes - Mapping of bucket names to their types
|
|
1132
|
+
* @param {string} [params.cmrGranuleUrlType]
|
|
1133
|
+
* - Type of URLs to generate ('distribution' | 's3' | 'both')
|
|
1134
|
+
* @param {DistributionBucketMap} params.distributionBucketMap
|
|
1135
|
+
* - Maps buckets to distribution paths
|
|
1136
|
+
* @param {boolean} [params.updateGranuleIdentifiers]
|
|
1137
|
+
* - If true, update the GranuleUR and ProducerGranuleId in metadata
|
|
1138
|
+
* @param {any} [params.testOverrides]
|
|
1139
|
+
* - Optional test overrides for internal functions
|
|
1140
|
+
* @returns {Promise<{ metadataObject: any, etag: string }>}
|
|
1141
|
+
* The updated metadata object and resulting ETag
|
|
983
1142
|
*/
|
|
1143
|
+
|
|
984
1144
|
async function updateEcho10XMLMetadata({
|
|
1145
|
+
granuleId,
|
|
1146
|
+
producerGranuleId,
|
|
985
1147
|
cmrFile,
|
|
986
1148
|
files,
|
|
987
1149
|
distEndpoint,
|
|
988
1150
|
bucketTypes,
|
|
989
1151
|
cmrGranuleUrlType = 'both',
|
|
990
1152
|
distributionBucketMap,
|
|
1153
|
+
updateGranuleIdentifiers = false,
|
|
1154
|
+
testOverrides = {},
|
|
991
1155
|
}) {
|
|
1156
|
+
const {
|
|
1157
|
+
generateEcho10XMLStringMethod = generateEcho10XMLString,
|
|
1158
|
+
uploadEcho10CMRFileMethod = uploadEcho10CMRFile,
|
|
1159
|
+
metadataObjectFromCMRXMLFileMethod = metadataObjectFromCMRXMLFile,
|
|
1160
|
+
} = testOverrides;
|
|
1161
|
+
|
|
992
1162
|
// add/replace the OnlineAccessUrls
|
|
993
1163
|
const filename = getS3UrlOfFile(cmrFile);
|
|
994
|
-
const metadataObject = await
|
|
1164
|
+
const metadataObject = await metadataObjectFromCMRXMLFileMethod(filename);
|
|
995
1165
|
|
|
996
|
-
|
|
1166
|
+
let updatedMetadataObject = updateEcho10XMLMetadataObjectUrls({
|
|
997
1167
|
metadataObject,
|
|
998
1168
|
files,
|
|
999
1169
|
distEndpoint,
|
|
@@ -1001,8 +1171,25 @@ async function updateEcho10XMLMetadata({
|
|
|
1001
1171
|
cmrGranuleUrlType,
|
|
1002
1172
|
distributionBucketMap,
|
|
1003
1173
|
});
|
|
1004
|
-
|
|
1005
|
-
|
|
1174
|
+
|
|
1175
|
+
if (updateGranuleIdentifiers) {
|
|
1176
|
+
// Type checks are needed as this callers/API are not all typed/ts converted yet
|
|
1177
|
+
try {
|
|
1178
|
+
checkRequiredMetadataParms({ producerGranuleId, granuleId });
|
|
1179
|
+
} catch (error) {
|
|
1180
|
+
throw new Error(
|
|
1181
|
+
`updateGranuleIdentifiers was set, but producerGranuleId ${producerGranuleId} or granuleId ${granuleId} is not set.`,
|
|
1182
|
+
{ cause: error }
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
1185
|
+
updatedMetadataObject = updateEcho10XMLGranuleUrAndGranuleIdentifier({
|
|
1186
|
+
granuleUr: granuleId,
|
|
1187
|
+
producerGranuleId,
|
|
1188
|
+
xml: updatedMetadataObject,
|
|
1189
|
+
});
|
|
1190
|
+
}
|
|
1191
|
+
const xml = generateEcho10XMLStringMethod(updatedMetadataObject.Granule);
|
|
1192
|
+
const { ETag: etag } = await uploadEcho10CMRFileMethod(xml, cmrFile);
|
|
1006
1193
|
return { metadataObject: updatedMetadataObject, etag };
|
|
1007
1194
|
}
|
|
1008
1195
|
|
|
@@ -1011,12 +1198,17 @@ async function updateEcho10XMLMetadata({
|
|
|
1011
1198
|
*
|
|
1012
1199
|
* @param {Object} params - parameter object
|
|
1013
1200
|
* @param {string} params.granuleId - granuleId
|
|
1014
|
-
* @param {
|
|
1015
|
-
* @param {
|
|
1201
|
+
* @param {string} [params.producerGranuleId] - producer granuleId
|
|
1202
|
+
* @param {CmrFile} params.cmrFile - cmr file to be updated
|
|
1203
|
+
* @param {ApiFileWithFilePath[]} params.files - array of file objects
|
|
1016
1204
|
* @param {string} params.distEndpoint - distribution enpoint from config
|
|
1017
1205
|
* @param {boolean} params.published - indicate if publish is needed
|
|
1018
1206
|
* @param {{ [key: string]: string }} params.bucketTypes - map of bucket names to bucket types
|
|
1019
1207
|
* @param {string} params.cmrGranuleUrlType - type of granule CMR url
|
|
1208
|
+
* @param {boolean} [params.updateGranuleIdentifiers]
|
|
1209
|
+
* - If true, update the GranuleUR and ProducerGranuleId in metadata
|
|
1210
|
+
* @param {any} [params.testOverrides]
|
|
1211
|
+
* - Optional test overrides for internal functions
|
|
1020
1212
|
* @param {DistributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path
|
|
1021
1213
|
* mapping for all distribution buckets
|
|
1022
1214
|
* @returns {Promise<Object>} CMR file object with the `etag` of the newly
|
|
@@ -1024,26 +1216,37 @@ async function updateEcho10XMLMetadata({
|
|
|
1024
1216
|
*/
|
|
1025
1217
|
async function updateCMRMetadata({
|
|
1026
1218
|
granuleId,
|
|
1219
|
+
producerGranuleId,
|
|
1027
1220
|
cmrFile,
|
|
1028
1221
|
files,
|
|
1029
1222
|
distEndpoint,
|
|
1030
1223
|
published,
|
|
1031
1224
|
bucketTypes,
|
|
1032
1225
|
cmrGranuleUrlType = 'both',
|
|
1226
|
+
updateGranuleIdentifiers = false,
|
|
1033
1227
|
distributionBucketMap,
|
|
1228
|
+
testOverrides = {},
|
|
1034
1229
|
}) {
|
|
1230
|
+
const {
|
|
1231
|
+
publish2CMRMethod = publish2CMR,
|
|
1232
|
+
getCmrSettingsMethod = getCmrSettings,
|
|
1233
|
+
} = testOverrides;
|
|
1234
|
+
|
|
1035
1235
|
const filename = getS3UrlOfFile(cmrFile);
|
|
1036
1236
|
|
|
1037
|
-
log.debug(`cmrjs.updateCMRMetadata granuleId ${granuleId}
|
|
1237
|
+
log.debug(`cmrjs.updateCMRMetadata granuleId ${granuleId} cmrMetadata file ${filename}`);
|
|
1038
1238
|
|
|
1039
|
-
const cmrCredentials = (published) ? await
|
|
1239
|
+
const cmrCredentials = (published) ? await getCmrSettingsMethod() : {};
|
|
1040
1240
|
const params = {
|
|
1041
|
-
cmrFile,
|
|
1042
|
-
files,
|
|
1043
|
-
distEndpoint,
|
|
1044
1241
|
bucketTypes,
|
|
1242
|
+
cmrFile,
|
|
1045
1243
|
cmrGranuleUrlType,
|
|
1244
|
+
distEndpoint,
|
|
1046
1245
|
distributionBucketMap,
|
|
1246
|
+
files,
|
|
1247
|
+
granuleId,
|
|
1248
|
+
producerGranuleId: producerGranuleId || granuleId,
|
|
1249
|
+
updateGranuleIdentifiers,
|
|
1047
1250
|
};
|
|
1048
1251
|
|
|
1049
1252
|
let metadataObject;
|
|
@@ -1065,7 +1268,7 @@ async function updateCMRMetadata({
|
|
|
1065
1268
|
granuleId,
|
|
1066
1269
|
};
|
|
1067
1270
|
|
|
1068
|
-
return { ...await
|
|
1271
|
+
return { ...await publish2CMRMethod(cmrPublishObject, cmrCredentials), etag };
|
|
1069
1272
|
}
|
|
1070
1273
|
|
|
1071
1274
|
return { ...cmrFile, etag };
|
|
@@ -1400,21 +1603,23 @@ const getCMRCollectionId = (
|
|
|
1400
1603
|
|
|
1401
1604
|
module.exports = {
|
|
1402
1605
|
addEtagsToFileObjects,
|
|
1606
|
+
buildCMRQuery,
|
|
1403
1607
|
constructCmrConceptLink,
|
|
1404
1608
|
constructOnlineAccessUrl,
|
|
1405
1609
|
constructOnlineAccessUrls,
|
|
1406
1610
|
generateEcho10XMLString,
|
|
1407
1611
|
generateFileUrl,
|
|
1408
|
-
granuleToCmrFileObject,
|
|
1409
|
-
getCmrSettings,
|
|
1410
1612
|
getCMRCollectionId,
|
|
1613
|
+
getCmrSettings,
|
|
1614
|
+
getCollectionsByShortNameAndVersion,
|
|
1411
1615
|
getFileDescription,
|
|
1412
1616
|
getFilename,
|
|
1413
1617
|
getGranuleTemporalInfo,
|
|
1414
|
-
getCollectionsByShortNameAndVersion,
|
|
1415
1618
|
getS3UrlOfFile,
|
|
1416
1619
|
getUserAccessibleBuckets,
|
|
1620
|
+
getXMLMetadataAsString,
|
|
1417
1621
|
granulesToCmrFileObjects,
|
|
1622
|
+
granuleToCmrFileObject,
|
|
1418
1623
|
isCMRFile,
|
|
1419
1624
|
isCMRFilename,
|
|
1420
1625
|
isCMRISOFilename,
|
|
@@ -1424,15 +1629,18 @@ module.exports = {
|
|
|
1424
1629
|
isUMMGFilename,
|
|
1425
1630
|
mapFileEtags,
|
|
1426
1631
|
metadataObjectFromCMRFile,
|
|
1632
|
+
parseXmlString,
|
|
1427
1633
|
publish2CMR,
|
|
1428
1634
|
reconcileCMRMetadata,
|
|
1429
1635
|
removeEtagsFromFileObjects,
|
|
1430
1636
|
removeFromCMR,
|
|
1431
|
-
updateCMRMetadata,
|
|
1432
|
-
updateEcho10XMLMetadataObject,
|
|
1433
|
-
updateUMMGMetadataObject,
|
|
1434
1637
|
setECHO10Collection,
|
|
1435
1638
|
setUMMGCollection,
|
|
1639
|
+
updateCMRMetadata,
|
|
1640
|
+
updateEcho10XMLMetadata,
|
|
1641
|
+
updateEcho10XMLMetadataObjectUrls,
|
|
1642
|
+
updateUMMGMetadata,
|
|
1643
|
+
updateUMMGMetadataObject,
|
|
1436
1644
|
uploadEcho10CMRFile,
|
|
1437
1645
|
uploadUMMGJSONCMRFile,
|
|
1438
1646
|
};
|