@cumulus/ingest 20.3.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/granule.d.ts +15 -3
- package/granule.d.ts.map +1 -1
- package/granule.js +31 -4
- package/granule.js.map +1 -1
- package/package.json +13 -13
- package/providerClientUtils.d.ts +3 -2
- package/providerClientUtils.d.ts.map +1 -1
- package/providerClientUtils.js +2 -1
- package/providerClientUtils.js.map +1 -1
- package/src/granule.ts +36 -6
- package/src/providerClientUtils.js +2 -1
- package/src/url-path-template.js +34 -26
- package/tsconfig.tsbuildinfo +1 -1
- package/url-path-template.d.ts.map +1 -1
- package/url-path-template.js +29 -25
- package/url-path-template.js.map +1 -1
package/granule.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
2
1
|
import { FilePgModel, Knex } from '@cumulus/db';
|
|
2
|
+
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
3
3
|
export interface EventWithDuplicateHandling {
|
|
4
4
|
config: {
|
|
5
5
|
collection: {
|
|
@@ -131,7 +131,7 @@ export declare function handleDuplicateFile(params: {
|
|
|
131
131
|
}) => Promise<void>;
|
|
132
132
|
ACL?: string;
|
|
133
133
|
sourceBucket?: string;
|
|
134
|
-
fileRemotePath
|
|
134
|
+
fileRemotePath?: string;
|
|
135
135
|
s3Object?: {
|
|
136
136
|
moveObject: Function;
|
|
137
137
|
};
|
|
@@ -180,6 +180,18 @@ export declare function moveGranuleFile(moveFileParam: MoveFileParams, filesPgMo
|
|
|
180
180
|
* @returns {string} - filename with timestamp removed
|
|
181
181
|
*/
|
|
182
182
|
export declare function unversionFilename(filename: string): string;
|
|
183
|
+
/**
|
|
184
|
+
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
185
|
+
* a producer provided granule object
|
|
186
|
+
*
|
|
187
|
+
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
188
|
+
* assigned by the granule producer
|
|
189
|
+
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
190
|
+
* @param hashLength - The length of the hash to append to the granuleId.
|
|
191
|
+
* @param includeTimestampHashKey - Boolean value for whether hash string should contain timestamp
|
|
192
|
+
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
193
|
+
*/
|
|
194
|
+
export declare function generateUniqueGranuleId(id: string, collectionId: string, hashLength: number, includeTimestampHashKey?: boolean): string;
|
|
183
195
|
/**
|
|
184
196
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
185
197
|
*
|
|
@@ -187,7 +199,7 @@ export declare function unversionFilename(filename: string): string;
|
|
|
187
199
|
* @param {Object} event.config - the config object
|
|
188
200
|
* @param {Object} event.config.collection - collection object.
|
|
189
201
|
|
|
190
|
-
* @returns {
|
|
202
|
+
* @returns {DuplicateHandling} - duplicate handling directive.
|
|
191
203
|
*/
|
|
192
204
|
export declare function duplicateHandlingType(event: EventWithDuplicateHandling): DuplicateHandling;
|
|
193
205
|
//# sourceMappingURL=granule.d.ts.map
|
package/granule.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,IAAI,EAAkC,MAAM,aAAa,CAAC;AAGhF,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAE5D,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE;QACN,UAAU,EAAE;YACV,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;SACtC,CAAC;QACF,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;KACvC,CAAC;IACF,cAAc,CAAC,EAAE;QACf,eAAe,CAAC,EAAE;YAChB,uBAAuB,CAAC,EAAE,OAAO,CAAA;SAClC,CAAA;KACF,CAAC;CACH;AAED,MAAM,WAAW,IAAI;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,IAAI,EAAE,IAAI,CAAA;CACX;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;;IAMI;AACJ,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED;;;;;;IAMI;AACJ,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,eAAe,EAAE,CAAC,CAc5B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,6BAA6B,CACjD,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,oBAAoB,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,EACvE,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,eAAe,EAAE,CAAC,CAoC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAAE;IAChD,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,EAAE,MAAM,CAAC;KACxB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE;QAAE,UAAU,EAAE,QAAQ,CAAA;KAAE,CAAC;IACpC,qCAAqC,CAAC,EAAE,QAAQ,CAAC;CAClD,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAuE7B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,SAAU,IAAI,KAAG,MAAM,GAAG,SAQnD,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,IAAI,EAAE,EACnB,YAAY,EAAE;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CACvB,EAAE,GACF,cAAc,EAAE,CAsClB;AAED;;;;;;;;EAQE;AACF,wBAAsB,eAAe,CACnC,aAAa,EAAE,cAAc,EAC7B,YAAY,EAAE,WAAW,EACzB,GAAG,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,EAC5B,wBAAwB,EAAE,MAAM,GAAG,SAAS,GAC3C,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAuDrC;AAaD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAI1D;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,CAAC,EAAE,OAAO,GACtF,MAAM,CAOR;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,0BAA0B,GAChC,iBAAiB,CAQnB"}
|
package/granule.js
CHANGED
|
@@ -26,13 +26,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.duplicateHandlingType = exports.unversionFilename = exports.moveGranuleFile = exports.generateMoveFileParams = exports.getNameOfFile = exports.handleDuplicateFile = exports.moveGranuleFileWithVersioning = exports.listVersionedObjects = exports.renameS3FileWithTimestamp = void 0;
|
|
30
|
-
const
|
|
29
|
+
exports.duplicateHandlingType = exports.generateUniqueGranuleId = exports.unversionFilename = exports.moveGranuleFile = exports.generateMoveFileParams = exports.getNameOfFile = exports.handleDuplicateFile = exports.moveGranuleFileWithVersioning = exports.listVersionedObjects = exports.renameS3FileWithTimestamp = void 0;
|
|
30
|
+
const crypto = __importStar(require("crypto"));
|
|
31
31
|
const moment_1 = __importDefault(require("moment"));
|
|
32
|
-
const services_1 = require("@cumulus/aws-client/services");
|
|
33
32
|
const S3 = __importStar(require("@cumulus/aws-client/S3"));
|
|
33
|
+
const services_1 = require("@cumulus/aws-client/services");
|
|
34
34
|
const log = __importStar(require("@cumulus/common/log"));
|
|
35
35
|
const db_1 = require("@cumulus/db");
|
|
36
|
+
const errors = __importStar(require("@cumulus/errors"));
|
|
36
37
|
const errors_1 = require("@cumulus/errors");
|
|
37
38
|
/**
|
|
38
39
|
* rename s3 file with timestamp
|
|
@@ -175,6 +176,9 @@ async function handleDuplicateFile(params) {
|
|
|
175
176
|
else if (duplicateHandling === 'version') {
|
|
176
177
|
// sync to staging location if required
|
|
177
178
|
if (syncFileFunction) {
|
|
179
|
+
if (!fileRemotePath) {
|
|
180
|
+
throw new Error('fileRemotePath must be defined if syncFileFunction is provided');
|
|
181
|
+
}
|
|
178
182
|
await syncFileFunction({
|
|
179
183
|
bucket: sourceBucket,
|
|
180
184
|
destinationBucket: source.Bucket,
|
|
@@ -193,6 +197,9 @@ async function handleDuplicateFile(params) {
|
|
|
193
197
|
}
|
|
194
198
|
else if (duplicateHandling === 'replace') {
|
|
195
199
|
if (syncFileFunction) {
|
|
200
|
+
if (!fileRemotePath) {
|
|
201
|
+
throw new Error('fileRemotePath must be defined if syncFileFunction is provided');
|
|
202
|
+
}
|
|
196
203
|
// sync directly to target location
|
|
197
204
|
await syncFileFunction({
|
|
198
205
|
destinationBucket: target.Bucket,
|
|
@@ -361,6 +368,26 @@ function unversionFilename(filename) {
|
|
|
361
368
|
: filename;
|
|
362
369
|
}
|
|
363
370
|
exports.unversionFilename = unversionFilename;
|
|
371
|
+
/**
|
|
372
|
+
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
373
|
+
* a producer provided granule object
|
|
374
|
+
*
|
|
375
|
+
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
376
|
+
* assigned by the granule producer
|
|
377
|
+
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
378
|
+
* @param hashLength - The length of the hash to append to the granuleId.
|
|
379
|
+
* @param includeTimestampHashKey - Boolean value for whether hash string should contain timestamp
|
|
380
|
+
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
381
|
+
*/
|
|
382
|
+
function generateUniqueGranuleId(id, collectionId, hashLength, includeTimestampHashKey) {
|
|
383
|
+
// use MD5 to generate truncated hash of granule object
|
|
384
|
+
const hashStringWithTimestamp = `${collectionId}_${process.hrtime.bigint().toString()}`;
|
|
385
|
+
const hashStringWithoutTimestamp = `${collectionId}`;
|
|
386
|
+
const hashString = includeTimestampHashKey ? hashStringWithTimestamp : hashStringWithoutTimestamp;
|
|
387
|
+
const hashBuffer = crypto.createHash('md5').update(hashString).digest();
|
|
388
|
+
return `${id}_${hashBuffer.toString('base64url').replace(/_/g, '').slice(0, hashLength)}`;
|
|
389
|
+
}
|
|
390
|
+
exports.generateUniqueGranuleId = generateUniqueGranuleId;
|
|
364
391
|
/**
|
|
365
392
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
366
393
|
*
|
|
@@ -368,7 +395,7 @@ exports.unversionFilename = unversionFilename;
|
|
|
368
395
|
* @param {Object} event.config - the config object
|
|
369
396
|
* @param {Object} event.config.collection - collection object.
|
|
370
397
|
|
|
371
|
-
* @returns {
|
|
398
|
+
* @returns {DuplicateHandling} - duplicate handling directive.
|
|
372
399
|
*/
|
|
373
400
|
function duplicateHandlingType(event) {
|
|
374
401
|
if (event?.cumulus_config?.cumulus_context?.forceDuplicateOverwrite) {
|
package/granule.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA
|
|
1
|
+
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AACjC,oDAA4B;AAE5B,2DAA6C;AAC7C,2DAAkD;AAClD,yDAA2C;AAC3C,oCAAgF;AAChF,wDAA0C;AAC1C,4CAAqD;AAkDrD;;;;;;IAMI;AACG,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,GAAW;IAEX,MAAM,YAAY,GAAG,oBAAoB,CAAC;IAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;IAElE,IAAI,CAAC,SAAS,EAAE;QACd,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,IAAI,GAAG,wCAAwC,CAAC,CAAC;KAChF;IAED,IAAI,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;IAEzE,qDAAqD;IACrD,4CAA4C;IAC5C,OAAO,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE;QACnE,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;KAC7F;IAED,GAAG,CAAC,KAAK,CAAC,sCAAsC,MAAM,IAAI,GAAG,OAAO,UAAU,EAAE,CAAC,CAAC;IAElF,MAAM,EAAE,CAAC,UAAU,CAAC;QAClB,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,GAAG;QACd,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,UAAU;QAC1B,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;AACL,CAAC;AA5BD,8DA4BC;AAED;;;;;;IAMI;AACG,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,GAAW;IAEX,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC;QACtC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,GAAG,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE;QACV,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,EAAE,MAAM;YACd,GAAG;YACH,IAAI,EAAE,IAAI;SACX,CAAC,CAAC,CAAC;KACL;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAjBD,oDAiBC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACI,KAAK,UAAU,6BAA6B,CACjD,MAAuC,EACvC,MAAuC,EACvC,uBAAqE,EAAE,EACvE,GAAY;IAEZ,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC;IACxD,sFAAsF;IACtF,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,mBAAmB,CAAC;QACjD,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,YAAY,IAAI,OAAO;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,QAAQ,IAAI,MAAM,EAAE,CAAC,mBAAmB,CAAC;QAC7D,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,2FAA2F;IAC3F,yEAAyE;IACzE,IAAI,aAAa,KAAK,aAAa,EAAE;QACnC,MAAM,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACpD;SAAM;QACL,GAAG,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QACvC,MAAM,yBAAyB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAE3D,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;YACd,GAAG;SACJ,CAAC,CAAC;KACJ;IACD,uBAAuB;IACvB,OAAO,oBAAoB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAzCD,sEAyCC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACI,KAAK,UAAU,mBAAmB,CAAC,MAgBzC;IACC,MAAM,EACJ,GAAG,EACH,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,EACd,qCAAqC,GAAG,6BAA6B,EACrE,QAAQ,GAAG,EAAE,EACb,MAAM,EACN,YAAY,EACZ,gBAAgB,EAChB,MAAM,GACP,GAAG,MAAM,CAAC;IAEX,IAAI,iBAAiB,KAAK,OAAO,EAAE;QACjC,wEAAwE;QACxE,sDAAsD;QACtD,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC,GAAG,sBAAsB,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;KAC3F;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,uCAAuC;QACvC,IAAI,gBAAgB,EAAE;YACpB,IAAI,CAAC,cAAc,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;aACnF;YACD,MAAM,gBAAgB,CAAC;gBACrB,MAAM,EAAE,YAAY;gBACpB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,cAAc;aACf,CAAC,CAAC;SACJ;QACD,IAAI,oBAAoB,GAAG,EAAE,CAAC;QAC9B,IAAI,gBAAgB,EAAE;YACpB,mBAAmB;YACnB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACnF,oBAAoB,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;SACnD;QACD,+BAA+B;QAC/B,OAAO,qCAAqC,CAC1C,MAAM,EACN,MAAM,EACN,oBAAoB,EACpB,GAAG,CACJ,CAAC;KACH;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,IAAI,gBAAgB,EAAE;YACpB,IAAI,CAAC,cAAc,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;aACnF;YACD,mCAAmC;YACnC,MAAM,gBAAgB,CAAC;gBACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,MAAM,EAAE,YAAY;gBACpB,cAAc;aACf,CAAC,CAAC;SACJ;aAAM;YACL,MAAM,QAAQ,CAAC,UAAU,CAAC;gBACxB,GAAG;gBACH,QAAQ,EAAE,IAAI;gBACd,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,YAAY,EAAE,MAAM,CAAC,MAAM;gBAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;aACtB,CAAC,CAAC;SACJ;QACD,mCAAmC;QACnC,IAAI,gBAAgB;YAAE,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACzE;IACD,uCAAuC;IACvC,OAAO,EAAE,CAAC;AACZ,CAAC;AAvFD,kDAuFC;AAED;;;;;;;GAOG;AACI,MAAM,aAAa,GAAG,CAAC,IAAU,EAAsB,EAAE;IAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC;IAE5C,IAAI,CAAC,QAAQ,EAAE;QACb,OAAO,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;KACnC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AARW,QAAA,aAAa,iBAQxB;AAEF;;;;;;;;;;;GAWG;AACH,SAAgB,sBAAsB,CACpC,WAAmB,EACnB,YAIG;IAEH,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAErC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAE5C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5E,wCAAwC;QACxC,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAElC,IAAI,MAAM,CAAC;QACX,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,GAAG;gBACP,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC;SACH;aAAM,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACvC;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,wEAAwE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACjH;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,QAAQ;YACpC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,IAAI,IAAA,qBAAa,EAAC,IAAI,CAAC,EAAE;YAClD,CAAC,CAAC,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAExB,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,OAAO,EAAE,IAAI,EAAE,CAAC;SACjB;QAED,MAAM,MAAM,GAAG;YACb,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,GAAG,EAAE,SAAS;SACf,CAAC;QAEF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AA7CD,wDA6CC;AAED;;;;;;;;EAQE;AACK,KAAK,UAAU,eAAe,CACnC,aAA6B,EAC7B,YAAyB,EACzB,GAA4B,EAC5B,wBAA4C;IAE5C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;IAE/C,IAAI,MAAM,IAAI,MAAM,EAAE;QACpB,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,wBAAwB,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;SAC7G;QACD,MAAM,gBAAgB,GAAG,MAAM,YAAY,CAAC,MAAM,CAChD,GAAG,EACH;YACE,kBAAkB,EAAE,wBAAwB;YAC5C,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,EACD;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,IAAA,qBAAa,EAAC,IAAI,CAAC;SAC/B,EAAE,CAAC,GAAG,CAAC,CACT,CAAC;QACF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE;YACjC,MAAM,IAAI,2BAAkB,CAAC,iEAAiE,CAAC,CAAC;SACjG;QACD,MAAM,eAAe,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAE5C,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,IAAA,mCAA8B,EAAC,eAAe,CAAC;YAClD,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,IAAA,qBAAa,EAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;SAC7C,CAAC;KACH;IAED,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE;QAC/C,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;KACvB;IAED,MAAM,kBAAkB,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;QACrD,kBAAkB,EAAE,wBAAwB;QAC5C,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,GAAG,EAAE,IAAI,CAAC,GAAG;KACd,CAAC,CAAC;IAEH,OAAO,IAAA,mCAA8B,EAAC,kBAAkB,CAAC,CAAC;AAC5D,CAAC;AA5DD,0CA4DC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,WAAW,GAAG,uGAAuG,CAAC;IAC5H,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,QAAgB;IAChD,OAAO,aAAa,CAAC,QAAQ,CAAC;QAC5B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC5C,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC;AAJD,8CAIC;AAED;;;;;;;;;;GAUG;AACH,SAAgB,uBAAuB,CACrC,EAAU,EAAE,YAAoB,EAAE,UAAkB,EAAE,uBAAiC;IAEvF,uDAAuD;IACvD,MAAM,uBAAuB,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;IACxF,MAAM,0BAA0B,GAAG,GAAG,YAAY,EAAE,CAAC;IACrD,MAAM,UAAU,GAAG,uBAAuB,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAClG,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC;IACxE,OAAO,GAAG,EAAE,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;AAC5F,CAAC;AATD,0DASC;AAED;;;;;;;;GAQG;AACH,SAAgB,qBAAqB,CACnC,KAAiC;IAEjC,IAAI,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,uBAAuB,EAAE;QACnE,OAAO,SAAS,CAAC;KAClB;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,iBAAiB;WAChC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB;WACzC,OAAO,CAAC;AACf,CAAC;AAVD,sDAUC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "21.0.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=20.12.2"
|
|
@@ -41,13 +41,13 @@
|
|
|
41
41
|
"author": "Cumulus Authors",
|
|
42
42
|
"license": "Apache-2.0",
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@cumulus/aws-client": "
|
|
45
|
-
"@cumulus/common": "
|
|
46
|
-
"@cumulus/db": "
|
|
47
|
-
"@cumulus/errors": "
|
|
48
|
-
"@cumulus/logger": "
|
|
49
|
-
"@cumulus/message": "
|
|
50
|
-
"@cumulus/sftp-client": "
|
|
44
|
+
"@cumulus/aws-client": "21.0.0",
|
|
45
|
+
"@cumulus/common": "21.0.0",
|
|
46
|
+
"@cumulus/db": "21.0.0",
|
|
47
|
+
"@cumulus/errors": "21.0.0",
|
|
48
|
+
"@cumulus/logger": "21.0.0",
|
|
49
|
+
"@cumulus/message": "21.0.0",
|
|
50
|
+
"@cumulus/sftp-client": "21.0.0",
|
|
51
51
|
"cksum": "^1.3.0",
|
|
52
52
|
"encodeurl": "^1.0.2",
|
|
53
53
|
"fs-extra": "^5.0.0",
|
|
@@ -61,10 +61,10 @@
|
|
|
61
61
|
"tough-cookie": "~4.0.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cumulus/checksum": "
|
|
65
|
-
"@cumulus/cmrjs": "
|
|
66
|
-
"@cumulus/test-data": "
|
|
67
|
-
"@cumulus/types": "
|
|
64
|
+
"@cumulus/checksum": "21.0.0",
|
|
65
|
+
"@cumulus/cmrjs": "21.0.0",
|
|
66
|
+
"@cumulus/test-data": "21.0.0",
|
|
67
|
+
"@cumulus/types": "21.0.0"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "19bb3477969662a9e0b300f10f6df23b6c0654db"
|
|
70
70
|
}
|
package/providerClientUtils.d.ts
CHANGED
|
@@ -11,13 +11,14 @@ export function buildProviderClient(providerConfig?: Object): Object;
|
|
|
11
11
|
* @param {Object} param
|
|
12
12
|
* @param {Object} param.providerClient - a provider client
|
|
13
13
|
* @param {string} param.remotePath - the path of the file to fetch
|
|
14
|
-
* @param {string} param.remoteAltBucket - alternate per-file bucket override to
|
|
14
|
+
* @param {string} [param.remoteAltBucket] - alternate per-file bucket override to
|
|
15
|
+
* the providerClient
|
|
15
16
|
* bucket
|
|
16
17
|
* @returns {Promise<string>} the contents of the remote file
|
|
17
18
|
*/
|
|
18
19
|
export function fetchTextFile({ providerClient, remotePath, remoteAltBucket }: {
|
|
19
20
|
providerClient: Object;
|
|
20
21
|
remotePath: string;
|
|
21
|
-
remoteAltBucket
|
|
22
|
+
remoteAltBucket?: string | undefined;
|
|
22
23
|
}): Promise<string>;
|
|
23
24
|
//# sourceMappingURL=providerClientUtils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providerClientUtils.d.ts","sourceRoot":"","sources":["src/providerClientUtils.js"],"names":[],"mappings":"AAWA;;;;;GAKG;AACH,qDAHW,MAAM,GACJ,MAAM,CAgBlB;AAED
|
|
1
|
+
{"version":3,"file":"providerClientUtils.d.ts","sourceRoot":"","sources":["src/providerClientUtils.js"],"names":[],"mappings":"AAWA;;;;;GAKG;AACH,qDAHW,MAAM,GACJ,MAAM,CAgBlB;AAED;;;;;;;;;;GAUG;AACH;IAPyB,cAAc,EAA5B,MAAM;IACQ,UAAU,EAAxB,MAAM;IACS,eAAe;IAG5B,QAAQ,MAAM,CAAC,CAW3B"}
|
package/providerClientUtils.js
CHANGED
|
@@ -34,7 +34,8 @@ const buildProviderClient = (providerConfig = {}) => {
|
|
|
34
34
|
* @param {Object} param
|
|
35
35
|
* @param {Object} param.providerClient - a provider client
|
|
36
36
|
* @param {string} param.remotePath - the path of the file to fetch
|
|
37
|
-
* @param {string} param.remoteAltBucket - alternate per-file bucket override to
|
|
37
|
+
* @param {string} [param.remoteAltBucket] - alternate per-file bucket override to
|
|
38
|
+
* the providerClient
|
|
38
39
|
* bucket
|
|
39
40
|
* @returns {Promise<string>} the contents of the remote file
|
|
40
41
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"providerClientUtils.js","sourceRoot":"","sources":["src/providerClientUtils.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACzB,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAC7B,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AACzD,MAAM,kBAAkB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAC3D,MAAM,gBAAgB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AACvD,MAAM,kBAAkB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,CAAC,cAAc,GAAG,EAAE,EAAE,EAAE;IAClD,QAAQ,cAAc,CAAC,QAAQ,EAAE;QACjC,KAAK,KAAK;YACR,OAAO,IAAI,iBAAiB,CAAC,cAAc,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO;YACV,OAAO,IAAI,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAChD,KAAK,IAAI;YACP,OAAO,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/D,KAAK,MAAM;YACT,OAAO,IAAI,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAChD;YACE,MAAM,IAAI,KAAK,CAAC,YAAY,cAAc,CAAC,QAAQ,oBAAoB,CAAC,CAAC;KAC1E;AACH,CAAC,CAAC;AAEF
|
|
1
|
+
{"version":3,"file":"providerClientUtils.js","sourceRoot":"","sources":["src/providerClientUtils.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACzB,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAC7B,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC/D,MAAM,iBAAiB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AACzD,MAAM,kBAAkB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAC3D,MAAM,gBAAgB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AACvD,MAAM,kBAAkB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,CAAC,cAAc,GAAG,EAAE,EAAE,EAAE;IAClD,QAAQ,cAAc,CAAC,QAAQ,EAAE;QACjC,KAAK,KAAK;YACR,OAAO,IAAI,iBAAiB,CAAC,cAAc,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO;YACV,OAAO,IAAI,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAChD,KAAK,IAAI;YACP,OAAO,IAAI,gBAAgB,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/D,KAAK,MAAM;YACT,OAAO,IAAI,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAChD;YACE,MAAM,IAAI,KAAK,CAAC,YAAY,cAAc,CAAC,QAAQ,oBAAoB,CAAC,CAAC;KAC1E;AACH,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,aAAa,GAAG,KAAK,EAAE,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,EAAE,EAAE;IAC9E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;IACzD,IAAI;QACF,MAAM,cAAc,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC,CAAC;QAC1E,OAAO,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;KAC7C;YAAS;QACR,8CAA8C;QAC9C,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;KAC5C;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,OAAO,GAAG;IACf,mBAAmB;IACnB,aAAa;CACd,CAAC"}
|
package/src/granule.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import * as
|
|
1
|
+
import * as crypto from 'crypto';
|
|
2
2
|
import moment from 'moment';
|
|
3
|
-
|
|
3
|
+
|
|
4
4
|
import * as S3 from '@cumulus/aws-client/S3';
|
|
5
|
+
import { s3 } from '@cumulus/aws-client/services';
|
|
5
6
|
import * as log from '@cumulus/common/log';
|
|
6
|
-
import {
|
|
7
|
-
import
|
|
7
|
+
import { FilePgModel, Knex, translatePostgresFileToApiFile } from '@cumulus/db';
|
|
8
|
+
import * as errors from '@cumulus/errors';
|
|
8
9
|
import { RecordDoesNotExist } from '@cumulus/errors';
|
|
10
|
+
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
9
11
|
|
|
10
12
|
export interface EventWithDuplicateHandling {
|
|
11
13
|
config: {
|
|
@@ -217,7 +219,7 @@ export async function handleDuplicateFile(params: {
|
|
|
217
219
|
}) => Promise<void>,
|
|
218
220
|
ACL?: string,
|
|
219
221
|
sourceBucket?: string,
|
|
220
|
-
fileRemotePath
|
|
222
|
+
fileRemotePath?: string,
|
|
221
223
|
s3Object?: { moveObject: Function },
|
|
222
224
|
moveGranuleFileWithVersioningFunction?: Function,
|
|
223
225
|
}): Promise<VersionedObject[]> {
|
|
@@ -241,6 +243,9 @@ export async function handleDuplicateFile(params: {
|
|
|
241
243
|
} else if (duplicateHandling === 'version') {
|
|
242
244
|
// sync to staging location if required
|
|
243
245
|
if (syncFileFunction) {
|
|
246
|
+
if (!fileRemotePath) {
|
|
247
|
+
throw new Error('fileRemotePath must be defined if syncFileFunction is provided');
|
|
248
|
+
}
|
|
244
249
|
await syncFileFunction({
|
|
245
250
|
bucket: sourceBucket,
|
|
246
251
|
destinationBucket: source.Bucket,
|
|
@@ -263,6 +268,9 @@ export async function handleDuplicateFile(params: {
|
|
|
263
268
|
);
|
|
264
269
|
} else if (duplicateHandling === 'replace') {
|
|
265
270
|
if (syncFileFunction) {
|
|
271
|
+
if (!fileRemotePath) {
|
|
272
|
+
throw new Error('fileRemotePath must be defined if syncFileFunction is provided');
|
|
273
|
+
}
|
|
266
274
|
// sync directly to target location
|
|
267
275
|
await syncFileFunction({
|
|
268
276
|
destinationBucket: target.Bucket,
|
|
@@ -458,6 +466,28 @@ export function unversionFilename(filename: string): string {
|
|
|
458
466
|
: filename;
|
|
459
467
|
}
|
|
460
468
|
|
|
469
|
+
/**
|
|
470
|
+
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
471
|
+
* a producer provided granule object
|
|
472
|
+
*
|
|
473
|
+
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
474
|
+
* assigned by the granule producer
|
|
475
|
+
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
476
|
+
* @param hashLength - The length of the hash to append to the granuleId.
|
|
477
|
+
* @param includeTimestampHashKey - Boolean value for whether hash string should contain timestamp
|
|
478
|
+
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
479
|
+
*/
|
|
480
|
+
export function generateUniqueGranuleId(
|
|
481
|
+
id: string, collectionId: string, hashLength: number, includeTimestampHashKey?: boolean
|
|
482
|
+
): string {
|
|
483
|
+
// use MD5 to generate truncated hash of granule object
|
|
484
|
+
const hashStringWithTimestamp = `${collectionId}_${process.hrtime.bigint().toString()}`;
|
|
485
|
+
const hashStringWithoutTimestamp = `${collectionId}`;
|
|
486
|
+
const hashString = includeTimestampHashKey ? hashStringWithTimestamp : hashStringWithoutTimestamp;
|
|
487
|
+
const hashBuffer = crypto.createHash('md5').update(hashString).digest();
|
|
488
|
+
return `${id}_${hashBuffer.toString('base64url').replace(/_/g, '').slice(0, hashLength)}`;
|
|
489
|
+
}
|
|
490
|
+
|
|
461
491
|
/**
|
|
462
492
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
463
493
|
*
|
|
@@ -465,7 +495,7 @@ export function unversionFilename(filename: string): string {
|
|
|
465
495
|
* @param {Object} event.config - the config object
|
|
466
496
|
* @param {Object} event.config.collection - collection object.
|
|
467
497
|
|
|
468
|
-
* @returns {
|
|
498
|
+
* @returns {DuplicateHandling} - duplicate handling directive.
|
|
469
499
|
*/
|
|
470
500
|
export function duplicateHandlingType(
|
|
471
501
|
event: EventWithDuplicateHandling
|
|
@@ -37,7 +37,8 @@ const buildProviderClient = (providerConfig = {}) => {
|
|
|
37
37
|
* @param {Object} param
|
|
38
38
|
* @param {Object} param.providerClient - a provider client
|
|
39
39
|
* @param {string} param.remotePath - the path of the file to fetch
|
|
40
|
-
* @param {string} param.remoteAltBucket - alternate per-file bucket override to
|
|
40
|
+
* @param {string} [param.remoteAltBucket] - alternate per-file bucket override to
|
|
41
|
+
* the providerClient
|
|
41
42
|
* bucket
|
|
42
43
|
* @returns {Promise<string>} the contents of the remote file
|
|
43
44
|
*/
|
package/src/url-path-template.js
CHANGED
|
@@ -3,35 +3,51 @@ const moment = require('moment');
|
|
|
3
3
|
const path = require('path');
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
|
-
* evaluate the operation specified in template
|
|
7
|
-
*
|
|
8
|
-
* @param {string} name - the name of the operation
|
|
9
|
-
* @param {
|
|
10
|
-
* @
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
6
|
+
* evaluate the operation specified in template
|
|
7
|
+
*
|
|
8
|
+
* @param {string} name - the name of the operation
|
|
9
|
+
* @param {object} args - the args (in array) of the operation
|
|
10
|
+
* @param {object} context - the metadata used in the template
|
|
11
|
+
* @returns {string} - the return value of the operation
|
|
12
|
+
*/
|
|
13
|
+
function evaluateOperation(name, args, context) {
|
|
14
|
+
// args[0] is either an object path or a constant
|
|
15
|
+
// e.g. extractPath(file.path) or extractPath('/a/b/c')
|
|
16
|
+
// assume args[0] is an object path if it starts with a key of context
|
|
17
|
+
const isObjectPath = Object.keys(context).includes(args[0].split('.')[0]);
|
|
18
|
+
const jsonPathValue = get(context, args[0], isObjectPath ? undefined : args[0]);
|
|
19
|
+
|
|
20
|
+
if (name !== 'defaultTo' && !jsonPathValue) throw new Error(`Could not resolve path ${args[0]}`);
|
|
21
|
+
|
|
14
22
|
switch (name) {
|
|
15
23
|
case 'extractYear': {
|
|
16
|
-
return new Date(
|
|
24
|
+
return new Date(jsonPathValue).getUTCFullYear();
|
|
17
25
|
}
|
|
18
26
|
case 'extractMonth': {
|
|
19
|
-
return (new Date(
|
|
27
|
+
return (new Date(jsonPathValue).getUTCMonth() + 1).toString();
|
|
20
28
|
}
|
|
21
29
|
case 'extractDate': {
|
|
22
|
-
return new Date(
|
|
30
|
+
return new Date(jsonPathValue).getUTCDate().toString();
|
|
23
31
|
}
|
|
24
32
|
case 'extractHour': {
|
|
25
|
-
return new Date(
|
|
33
|
+
return new Date(jsonPathValue).getUTCHours().toString();
|
|
26
34
|
}
|
|
27
35
|
case 'dateFormat': {
|
|
28
|
-
return moment.utc(
|
|
36
|
+
return moment.utc(jsonPathValue).format(args[1]);
|
|
29
37
|
}
|
|
30
38
|
case 'substring': {
|
|
31
|
-
return String.prototype.substring.apply(String(
|
|
39
|
+
return String.prototype.substring.apply(String(jsonPathValue), args.slice(1));
|
|
32
40
|
}
|
|
33
41
|
case 'extractPath': {
|
|
34
|
-
return path.dirname(
|
|
42
|
+
return path.dirname(jsonPathValue);
|
|
43
|
+
}
|
|
44
|
+
case 'defaultTo': {
|
|
45
|
+
const isObjectPathSecondArg = Object.keys(context).includes(args[1].split('.')[0]);
|
|
46
|
+
const jsonPathValueSecondArg = get(
|
|
47
|
+
context, args[1],
|
|
48
|
+
isObjectPathSecondArg ? undefined : args[1]
|
|
49
|
+
);
|
|
50
|
+
return jsonPathValue || jsonPathValueSecondArg;
|
|
35
51
|
}
|
|
36
52
|
default:
|
|
37
53
|
throw new Error(`Could not support operation ${name}`);
|
|
@@ -41,7 +57,7 @@ function evaluateOperation(name, args) {
|
|
|
41
57
|
/**
|
|
42
58
|
* retrieve the actual value of the matched string and return it
|
|
43
59
|
*
|
|
44
|
-
* @param {
|
|
60
|
+
* @param {object} context - the metadata used in the template
|
|
45
61
|
* @param {string} submatch - the parenthesized submatch string
|
|
46
62
|
* @returns {string} - the value of the matched string
|
|
47
63
|
*/
|
|
@@ -54,14 +70,8 @@ function templateReplacer(context, submatch) {
|
|
|
54
70
|
if (submatch.match(expressionRegex)) {
|
|
55
71
|
const name = matches[1];
|
|
56
72
|
const args = matches[2].split(/\s*,\s*/);
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
// assume args[0] is an object path if it starts with a key of context
|
|
60
|
-
const isObjectPath = Object.keys(context).includes(args[0].split('.')[0]);
|
|
61
|
-
const jsonPathValue = get(context, args[0], isObjectPath ? undefined : args[0]);
|
|
62
|
-
if (!jsonPathValue) throw new Error(`Could not resolve path ${args[0]}`);
|
|
63
|
-
args[0] = jsonPathValue;
|
|
64
|
-
return evaluateOperation(name, args);
|
|
73
|
+
|
|
74
|
+
return evaluateOperation(name, args, context);
|
|
65
75
|
}
|
|
66
76
|
|
|
67
77
|
const jsonPathValue = get(context, submatch);
|
|
@@ -79,12 +89,10 @@ function templateReplacer(context, submatch) {
|
|
|
79
89
|
**/
|
|
80
90
|
function urlPathTemplate(pathTemplate, context) {
|
|
81
91
|
const templateRegex = /{([^{}]+)}/g;
|
|
82
|
-
|
|
83
92
|
try {
|
|
84
93
|
// match: The matched substring, submatch: The parenthesized submatch string
|
|
85
94
|
const replacedPath = pathTemplate.replace(templateRegex, (match, submatch) =>
|
|
86
95
|
templateReplacer(context, submatch));
|
|
87
|
-
|
|
88
96
|
if (replacedPath.match(templateRegex)) {
|
|
89
97
|
return urlPathTemplate(replacedPath, context);
|
|
90
98
|
}
|