@cumulus/ingest 20.1.2 → 20.1.3-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/granule.d.ts +12 -1
- package/granule.d.ts.map +1 -1
- package/granule.js +22 -3
- package/granule.js.map +1 -1
- package/package.json +13 -13
- package/src/granule.ts +26 -4
- package/tsconfig.tsbuildinfo +1 -1
package/granule.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
2
1
|
import { FilePgModel, Knex } from '@cumulus/db';
|
|
2
|
+
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
3
3
|
export interface EventWithDuplicateHandling {
|
|
4
4
|
config: {
|
|
5
5
|
collection: {
|
|
@@ -180,6 +180,17 @@ export declare function moveGranuleFile(moveFileParam: MoveFileParams, filesPgMo
|
|
|
180
180
|
* @returns {string} - filename with timestamp removed
|
|
181
181
|
*/
|
|
182
182
|
export declare function unversionFilename(filename: string): string;
|
|
183
|
+
/**
|
|
184
|
+
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
185
|
+
* a producer provided granule object
|
|
186
|
+
*
|
|
187
|
+
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
188
|
+
* assigned by the granule producer
|
|
189
|
+
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
190
|
+
* @param hashLength - The length of the hash to append to the granuleId.
|
|
191
|
+
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
192
|
+
*/
|
|
193
|
+
export declare function generateUniqueGranuleId(id: string, collectionId: string, hashLength: number): string;
|
|
183
194
|
/**
|
|
184
195
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
185
196
|
*
|
package/granule.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,IAAI,EAAkC,MAAM,aAAa,CAAC;AAGhF,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAE5D,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE;QACN,UAAU,EAAE;YACV,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;SACtC,CAAC;QACF,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;KACvC,CAAC;IACF,cAAc,CAAC,EAAE;QACf,eAAe,CAAC,EAAE;YAChB,uBAAuB,CAAC,EAAE,OAAO,CAAA;SAClC,CAAA;KACF,CAAC;CACH;AAED,MAAM,WAAW,IAAI;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,IAAI,EAAE,IAAI,CAAA;CACX;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;;IAMI;AACJ,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED;;;;;;IAMI;AACJ,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,eAAe,EAAE,CAAC,CAc5B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,6BAA6B,CACjD,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,oBAAoB,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,EACvE,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,eAAe,EAAE,CAAC,CAoC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAAE;IAChD,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,EAAE,MAAM,CAAC;KACxB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE;QAAE,UAAU,EAAE,QAAQ,CAAA;KAAE,CAAC;IACpC,qCAAqC,CAAC,EAAE,QAAQ,CAAC;CAClD,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAiE7B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,SAAU,IAAI,KAAG,MAAM,GAAG,SAQnD,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,IAAI,EAAE,EACnB,YAAY,EAAE;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CACvB,EAAE,GACF,cAAc,EAAE,CAsClB;AAED;;;;;;;;EAQE;AACF,wBAAsB,eAAe,CACnC,aAAa,EAAE,cAAc,EAC7B,YAAY,EAAE,WAAW,EACzB,GAAG,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,EAC5B,wBAAwB,EAAE,MAAM,GAAG,SAAS,GAC3C,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAuDrC;AAaD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAI1D;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GACnD,MAAM,CAMR;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,0BAA0B,GAChC,iBAAiB,CAQnB"}
|
package/granule.js
CHANGED
|
@@ -26,13 +26,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.duplicateHandlingType = exports.unversionFilename = exports.moveGranuleFile = exports.generateMoveFileParams = exports.getNameOfFile = exports.handleDuplicateFile = exports.moveGranuleFileWithVersioning = exports.listVersionedObjects = exports.renameS3FileWithTimestamp = void 0;
|
|
30
|
-
const
|
|
29
|
+
exports.duplicateHandlingType = exports.generateUniqueGranuleId = exports.unversionFilename = exports.moveGranuleFile = exports.generateMoveFileParams = exports.getNameOfFile = exports.handleDuplicateFile = exports.moveGranuleFileWithVersioning = exports.listVersionedObjects = exports.renameS3FileWithTimestamp = void 0;
|
|
30
|
+
const crypto = __importStar(require("crypto"));
|
|
31
31
|
const moment_1 = __importDefault(require("moment"));
|
|
32
|
-
const services_1 = require("@cumulus/aws-client/services");
|
|
33
32
|
const S3 = __importStar(require("@cumulus/aws-client/S3"));
|
|
33
|
+
const services_1 = require("@cumulus/aws-client/services");
|
|
34
34
|
const log = __importStar(require("@cumulus/common/log"));
|
|
35
35
|
const db_1 = require("@cumulus/db");
|
|
36
|
+
const errors = __importStar(require("@cumulus/errors"));
|
|
36
37
|
const errors_1 = require("@cumulus/errors");
|
|
37
38
|
/**
|
|
38
39
|
* rename s3 file with timestamp
|
|
@@ -361,6 +362,24 @@ function unversionFilename(filename) {
|
|
|
361
362
|
: filename;
|
|
362
363
|
}
|
|
363
364
|
exports.unversionFilename = unversionFilename;
|
|
365
|
+
/**
|
|
366
|
+
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
367
|
+
* a producer provided granule object
|
|
368
|
+
*
|
|
369
|
+
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
370
|
+
* assigned by the granule producer
|
|
371
|
+
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
372
|
+
* @param hashLength - The length of the hash to append to the granuleId.
|
|
373
|
+
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
374
|
+
*/
|
|
375
|
+
function generateUniqueGranuleId(id, collectionId, hashLength) {
|
|
376
|
+
// use MD5 to generate truncated hash of granule object
|
|
377
|
+
// in the format
|
|
378
|
+
const hashString = `${collectionId}_${process.hrtime.bigint().toString()}`;
|
|
379
|
+
const hashBuffer = crypto.createHash('md5').update(hashString).digest();
|
|
380
|
+
return `${id}_${hashBuffer.toString('base64url').replace(/_/g, '').slice(0, hashLength)}`;
|
|
381
|
+
}
|
|
382
|
+
exports.generateUniqueGranuleId = generateUniqueGranuleId;
|
|
364
383
|
/**
|
|
365
384
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
366
385
|
*
|
package/granule.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA
|
|
1
|
+
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AACjC,oDAA4B;AAE5B,2DAA6C;AAC7C,2DAAkD;AAClD,yDAA2C;AAC3C,oCAAgF;AAChF,wDAA0C;AAC1C,4CAAqD;AAkDrD;;;;;;IAMI;AACG,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,GAAW;IAEX,MAAM,YAAY,GAAG,oBAAoB,CAAC;IAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;IAElE,IAAI,CAAC,SAAS,EAAE;QACd,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,IAAI,GAAG,wCAAwC,CAAC,CAAC;KAChF;IAED,IAAI,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;IAEzE,qDAAqD;IACrD,4CAA4C;IAC5C,OAAO,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE;QACnE,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;KAC7F;IAED,GAAG,CAAC,KAAK,CAAC,sCAAsC,MAAM,IAAI,GAAG,OAAO,UAAU,EAAE,CAAC,CAAC;IAElF,MAAM,EAAE,CAAC,UAAU,CAAC;QAClB,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,GAAG;QACd,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,UAAU;QAC1B,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;AACL,CAAC;AA5BD,8DA4BC;AAED;;;;;;IAMI;AACG,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,GAAW;IAEX,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC;QACtC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,GAAG,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE;QACV,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,EAAE,MAAM;YACd,GAAG;YACH,IAAI,EAAE,IAAI;SACX,CAAC,CAAC,CAAC;KACL;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAjBD,oDAiBC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACI,KAAK,UAAU,6BAA6B,CACjD,MAAuC,EACvC,MAAuC,EACvC,uBAAqE,EAAE,EACvE,GAAY;IAEZ,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC;IACxD,sFAAsF;IACtF,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,mBAAmB,CAAC;QACjD,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,YAAY,IAAI,OAAO;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,QAAQ,IAAI,MAAM,EAAE,CAAC,mBAAmB,CAAC;QAC7D,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,2FAA2F;IAC3F,yEAAyE;IACzE,IAAI,aAAa,KAAK,aAAa,EAAE;QACnC,MAAM,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACpD;SAAM;QACL,GAAG,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QACvC,MAAM,yBAAyB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAE3D,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;YACd,GAAG;SACJ,CAAC,CAAC;KACJ;IACD,uBAAuB;IACvB,OAAO,oBAAoB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAzCD,sEAyCC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACI,KAAK,UAAU,mBAAmB,CAAC,MAgBzC;IACC,MAAM,EACJ,GAAG,EACH,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,EACd,qCAAqC,GAAG,6BAA6B,EACrE,QAAQ,GAAG,EAAE,EACb,MAAM,EACN,YAAY,EACZ,gBAAgB,EAChB,MAAM,GACP,GAAG,MAAM,CAAC;IAEX,IAAI,iBAAiB,KAAK,OAAO,EAAE;QACjC,wEAAwE;QACxE,sDAAsD;QACtD,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC,GAAG,sBAAsB,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;KAC3F;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,uCAAuC;QACvC,IAAI,gBAAgB,EAAE;YACpB,MAAM,gBAAgB,CAAC;gBACrB,MAAM,EAAE,YAAY;gBACpB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,cAAc;aACf,CAAC,CAAC;SACJ;QACD,IAAI,oBAAoB,GAAG,EAAE,CAAC;QAC9B,IAAI,gBAAgB,EAAE;YACpB,mBAAmB;YACnB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACnF,oBAAoB,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;SACnD;QACD,+BAA+B;QAC/B,OAAO,qCAAqC,CAC1C,MAAM,EACN,MAAM,EACN,oBAAoB,EACpB,GAAG,CACJ,CAAC;KACH;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,IAAI,gBAAgB,EAAE;YACpB,mCAAmC;YACnC,MAAM,gBAAgB,CAAC;gBACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,MAAM,EAAE,YAAY;gBACpB,cAAc;aACf,CAAC,CAAC;SACJ;aAAM;YACL,MAAM,QAAQ,CAAC,UAAU,CAAC;gBACxB,GAAG;gBACH,QAAQ,EAAE,IAAI;gBACd,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,YAAY,EAAE,MAAM,CAAC,MAAM;gBAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;aACtB,CAAC,CAAC;SACJ;QACD,mCAAmC;QACnC,IAAI,gBAAgB;YAAE,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACzE;IACD,uCAAuC;IACvC,OAAO,EAAE,CAAC;AACZ,CAAC;AAjFD,kDAiFC;AAED;;;;;;;GAOG;AACI,MAAM,aAAa,GAAG,CAAC,IAAU,EAAsB,EAAE;IAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC;IAE5C,IAAI,CAAC,QAAQ,EAAE;QACb,OAAO,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;KACnC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AARW,QAAA,aAAa,iBAQxB;AAEF;;;;;;;;;;;GAWG;AACH,SAAgB,sBAAsB,CACpC,WAAmB,EACnB,YAIG;IAEH,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAErC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAE5C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5E,wCAAwC;QACxC,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAElC,IAAI,MAAM,CAAC;QACX,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,GAAG;gBACP,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC;SACH;aAAM,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACvC;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,wEAAwE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACjH;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,QAAQ;YACpC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,IAAI,IAAA,qBAAa,EAAC,IAAI,CAAC,EAAE;YAClD,CAAC,CAAC,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAExB,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,OAAO,EAAE,IAAI,EAAE,CAAC;SACjB;QAED,MAAM,MAAM,GAAG;YACb,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,GAAG,EAAE,SAAS;SACf,CAAC;QAEF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AA7CD,wDA6CC;AAED;;;;;;;;EAQE;AACK,KAAK,UAAU,eAAe,CACnC,aAA6B,EAC7B,YAAyB,EACzB,GAA4B,EAC5B,wBAA4C;IAE5C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;IAE/C,IAAI,MAAM,IAAI,MAAM,EAAE;QACpB,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,wBAAwB,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;SAC7G;QACD,MAAM,gBAAgB,GAAG,MAAM,YAAY,CAAC,MAAM,CAChD,GAAG,EACH;YACE,kBAAkB,EAAE,wBAAwB;YAC5C,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,EACD;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,IAAA,qBAAa,EAAC,IAAI,CAAC;SAC/B,EAAE,CAAC,GAAG,CAAC,CACT,CAAC;QACF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE;YACjC,MAAM,IAAI,2BAAkB,CAAC,iEAAiE,CAAC,CAAC;SACjG;QACD,MAAM,eAAe,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAE5C,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,IAAA,mCAA8B,EAAC,eAAe,CAAC;YAClD,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,IAAA,qBAAa,EAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;SAC7C,CAAC;KACH;IAED,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE;QAC/C,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;KACvB;IAED,MAAM,kBAAkB,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;QACrD,kBAAkB,EAAE,wBAAwB;QAC5C,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,GAAG,EAAE,IAAI,CAAC,GAAG;KACd,CAAC,CAAC;IAEH,OAAO,IAAA,mCAA8B,EAAC,kBAAkB,CAAC,CAAC;AAC5D,CAAC;AA5DD,0CA4DC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,WAAW,GAAG,uGAAuG,CAAC;IAC5H,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,QAAgB;IAChD,OAAO,aAAa,CAAC,QAAQ,CAAC;QAC5B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC5C,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC;AAJD,8CAIC;AAED;;;;;;;;;GASG;AACH,SAAgB,uBAAuB,CACrC,EAAU,EAAE,YAAoB,EAAE,UAAkB;IAEpD,uDAAuD;IACvD,gBAAgB;IAChB,MAAM,UAAU,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC;IACxE,OAAO,GAAG,EAAE,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;AAC5F,CAAC;AARD,0DAQC;AAED;;;;;;;;GAQG;AACH,SAAgB,qBAAqB,CACnC,KAAiC;IAEjC,IAAI,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,uBAAuB,EAAE;QACnE,OAAO,SAAS,CAAC;KAClB;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,iBAAiB;WAChC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB;WACzC,OAAO,CAAC;AACf,CAAC;AAVD,sDAUC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "20.1.
|
|
3
|
+
"version": "20.1.3-alpha.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=20.12.2"
|
|
@@ -41,13 +41,13 @@
|
|
|
41
41
|
"author": "Cumulus Authors",
|
|
42
42
|
"license": "Apache-2.0",
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@cumulus/aws-client": "20.1.
|
|
45
|
-
"@cumulus/common": "20.1.
|
|
46
|
-
"@cumulus/db": "20.1.
|
|
47
|
-
"@cumulus/errors": "20.1.
|
|
48
|
-
"@cumulus/logger": "20.1.
|
|
49
|
-
"@cumulus/message": "20.1.
|
|
50
|
-
"@cumulus/sftp-client": "20.1.
|
|
44
|
+
"@cumulus/aws-client": "20.1.3-alpha.0",
|
|
45
|
+
"@cumulus/common": "20.1.3-alpha.0",
|
|
46
|
+
"@cumulus/db": "20.1.3-alpha.0",
|
|
47
|
+
"@cumulus/errors": "20.1.3-alpha.0",
|
|
48
|
+
"@cumulus/logger": "20.1.3-alpha.0",
|
|
49
|
+
"@cumulus/message": "20.1.3-alpha.0",
|
|
50
|
+
"@cumulus/sftp-client": "20.1.3-alpha.0",
|
|
51
51
|
"cksum": "^1.3.0",
|
|
52
52
|
"encodeurl": "^1.0.2",
|
|
53
53
|
"fs-extra": "^5.0.0",
|
|
@@ -61,10 +61,10 @@
|
|
|
61
61
|
"tough-cookie": "~4.0.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cumulus/checksum": "20.1.
|
|
65
|
-
"@cumulus/cmrjs": "20.1.
|
|
66
|
-
"@cumulus/test-data": "20.1.
|
|
67
|
-
"@cumulus/types": "20.1.
|
|
64
|
+
"@cumulus/checksum": "20.1.3-alpha.0",
|
|
65
|
+
"@cumulus/cmrjs": "20.1.3-alpha.0",
|
|
66
|
+
"@cumulus/test-data": "20.1.3-alpha.0",
|
|
67
|
+
"@cumulus/types": "20.1.3-alpha.0"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "7c70abafba784906805f7fac5c4e8b5e53f51c3b"
|
|
70
70
|
}
|
package/src/granule.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import * as
|
|
1
|
+
import * as crypto from 'crypto';
|
|
2
2
|
import moment from 'moment';
|
|
3
|
-
|
|
3
|
+
|
|
4
4
|
import * as S3 from '@cumulus/aws-client/S3';
|
|
5
|
+
import { s3 } from '@cumulus/aws-client/services';
|
|
5
6
|
import * as log from '@cumulus/common/log';
|
|
6
|
-
import {
|
|
7
|
-
import
|
|
7
|
+
import { FilePgModel, Knex, translatePostgresFileToApiFile } from '@cumulus/db';
|
|
8
|
+
import * as errors from '@cumulus/errors';
|
|
8
9
|
import { RecordDoesNotExist } from '@cumulus/errors';
|
|
10
|
+
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
9
11
|
|
|
10
12
|
export interface EventWithDuplicateHandling {
|
|
11
13
|
config: {
|
|
@@ -458,6 +460,26 @@ export function unversionFilename(filename: string): string {
|
|
|
458
460
|
: filename;
|
|
459
461
|
}
|
|
460
462
|
|
|
463
|
+
/**
|
|
464
|
+
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
465
|
+
* a producer provided granule object
|
|
466
|
+
*
|
|
467
|
+
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
468
|
+
* assigned by the granule producer
|
|
469
|
+
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
470
|
+
* @param hashLength - The length of the hash to append to the granuleId.
|
|
471
|
+
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
472
|
+
*/
|
|
473
|
+
export function generateUniqueGranuleId(
|
|
474
|
+
id: string, collectionId: string, hashLength: number
|
|
475
|
+
): string {
|
|
476
|
+
// use MD5 to generate truncated hash of granule object
|
|
477
|
+
// in the format
|
|
478
|
+
const hashString = `${collectionId}_${process.hrtime.bigint().toString()}`;
|
|
479
|
+
const hashBuffer = crypto.createHash('md5').update(hashString).digest();
|
|
480
|
+
return `${id}_${hashBuffer.toString('base64url').replace(/_/g, '').slice(0, hashLength)}`;
|
|
481
|
+
}
|
|
482
|
+
|
|
461
483
|
/**
|
|
462
484
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
463
485
|
*
|