@cumulus/ingest 20.1.3-alpha.2 → 20.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/granule.d.ts +1 -12
- package/granule.d.ts.map +1 -1
- package/granule.js +3 -22
- package/granule.js.map +1 -1
- package/package.json +13 -13
- package/src/granule.ts +4 -26
- package/tsconfig.tsbuildinfo +1 -1
package/granule.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { FilePgModel, Knex } from '@cumulus/db';
|
|
2
1
|
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
2
|
+
import { FilePgModel, Knex } from '@cumulus/db';
|
|
3
3
|
export interface EventWithDuplicateHandling {
|
|
4
4
|
config: {
|
|
5
5
|
collection: {
|
|
@@ -180,17 +180,6 @@ export declare function moveGranuleFile(moveFileParam: MoveFileParams, filesPgMo
|
|
|
180
180
|
* @returns {string} - filename with timestamp removed
|
|
181
181
|
*/
|
|
182
182
|
export declare function unversionFilename(filename: string): string;
|
|
183
|
-
/**
|
|
184
|
-
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
185
|
-
* a producer provided granule object
|
|
186
|
-
*
|
|
187
|
-
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
188
|
-
* assigned by the granule producer
|
|
189
|
-
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
190
|
-
* @param hashLength - The length of the hash to append to the granuleId.
|
|
191
|
-
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
192
|
-
*/
|
|
193
|
-
export declare function generateUniqueGranuleId(id: string, collectionId: string, hashLength: number): string;
|
|
194
183
|
/**
|
|
195
184
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
196
185
|
*
|
package/granule.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAkC,IAAI,EAAE,MAAM,aAAa,CAAC;AAGhF,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE;QACN,UAAU,EAAE;YACV,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;SACtC,CAAC;QACF,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;KACvC,CAAC;IACF,cAAc,CAAC,EAAE;QACf,eAAe,CAAC,EAAE;YAChB,uBAAuB,CAAC,EAAE,OAAO,CAAA;SAClC,CAAA;KACF,CAAC;CACH;AAED,MAAM,WAAW,IAAI;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,IAAI,EAAE,IAAI,CAAA;CACX;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;;IAMI;AACJ,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED;;;;;;IAMI;AACJ,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,eAAe,EAAE,CAAC,CAc5B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,6BAA6B,CACjD,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,oBAAoB,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,EACvE,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,eAAe,EAAE,CAAC,CAoC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAAE;IAChD,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,EAAE,MAAM,CAAC;KACxB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE;QAAE,UAAU,EAAE,QAAQ,CAAA;KAAE,CAAC;IACpC,qCAAqC,CAAC,EAAE,QAAQ,CAAC;CAClD,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAiE7B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,SAAU,IAAI,KAAG,MAAM,GAAG,SAQnD,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,IAAI,EAAE,EACnB,YAAY,EAAE;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CACvB,EAAE,GACF,cAAc,EAAE,CAsClB;AAED;;;;;;;;EAQE;AACF,wBAAsB,eAAe,CACnC,aAAa,EAAE,cAAc,EAC7B,YAAY,EAAE,WAAW,EACzB,GAAG,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,EAC5B,wBAAwB,EAAE,MAAM,GAAG,SAAS,GAC3C,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAuDrC;AAaD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAI1D;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,0BAA0B,GAChC,iBAAiB,CAQnB"}
|
package/granule.js
CHANGED
|
@@ -26,14 +26,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.duplicateHandlingType = exports.
|
|
30
|
-
const
|
|
29
|
+
exports.duplicateHandlingType = exports.unversionFilename = exports.moveGranuleFile = exports.generateMoveFileParams = exports.getNameOfFile = exports.handleDuplicateFile = exports.moveGranuleFileWithVersioning = exports.listVersionedObjects = exports.renameS3FileWithTimestamp = void 0;
|
|
30
|
+
const errors = __importStar(require("@cumulus/errors"));
|
|
31
31
|
const moment_1 = __importDefault(require("moment"));
|
|
32
|
-
const S3 = __importStar(require("@cumulus/aws-client/S3"));
|
|
33
32
|
const services_1 = require("@cumulus/aws-client/services");
|
|
33
|
+
const S3 = __importStar(require("@cumulus/aws-client/S3"));
|
|
34
34
|
const log = __importStar(require("@cumulus/common/log"));
|
|
35
35
|
const db_1 = require("@cumulus/db");
|
|
36
|
-
const errors = __importStar(require("@cumulus/errors"));
|
|
37
36
|
const errors_1 = require("@cumulus/errors");
|
|
38
37
|
/**
|
|
39
38
|
* rename s3 file with timestamp
|
|
@@ -362,24 +361,6 @@ function unversionFilename(filename) {
|
|
|
362
361
|
: filename;
|
|
363
362
|
}
|
|
364
363
|
exports.unversionFilename = unversionFilename;
|
|
365
|
-
/**
|
|
366
|
-
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
367
|
-
* a producer provided granule object
|
|
368
|
-
*
|
|
369
|
-
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
370
|
-
* assigned by the granule producer
|
|
371
|
-
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
372
|
-
* @param hashLength - The length of the hash to append to the granuleId.
|
|
373
|
-
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
374
|
-
*/
|
|
375
|
-
function generateUniqueGranuleId(id, collectionId, hashLength) {
|
|
376
|
-
// use MD5 to generate truncated hash of granule object
|
|
377
|
-
// in the format
|
|
378
|
-
const hashString = `${collectionId}_${process.hrtime.bigint().toString()}`;
|
|
379
|
-
const hashBuffer = crypto.createHash('md5').update(hashString).digest();
|
|
380
|
-
return `${id}_${hashBuffer.toString('base64url').replace(/_/g, '').slice(0, hashLength)}`;
|
|
381
|
-
}
|
|
382
|
-
exports.generateUniqueGranuleId = generateUniqueGranuleId;
|
|
383
364
|
/**
|
|
384
365
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
385
366
|
*
|
package/granule.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA
|
|
1
|
+
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,wDAA0C;AAC1C,oDAA4B;AAC5B,2DAAkD;AAClD,2DAA6C;AAC7C,yDAA2C;AAE3C,oCAAgF;AAChF,4CAAqD;AAiDrD;;;;;;IAMI;AACG,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,GAAW;IAEX,MAAM,YAAY,GAAG,oBAAoB,CAAC;IAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;IAElE,IAAI,CAAC,SAAS,EAAE;QACd,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,IAAI,GAAG,wCAAwC,CAAC,CAAC;KAChF;IAED,IAAI,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;IAEzE,qDAAqD;IACrD,4CAA4C;IAC5C,OAAO,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE;QACnE,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;KAC7F;IAED,GAAG,CAAC,KAAK,CAAC,sCAAsC,MAAM,IAAI,GAAG,OAAO,UAAU,EAAE,CAAC,CAAC;IAElF,MAAM,EAAE,CAAC,UAAU,CAAC;QAClB,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,GAAG;QACd,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,UAAU;QAC1B,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;AACL,CAAC;AA5BD,8DA4BC;AAED;;;;;;IAMI;AACG,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,GAAW;IAEX,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC;QACtC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,GAAG,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE;QACV,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,MAAM,EAAE,MAAM;YACd,GAAG;YACH,IAAI,EAAE,IAAI;SACX,CAAC,CAAC,CAAC;KACL;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAjBD,oDAiBC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACI,KAAK,UAAU,6BAA6B,CACjD,MAAuC,EACvC,MAAuC,EACvC,uBAAqE,EAAE,EACvE,GAAY;IAEZ,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC;IACxD,sFAAsF;IACtF,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,mBAAmB,CAAC;QACjD,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,YAAY,IAAI,OAAO;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,QAAQ,IAAI,MAAM,EAAE,CAAC,mBAAmB,CAAC;QAC7D,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,2FAA2F;IAC3F,yEAAyE;IACzE,IAAI,aAAa,KAAK,aAAa,EAAE;QACnC,MAAM,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACpD;SAAM;QACL,GAAG,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QACvC,MAAM,yBAAyB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAE3D,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;YACd,GAAG;SACJ,CAAC,CAAC;KACJ;IACD,uBAAuB;IACvB,OAAO,oBAAoB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAzCD,sEAyCC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACI,KAAK,UAAU,mBAAmB,CAAC,MAgBzC;IACC,MAAM,EACJ,GAAG,EACH,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,EACd,qCAAqC,GAAG,6BAA6B,EACrE,QAAQ,GAAG,EAAE,EACb,MAAM,EACN,YAAY,EACZ,gBAAgB,EAChB,MAAM,GACP,GAAG,MAAM,CAAC;IAEX,IAAI,iBAAiB,KAAK,OAAO,EAAE;QACjC,wEAAwE;QACxE,sDAAsD;QACtD,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC,GAAG,sBAAsB,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;KAC3F;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,uCAAuC;QACvC,IAAI,gBAAgB,EAAE;YACpB,MAAM,gBAAgB,CAAC;gBACrB,MAAM,EAAE,YAAY;gBACpB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,cAAc;aACf,CAAC,CAAC;SACJ;QACD,IAAI,oBAAoB,GAAG,EAAE,CAAC;QAC9B,IAAI,gBAAgB,EAAE;YACpB,mBAAmB;YACnB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACnF,oBAAoB,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;SACnD;QACD,+BAA+B;QAC/B,OAAO,qCAAqC,CAC1C,MAAM,EACN,MAAM,EACN,oBAAoB,EACpB,GAAG,CACJ,CAAC;KACH;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,IAAI,gBAAgB,EAAE;YACpB,mCAAmC;YACnC,MAAM,gBAAgB,CAAC;gBACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,MAAM,EAAE,YAAY;gBACpB,cAAc;aACf,CAAC,CAAC;SACJ;aAAM;YACL,MAAM,QAAQ,CAAC,UAAU,CAAC;gBACxB,GAAG;gBACH,QAAQ,EAAE,IAAI;gBACd,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,YAAY,EAAE,MAAM,CAAC,MAAM;gBAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;aACtB,CAAC,CAAC;SACJ;QACD,mCAAmC;QACnC,IAAI,gBAAgB;YAAE,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACzE;IACD,uCAAuC;IACvC,OAAO,EAAE,CAAC;AACZ,CAAC;AAjFD,kDAiFC;AAED;;;;;;;GAOG;AACI,MAAM,aAAa,GAAG,CAAC,IAAU,EAAsB,EAAE;IAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC;IAE5C,IAAI,CAAC,QAAQ,EAAE;QACb,OAAO,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;KACnC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AARW,QAAA,aAAa,iBAQxB;AAEF;;;;;;;;;;;GAWG;AACH,SAAgB,sBAAsB,CACpC,WAAmB,EACnB,YAIG;IAEH,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAErC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAE5C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5E,wCAAwC;QACxC,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAElC,IAAI,MAAM,CAAC;QACX,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,GAAG;gBACP,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC;SACH;aAAM,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACvC;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,wEAAwE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACjH;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,QAAQ;YACpC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,IAAI,IAAA,qBAAa,EAAC,IAAI,CAAC,EAAE;YAClD,CAAC,CAAC,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAExB,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,OAAO,EAAE,IAAI,EAAE,CAAC;SACjB;QAED,MAAM,MAAM,GAAG;YACb,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,GAAG,EAAE,SAAS;SACf,CAAC;QAEF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AA7CD,wDA6CC;AAED;;;;;;;;EAQE;AACK,KAAK,UAAU,eAAe,CACnC,aAA6B,EAC7B,YAAyB,EACzB,GAA4B,EAC5B,wBAA4C;IAE5C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;IAE/C,IAAI,MAAM,IAAI,MAAM,EAAE;QACpB,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,wBAAwB,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;SAC7G;QACD,MAAM,gBAAgB,GAAG,MAAM,YAAY,CAAC,MAAM,CAChD,GAAG,EACH;YACE,kBAAkB,EAAE,wBAAwB;YAC5C,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,EACD;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,IAAA,qBAAa,EAAC,IAAI,CAAC;SAC/B,EAAE,CAAC,GAAG,CAAC,CACT,CAAC;QACF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE;YACjC,MAAM,IAAI,2BAAkB,CAAC,iEAAiE,CAAC,CAAC;SACjG;QACD,MAAM,eAAe,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAE5C,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,IAAA,mCAA8B,EAAC,eAAe,CAAC;YAClD,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,IAAA,qBAAa,EAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;SAC7C,CAAC;KACH;IAED,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE;QAC/C,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;KACvB;IAED,MAAM,kBAAkB,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;QACrD,kBAAkB,EAAE,wBAAwB;QAC5C,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,GAAG,EAAE,IAAI,CAAC,GAAG;KACd,CAAC,CAAC;IAEH,OAAO,IAAA,mCAA8B,EAAC,kBAAkB,CAAC,CAAC;AAC5D,CAAC;AA5DD,0CA4DC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,WAAW,GAAG,uGAAuG,CAAC;IAC5H,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,QAAgB;IAChD,OAAO,aAAa,CAAC,QAAQ,CAAC;QAC5B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC5C,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC;AAJD,8CAIC;AAED;;;;;;;;GAQG;AACH,SAAgB,qBAAqB,CACnC,KAAiC;IAEjC,IAAI,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,uBAAuB,EAAE;QACnE,OAAO,SAAS,CAAC;KAClB;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,iBAAiB;WAChC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB;WACzC,OAAO,CAAC;AACf,CAAC;AAVD,sDAUC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "20.
|
|
3
|
+
"version": "20.2.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=20.12.2"
|
|
@@ -41,13 +41,13 @@
|
|
|
41
41
|
"author": "Cumulus Authors",
|
|
42
42
|
"license": "Apache-2.0",
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@cumulus/aws-client": "20.
|
|
45
|
-
"@cumulus/common": "20.
|
|
46
|
-
"@cumulus/db": "20.
|
|
47
|
-
"@cumulus/errors": "20.
|
|
48
|
-
"@cumulus/logger": "20.
|
|
49
|
-
"@cumulus/message": "20.
|
|
50
|
-
"@cumulus/sftp-client": "20.
|
|
44
|
+
"@cumulus/aws-client": "20.2.0",
|
|
45
|
+
"@cumulus/common": "20.2.0",
|
|
46
|
+
"@cumulus/db": "20.2.0",
|
|
47
|
+
"@cumulus/errors": "20.2.0",
|
|
48
|
+
"@cumulus/logger": "20.2.0",
|
|
49
|
+
"@cumulus/message": "20.2.0",
|
|
50
|
+
"@cumulus/sftp-client": "20.2.0",
|
|
51
51
|
"cksum": "^1.3.0",
|
|
52
52
|
"encodeurl": "^1.0.2",
|
|
53
53
|
"fs-extra": "^5.0.0",
|
|
@@ -61,10 +61,10 @@
|
|
|
61
61
|
"tough-cookie": "~4.0.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cumulus/checksum": "20.
|
|
65
|
-
"@cumulus/cmrjs": "20.
|
|
66
|
-
"@cumulus/test-data": "20.
|
|
67
|
-
"@cumulus/types": "20.
|
|
64
|
+
"@cumulus/checksum": "20.2.0",
|
|
65
|
+
"@cumulus/cmrjs": "20.2.0",
|
|
66
|
+
"@cumulus/test-data": "20.2.0",
|
|
67
|
+
"@cumulus/types": "20.2.0"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "8afcafd2b63778a1ea666163aa268c2dfd7d60ec"
|
|
70
70
|
}
|
package/src/granule.ts
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import * as
|
|
1
|
+
import * as errors from '@cumulus/errors';
|
|
2
2
|
import moment from 'moment';
|
|
3
|
-
|
|
4
|
-
import * as S3 from '@cumulus/aws-client/S3';
|
|
5
3
|
import { s3 } from '@cumulus/aws-client/services';
|
|
4
|
+
import * as S3 from '@cumulus/aws-client/S3';
|
|
6
5
|
import * as log from '@cumulus/common/log';
|
|
7
|
-
import { FilePgModel, Knex, translatePostgresFileToApiFile } from '@cumulus/db';
|
|
8
|
-
import * as errors from '@cumulus/errors';
|
|
9
|
-
import { RecordDoesNotExist } from '@cumulus/errors';
|
|
10
6
|
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
7
|
+
import { FilePgModel, translatePostgresFileToApiFile, Knex } from '@cumulus/db';
|
|
8
|
+
import { RecordDoesNotExist } from '@cumulus/errors';
|
|
11
9
|
|
|
12
10
|
export interface EventWithDuplicateHandling {
|
|
13
11
|
config: {
|
|
@@ -460,26 +458,6 @@ export function unversionFilename(filename: string): string {
|
|
|
460
458
|
: filename;
|
|
461
459
|
}
|
|
462
460
|
|
|
463
|
-
/**
|
|
464
|
-
* Generates a unique granule ID by appending a truncated MD5 hash of values from
|
|
465
|
-
* a producer provided granule object
|
|
466
|
-
*
|
|
467
|
-
* @param id - An ID associated with the object to be hashed. Likely the ID
|
|
468
|
-
* assigned by the granule producer
|
|
469
|
-
* @param collectionId - The api collection ID (name___version) associated with the granule
|
|
470
|
-
* @param hashLength - The length of the hash to append to the granuleId.
|
|
471
|
-
* @returns - A unique granule ID in the format: granuleId_hash.
|
|
472
|
-
*/
|
|
473
|
-
export function generateUniqueGranuleId(
|
|
474
|
-
id: string, collectionId: string, hashLength: number
|
|
475
|
-
): string {
|
|
476
|
-
// use MD5 to generate truncated hash of granule object
|
|
477
|
-
// in the format
|
|
478
|
-
const hashString = `${collectionId}_${process.hrtime.bigint().toString()}`;
|
|
479
|
-
const hashBuffer = crypto.createHash('md5').update(hashString).digest();
|
|
480
|
-
return `${id}_${hashBuffer.toString('base64url').replace(/_/g, '').slice(0, hashLength)}`;
|
|
481
|
-
}
|
|
482
|
-
|
|
483
461
|
/**
|
|
484
462
|
* Returns a directive on how to act when duplicate files are encountered.
|
|
485
463
|
*
|