@cumulus/ingest 10.1.1 → 11.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/granule.d.ts +13 -24
- package/granule.d.ts.map +1 -1
- package/granule.js +55 -109
- package/granule.js.map +1 -1
- package/package.json +14 -14
- package/src/granule.ts +64 -123
- package/tsconfig.tsbuildinfo +1 -1
package/granule.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DuplicateHandling } from '@cumulus/types';
|
|
1
|
+
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
2
2
|
import { FilePgModel, Knex } from '@cumulus/db';
|
|
3
3
|
export interface EventWithDuplicateHandling {
|
|
4
4
|
config: {
|
|
@@ -19,6 +19,7 @@ export interface File {
|
|
|
19
19
|
fileName?: string;
|
|
20
20
|
name?: string;
|
|
21
21
|
filename?: string;
|
|
22
|
+
size?: string;
|
|
22
23
|
}
|
|
23
24
|
export interface MovedGranuleFile {
|
|
24
25
|
bucket: string;
|
|
@@ -136,6 +137,14 @@ export declare function handleDuplicateFile(params: {
|
|
|
136
137
|
};
|
|
137
138
|
moveGranuleFileWithVersioningFunction?: Function;
|
|
138
139
|
}): Promise<VersionedObject[]>;
|
|
140
|
+
/**
|
|
141
|
+
* Get the name of the file from the following properties (in order of preference):
|
|
142
|
+
* 1. fileName (e.g. 'granuleFileNamec.md')
|
|
143
|
+
* 2. name (e.g. 'granuleFileName.md')
|
|
144
|
+
* 3. key (e.g. 'stackname/filepath/granuleFileName.md')
|
|
145
|
+
* @param {File} file - file object with the above properties
|
|
146
|
+
* @returns {string | undefined} - The file name as a string or undefined
|
|
147
|
+
*/
|
|
139
148
|
export declare const getNameOfFile: (file: File) => string | undefined;
|
|
140
149
|
/**
|
|
141
150
|
* For each source file, see if there is a destination and generate the source
|
|
@@ -155,35 +164,15 @@ export declare function generateMoveFileParams(sourceFiles: File[], destinations
|
|
|
155
164
|
regex: string | RegExp;
|
|
156
165
|
}[]): MoveFileParams[];
|
|
157
166
|
/**
|
|
158
|
-
|
|
159
|
-
*
|
|
160
|
-
* @param {Array<Object>} sourceFiles - array of file objects, they are updated with destination
|
|
161
|
-
* location after the files are moved
|
|
162
|
-
* @param {string} sourceFiles.name - file name
|
|
163
|
-
* @param {string} sourceFiles.bucket - current bucket of file
|
|
164
|
-
* @param {string} sourceFiles.key - current S3 key of file
|
|
165
|
-
* @param {Array<Object>} destinations - array of objects defining the destination of granule files
|
|
166
|
-
* @param {string} destinations.regex - regex for matching filepath of file to new destination
|
|
167
|
-
* @param {string} destinations.bucket - aws bucket of the destination
|
|
168
|
-
* @param {string} destinations.filepath - file path/directory on the bucket for the destination
|
|
169
|
-
* @returns {Promise<Array>} returns array of source files updated with new locations.
|
|
170
|
-
*/
|
|
171
|
-
export declare function moveGranuleFiles(sourceFiles: File[], destinations: {
|
|
172
|
-
regex: string;
|
|
173
|
-
bucket: string;
|
|
174
|
-
filepath: string;
|
|
175
|
-
}[]): Promise<MovedGranuleFile[]>;
|
|
176
|
-
/**
|
|
177
|
-
* Moves a granule file and updates the postgres database accordingly
|
|
167
|
+
* Moves a granule file and updates the datastore accordingly
|
|
178
168
|
* @summary Moves a granule file record according to MoveFileParams and updates database accordingly
|
|
179
169
|
* @param {MoveFileParams} moveFileParam - Parameter object describing the move operation
|
|
180
170
|
* @param {FilePgModel} filesPgModel - FilePgModel instance
|
|
181
171
|
* @param {Knex.Transaction | Knex} trx - Knex transaction or (optionally) Knex object
|
|
182
172
|
* @param {number | undefined } postgresCumulusGranuleId - postgres internal granule id
|
|
183
|
-
* @
|
|
184
|
-
* @returns {ReturnValueDataTypeHere} Brief description of the returning value here.
|
|
173
|
+
* @returns {Promise<Object>} - Returns object of type Omit<ApiFile, 'granuleId>>
|
|
185
174
|
*/
|
|
186
|
-
export declare function moveGranuleFile(moveFileParam: MoveFileParams, filesPgModel: FilePgModel, trx: Knex.Transaction | Knex, postgresCumulusGranuleId: number | undefined
|
|
175
|
+
export declare function moveGranuleFile(moveFileParam: MoveFileParams, filesPgModel: FilePgModel, trx: Knex.Transaction | Knex, postgresCumulusGranuleId: number | undefined): Promise<Omit<ApiFile, 'granuleId'>>;
|
|
187
176
|
/**
|
|
188
177
|
* Returns the input filename stripping off any versioned timestamp.
|
|
189
178
|
*
|
package/granule.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"granule.d.ts","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAkC,IAAI,EAAE,MAAM,aAAa,CAAC;AAGhF,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE;QACN,UAAU,EAAE;YACV,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;SACtC,CAAC;QACF,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;KACvC,CAAC;IACF,cAAc,CAAC,EAAE;QACf,eAAe,CAAC,EAAE;YAChB,uBAAuB,CAAC,EAAE,OAAO,CAAA;SAClC,CAAA;KACF,CAAC;CACH;AAED,MAAM,WAAW,IAAI;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAA;KACZ,CAAC;IACF,IAAI,EAAE,IAAI,CAAA;CACX;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAA;CACb;AAED;;;;;;IAMI;AACJ,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED;;;;;;IAMI;AACJ,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,eAAe,EAAE,CAAC,CAW5B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,6BAA6B,CACjD,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EACvC,oBAAoB,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,EACvE,GAAG,CAAC,EAAE,MAAM,GACX,OAAO,CAAC,eAAe,EAAE,CAAC,CAoC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAAE;IAChD,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9E,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,EAAE,MAAM,CAAC;KACxB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE;QAAE,UAAU,EAAE,QAAQ,CAAA;KAAE,CAAC;IACpC,qCAAqC,CAAC,EAAE,QAAQ,CAAC;CAClD,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAiE7B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,SAAU,IAAI,KAAG,MAAM,GAAG,SAQnD,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,IAAI,EAAE,EACnB,YAAY,EAAE;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;CACvB,EAAE,GACF,cAAc,EAAE,CAsClB;AAED;;;;;;;;EAQE;AACF,wBAAsB,eAAe,CACnC,aAAa,EAAE,cAAc,EAC7B,YAAY,EAAE,WAAW,EACzB,GAAG,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,EAC5B,wBAAwB,EAAE,MAAM,GAAG,SAAS,GAC3C,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAuDrC;AAaD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAI1D;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,0BAA0B,GAChC,iBAAiB,CAQnB"}
|
package/granule.js
CHANGED
|
@@ -26,13 +26,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.duplicateHandlingType = exports.unversionFilename = exports.moveGranuleFile = exports.
|
|
29
|
+
exports.duplicateHandlingType = exports.unversionFilename = exports.moveGranuleFile = exports.generateMoveFileParams = exports.getNameOfFile = exports.handleDuplicateFile = exports.moveGranuleFileWithVersioning = exports.listVersionedObjects = exports.renameS3FileWithTimestamp = void 0;
|
|
30
30
|
const errors = __importStar(require("@cumulus/errors"));
|
|
31
31
|
const moment_1 = __importDefault(require("moment"));
|
|
32
32
|
const services_1 = require("@cumulus/aws-client/services");
|
|
33
33
|
const S3 = __importStar(require("@cumulus/aws-client/S3"));
|
|
34
34
|
const log = __importStar(require("@cumulus/common/log"));
|
|
35
|
-
const
|
|
35
|
+
const db_1 = require("@cumulus/db");
|
|
36
|
+
const errors_1 = require("@cumulus/errors");
|
|
36
37
|
/**
|
|
37
38
|
* rename s3 file with timestamp
|
|
38
39
|
*
|
|
@@ -215,7 +216,22 @@ async function handleDuplicateFile(params) {
|
|
|
215
216
|
return [];
|
|
216
217
|
}
|
|
217
218
|
exports.handleDuplicateFile = handleDuplicateFile;
|
|
218
|
-
|
|
219
|
+
/**
|
|
220
|
+
* Get the name of the file from the following properties (in order of preference):
|
|
221
|
+
* 1. fileName (e.g. 'granuleFileNamec.md')
|
|
222
|
+
* 2. name (e.g. 'granuleFileName.md')
|
|
223
|
+
* 3. key (e.g. 'stackname/filepath/granuleFileName.md')
|
|
224
|
+
* @param {File} file - file object with the above properties
|
|
225
|
+
* @returns {string | undefined} - The file name as a string or undefined
|
|
226
|
+
*/
|
|
227
|
+
const getNameOfFile = (file) => {
|
|
228
|
+
var _a, _b;
|
|
229
|
+
const fileName = (_a = file.fileName) !== null && _a !== void 0 ? _a : file.name;
|
|
230
|
+
if (!fileName) {
|
|
231
|
+
return (_b = file.key) === null || _b === void 0 ? void 0 : _b.split('/').pop();
|
|
232
|
+
}
|
|
233
|
+
return fileName;
|
|
234
|
+
};
|
|
219
235
|
exports.getNameOfFile = getNameOfFile;
|
|
220
236
|
/**
|
|
221
237
|
* For each source file, see if there is a destination and generate the source
|
|
@@ -266,129 +282,59 @@ function generateMoveFileParams(sourceFiles, destinations) {
|
|
|
266
282
|
}
|
|
267
283
|
exports.generateMoveFileParams = generateMoveFileParams;
|
|
268
284
|
/**
|
|
269
|
-
|
|
270
|
-
*
|
|
271
|
-
* @param {Array<Object>} sourceFiles - array of file objects, they are updated with destination
|
|
272
|
-
* location after the files are moved
|
|
273
|
-
* @param {string} sourceFiles.name - file name
|
|
274
|
-
* @param {string} sourceFiles.bucket - current bucket of file
|
|
275
|
-
* @param {string} sourceFiles.key - current S3 key of file
|
|
276
|
-
* @param {Array<Object>} destinations - array of objects defining the destination of granule files
|
|
277
|
-
* @param {string} destinations.regex - regex for matching filepath of file to new destination
|
|
278
|
-
* @param {string} destinations.bucket - aws bucket of the destination
|
|
279
|
-
* @param {string} destinations.filepath - file path/directory on the bucket for the destination
|
|
280
|
-
* @returns {Promise<Array>} returns array of source files updated with new locations.
|
|
281
|
-
*/
|
|
282
|
-
async function moveGranuleFiles(sourceFiles, destinations) {
|
|
283
|
-
(0, util_1.deprecate)('@cumulus/ingest/moveGranuleFiles', '9.0.0');
|
|
284
|
-
const moveFileParams = generateMoveFileParams(sourceFiles, destinations);
|
|
285
|
-
const movedGranuleFiles = [];
|
|
286
|
-
const moveFileRequests = moveFileParams.map(async (moveFileParam) => {
|
|
287
|
-
const { source, target, file } = moveFileParam;
|
|
288
|
-
if (source && target) {
|
|
289
|
-
log.debug('moveGranuleFiles', source, target);
|
|
290
|
-
await S3.moveObject({
|
|
291
|
-
sourceBucket: source.Bucket,
|
|
292
|
-
sourceKey: source.Key,
|
|
293
|
-
destinationBucket: target.Bucket,
|
|
294
|
-
destinationKey: target.Key,
|
|
295
|
-
copyTags: true,
|
|
296
|
-
});
|
|
297
|
-
movedGranuleFiles.push({
|
|
298
|
-
bucket: target.Bucket,
|
|
299
|
-
key: target.Key,
|
|
300
|
-
name: (0, exports.getNameOfFile)(file),
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
else {
|
|
304
|
-
let fileBucket;
|
|
305
|
-
let fileKey;
|
|
306
|
-
if (file.bucket && file.key) {
|
|
307
|
-
fileBucket = file.bucket;
|
|
308
|
-
fileKey = file.key;
|
|
309
|
-
}
|
|
310
|
-
else if (file.filename) {
|
|
311
|
-
const parsed = S3.parseS3Uri(file.filename);
|
|
312
|
-
fileBucket = parsed.Bucket;
|
|
313
|
-
fileKey = parsed.Key;
|
|
314
|
-
}
|
|
315
|
-
else {
|
|
316
|
-
throw new Error(`Unable to determine location of file: ${JSON.stringify(file)}`);
|
|
317
|
-
}
|
|
318
|
-
movedGranuleFiles.push({
|
|
319
|
-
bucket: fileBucket,
|
|
320
|
-
key: fileKey,
|
|
321
|
-
name: (0, exports.getNameOfFile)(file),
|
|
322
|
-
});
|
|
323
|
-
}
|
|
324
|
-
});
|
|
325
|
-
await Promise.all(moveFileRequests);
|
|
326
|
-
return movedGranuleFiles;
|
|
327
|
-
}
|
|
328
|
-
exports.moveGranuleFiles = moveGranuleFiles;
|
|
329
|
-
/**
|
|
330
|
-
* Moves a granule file and updates the postgres database accordingly
|
|
285
|
+
* Moves a granule file and updates the datastore accordingly
|
|
331
286
|
* @summary Moves a granule file record according to MoveFileParams and updates database accordingly
|
|
332
287
|
* @param {MoveFileParams} moveFileParam - Parameter object describing the move operation
|
|
333
288
|
* @param {FilePgModel} filesPgModel - FilePgModel instance
|
|
334
289
|
* @param {Knex.Transaction | Knex} trx - Knex transaction or (optionally) Knex object
|
|
335
290
|
* @param {number | undefined } postgresCumulusGranuleId - postgres internal granule id
|
|
336
|
-
* @
|
|
337
|
-
* @returns {ReturnValueDataTypeHere} Brief description of the returning value here.
|
|
291
|
+
* @returns {Promise<Object>} - Returns object of type Omit<ApiFile, 'granuleId>>
|
|
338
292
|
*/
|
|
339
|
-
async function moveGranuleFile(moveFileParam, filesPgModel, trx, postgresCumulusGranuleId
|
|
293
|
+
async function moveGranuleFile(moveFileParam, filesPgModel, trx, postgresCumulusGranuleId) {
|
|
340
294
|
const { source, target, file } = moveFileParam;
|
|
341
|
-
if (
|
|
295
|
+
if (source && target) {
|
|
342
296
|
log.debug('moveGranuleS3Object', source, target);
|
|
343
|
-
if (
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
file_name: (0, exports.getNameOfFile)(file),
|
|
358
|
-
});
|
|
297
|
+
if (!postgresCumulusGranuleId) {
|
|
298
|
+
throw new Error('postgresCumulusGranuleId must be defined to move granule file if writeToPostgres is true');
|
|
299
|
+
}
|
|
300
|
+
const updatedPgRecords = await filesPgModel.update(trx, {
|
|
301
|
+
granule_cumulus_id: postgresCumulusGranuleId,
|
|
302
|
+
bucket: source.Bucket,
|
|
303
|
+
key: source.Key,
|
|
304
|
+
}, {
|
|
305
|
+
bucket: target.Bucket,
|
|
306
|
+
key: target.Key,
|
|
307
|
+
file_name: (0, exports.getNameOfFile)(file),
|
|
308
|
+
}, ['*']);
|
|
309
|
+
if (updatedPgRecords.length !== 1) {
|
|
310
|
+
throw new errors_1.RecordDoesNotExist('Attempted to update granule on move, but granule does not exist');
|
|
359
311
|
}
|
|
312
|
+
const updatedPgRecord = updatedPgRecords[0];
|
|
360
313
|
await S3.moveObject({
|
|
361
|
-
sourceBucket:
|
|
362
|
-
sourceKey:
|
|
363
|
-
destinationBucket:
|
|
364
|
-
destinationKey:
|
|
314
|
+
sourceBucket: source.Bucket,
|
|
315
|
+
sourceKey: source.Key,
|
|
316
|
+
destinationBucket: target.Bucket,
|
|
317
|
+
destinationKey: target.Key,
|
|
365
318
|
copyTags: true,
|
|
366
319
|
});
|
|
367
320
|
return {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
321
|
+
...(0, db_1.translatePostgresFileToApiFile)(updatedPgRecord),
|
|
322
|
+
bucket: target.Bucket,
|
|
323
|
+
key: target.Key,
|
|
324
|
+
fileName: (0, exports.getNameOfFile)({ key: target.Key }),
|
|
371
325
|
};
|
|
372
326
|
}
|
|
373
|
-
|
|
374
|
-
let fileKey;
|
|
375
|
-
if (file.bucket && file.key) {
|
|
376
|
-
fileBucket = file.bucket;
|
|
377
|
-
fileKey = file.key;
|
|
378
|
-
}
|
|
379
|
-
else if (file.filename) {
|
|
327
|
+
if (!(file.bucket || file.key) && file.filename) {
|
|
380
328
|
const parsed = S3.parseS3Uri(file.filename);
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
}
|
|
384
|
-
else {
|
|
385
|
-
throw new Error(`Unable to determine location of file: ${JSON.stringify(file)}`);
|
|
329
|
+
file.bucket = parsed.Bucket;
|
|
330
|
+
file.key = parsed.Key;
|
|
386
331
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
};
|
|
332
|
+
const postgresFileRecord = await filesPgModel.get(trx, {
|
|
333
|
+
granule_cumulus_id: postgresCumulusGranuleId,
|
|
334
|
+
bucket: file.bucket,
|
|
335
|
+
key: file.key,
|
|
336
|
+
});
|
|
337
|
+
return (0, db_1.translatePostgresFileToApiFile)(postgresFileRecord);
|
|
392
338
|
}
|
|
393
339
|
exports.moveGranuleFile = moveGranuleFile;
|
|
394
340
|
/**
|
package/granule.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,wDAA0C;AAC1C,oDAA4B;AAC5B,2DAAkD;AAClD,2DAA6C;AAC7C,yDAA2C;
|
|
1
|
+
{"version":3,"file":"granule.js","sourceRoot":"","sources":["src/granule.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,wDAA0C;AAC1C,oDAA4B;AAC5B,2DAAkD;AAClD,2DAA6C;AAC7C,yDAA2C;AAE3C,oCAAgF;AAChF,4CAAqD;AAiDrD;;;;;;IAMI;AACG,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,GAAW;IAEX,MAAM,YAAY,GAAG,oBAAoB,CAAC;IAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;IAElE,IAAI,CAAC,SAAS,EAAE;QACd,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,IAAI,GAAG,wCAAwC,CAAC,CAAC;KAChF;IAED,IAAI,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;IAEzE,qDAAqD;IACrD,4CAA4C;IAC5C,OAAO,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE;QACnE,UAAU,GAAG,GAAG,GAAG,KAAK,gBAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;KAC7F;IAED,GAAG,CAAC,KAAK,CAAC,sCAAsC,MAAM,IAAI,GAAG,OAAO,UAAU,EAAE,CAAC,CAAC;IAElF,MAAM,EAAE,CAAC,UAAU,CAAC;QAClB,YAAY,EAAE,MAAM;QACpB,SAAS,EAAE,GAAG;QACd,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,UAAU;QAC1B,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;AACL,CAAC;AA5BD,8DA4BC;AAED;;;;;;IAMI;AACG,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,GAAW;IAEX,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC;QACtC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,GAAG,IAAI;KACnB,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACpC,MAAM,EAAE,MAAM;QACd,GAAG;QACH,IAAI,EAAE,IAAI;KACX,CAAC,CAAC,CAAC;AACN,CAAC;AAdD,oDAcC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACI,KAAK,UAAU,6BAA6B,CACjD,MAAuC,EACvC,MAAuC,EACvC,uBAAqE,EAAE,EACvE,GAAY;IAEZ,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC;IACxD,sFAAsF;IACtF,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,mBAAmB,CAAC;QACjD,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,OAAO;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,MAAM,EAAE,CAAC,mBAAmB,CAAC;QAC7D,EAAE,EAAE,IAAA,aAAE,GAAE;QACR,SAAS,EAAE,OAAO;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;KAChB,CAAC,CAAC;IAEH,2FAA2F;IAC3F,yEAAyE;IACzE,IAAI,aAAa,KAAK,aAAa,EAAE;QACnC,MAAM,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACpD;SAAM;QACL,GAAG,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QACvC,MAAM,yBAAyB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAE3D,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;YACd,GAAG;SACJ,CAAC,CAAC;KACJ;IACD,uBAAuB;IACvB,OAAO,oBAAoB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAzCD,sEAyCC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACI,KAAK,UAAU,mBAAmB,CAAC,MAgBzC;IACC,MAAM,EACJ,GAAG,EACH,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,EACd,qCAAqC,GAAG,6BAA6B,EACrE,QAAQ,GAAG,EAAE,EACb,MAAM,EACN,YAAY,EACZ,gBAAgB,EAChB,MAAM,GACP,GAAG,MAAM,CAAC;IAEX,IAAI,iBAAiB,KAAK,OAAO,EAAE;QACjC,wEAAwE;QACxE,sDAAsD;QACtD,MAAM,IAAI,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC,GAAG,sBAAsB,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;KAC3F;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,uCAAuC;QACvC,IAAI,gBAAgB,EAAE;YACpB,MAAM,gBAAgB,CAAC;gBACrB,MAAM,EAAE,YAAY;gBACpB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,cAAc;aACf,CAAC,CAAC;SACJ;QACD,IAAI,oBAAoB,GAAG,EAAE,CAAC;QAC9B,IAAI,gBAAgB,EAAE;YACpB,mBAAmB;YACnB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YACnF,oBAAoB,GAAG,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;SACnD;QACD,+BAA+B;QAC/B,OAAO,qCAAqC,CAC1C,MAAM,EACN,MAAM,EACN,oBAAoB,EACpB,GAAG,CACJ,CAAC;KACH;SAAM,IAAI,iBAAiB,KAAK,SAAS,EAAE;QAC1C,IAAI,gBAAgB,EAAE;YACpB,mCAAmC;YACnC,MAAM,gBAAgB,CAAC;gBACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,MAAM,EAAE,YAAY;gBACpB,cAAc;aACf,CAAC,CAAC;SACJ;aAAM;YACL,MAAM,QAAQ,CAAC,UAAU,CAAC;gBACxB,GAAG;gBACH,QAAQ,EAAE,IAAI;gBACd,iBAAiB,EAAE,MAAM,CAAC,MAAM;gBAChC,cAAc,EAAE,MAAM,CAAC,GAAG;gBAC1B,YAAY,EAAE,MAAM,CAAC,MAAM;gBAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;aACtB,CAAC,CAAC;SACJ;QACD,mCAAmC;QACnC,IAAI,gBAAgB;YAAE,MAAM,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;KACzE;IACD,uCAAuC;IACvC,OAAO,EAAE,CAAC;AACZ,CAAC;AAjFD,kDAiFC;AAED;;;;;;;GAOG;AACI,MAAM,aAAa,GAAG,CAAC,IAAU,EAAsB,EAAE;;IAC9D,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,QAAQ,mCAAI,IAAI,CAAC,IAAI,CAAC;IAE5C,IAAI,CAAC,QAAQ,EAAE;QACb,OAAO,MAAA,IAAI,CAAC,GAAG,0CAAE,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;KACnC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AARW,QAAA,aAAa,iBAQxB;AAEF;;;;;;;;;;;GAWG;AACH,SAAgB,sBAAsB,CACpC,WAAmB,EACnB,YAIG;IAEH,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAErC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAE5C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5E,wCAAwC;QACxC,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAElC,IAAI,MAAM,CAAC;QACX,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,GAAG;gBACP,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC;SACH;aAAM,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACvC;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAClF;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,QAAQ;YACpC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,IAAI,IAAA,qBAAa,EAAC,IAAI,CAAC,EAAE;YAClD,CAAC,CAAC,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC;QAExB,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,OAAO,EAAE,IAAI,EAAE,CAAC;SACjB;QAED,MAAM,MAAM,GAAG;YACb,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,GAAG,EAAE,SAAS;SACf,CAAC;QAEF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AA7CD,wDA6CC;AAED;;;;;;;;EAQE;AACK,KAAK,UAAU,eAAe,CACnC,aAA6B,EAC7B,YAAyB,EACzB,GAA4B,EAC5B,wBAA4C;IAE5C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;IAE/C,IAAI,MAAM,IAAI,MAAM,EAAE;QACpB,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,wBAAwB,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;SAC7G;QACD,MAAM,gBAAgB,GAAG,MAAM,YAAY,CAAC,MAAM,CAChD,GAAG,EACH;YACE,kBAAkB,EAAE,wBAAwB;YAC5C,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,EACD;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,IAAA,qBAAa,EAAC,IAAI,CAAC;SAC/B,EAAE,CAAC,GAAG,CAAC,CACT,CAAC;QACF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE;YACjC,MAAM,IAAI,2BAAkB,CAAC,iEAAiE,CAAC,CAAC;SACjG;QACD,MAAM,eAAe,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAE5C,MAAM,EAAE,CAAC,UAAU,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC,MAAM;YAC3B,SAAS,EAAE,MAAM,CAAC,GAAG;YACrB,iBAAiB,EAAE,MAAM,CAAC,MAAM;YAChC,cAAc,EAAE,MAAM,CAAC,GAAG;YAC1B,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,IAAA,mCAA8B,EAAC,eAAe,CAAC;YAClD,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,IAAA,qBAAa,EAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;SAC7C,CAAC;KACH;IAED,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE;QAC/C,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;KACvB;IAED,MAAM,kBAAkB,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;QACrD,kBAAkB,EAAE,wBAAwB;QAC5C,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,GAAG,EAAE,IAAI,CAAC,GAAG;KACd,CAAC,CAAC;IAEH,OAAO,IAAA,mCAA8B,EAAC,kBAAkB,CAAC,CAAC;AAC5D,CAAC;AA5DD,0CA4DC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,WAAW,GAAG,uGAAuG,CAAC;IAC5H,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,QAAgB;IAChD,OAAO,aAAa,CAAC,QAAQ,CAAC;QAC5B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC5C,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC;AAJD,8CAIC;AAED;;;;;;;;GAQG;AACH,SAAgB,qBAAqB,CACnC,KAAiC;;IAEjC,IAAI,MAAA,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,0CAAE,eAAe,0CAAE,uBAAuB,EAAE;QACnE,OAAO,SAAS,CAAC;KAClB;IAED,OAAO,MAAA,MAAA,KAAK,CAAC,MAAM,CAAC,iBAAiB,mCAChC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,mCACzC,OAAO,CAAC;AACf,CAAC;AAVD,sDAUC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "11.1.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=12.18.0"
|
|
@@ -38,13 +38,13 @@
|
|
|
38
38
|
"author": "Cumulus Authors",
|
|
39
39
|
"license": "Apache-2.0",
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@cumulus/aws-client": "
|
|
42
|
-
"@cumulus/common": "
|
|
43
|
-
"@cumulus/db": "
|
|
44
|
-
"@cumulus/errors": "
|
|
45
|
-
"@cumulus/logger": "
|
|
46
|
-
"@cumulus/message": "
|
|
47
|
-
"@cumulus/sftp-client": "
|
|
41
|
+
"@cumulus/aws-client": "11.1.0",
|
|
42
|
+
"@cumulus/common": "11.1.0",
|
|
43
|
+
"@cumulus/db": "11.1.0",
|
|
44
|
+
"@cumulus/errors": "11.1.0",
|
|
45
|
+
"@cumulus/logger": "11.1.0",
|
|
46
|
+
"@cumulus/message": "11.1.0",
|
|
47
|
+
"@cumulus/sftp-client": "11.1.0",
|
|
48
48
|
"aws-sdk": "^2.585.0",
|
|
49
49
|
"cksum": "^1.3.0",
|
|
50
50
|
"delay": "^4.3.0",
|
|
@@ -56,15 +56,15 @@
|
|
|
56
56
|
"jsftp": "https://github.com/jkovarik/jsftp.git#add_288",
|
|
57
57
|
"lodash": "^4.17.21",
|
|
58
58
|
"mime-types": "^2.1.22",
|
|
59
|
-
"moment": "2.29.
|
|
59
|
+
"moment": "2.29.2",
|
|
60
60
|
"simplecrawler": "^1.1.9",
|
|
61
61
|
"tough-cookie": "^4.0.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cumulus/checksum": "
|
|
65
|
-
"@cumulus/cmrjs": "
|
|
66
|
-
"@cumulus/test-data": "
|
|
67
|
-
"@cumulus/types": "
|
|
64
|
+
"@cumulus/checksum": "11.1.0",
|
|
65
|
+
"@cumulus/cmrjs": "11.1.0",
|
|
66
|
+
"@cumulus/test-data": "11.1.0",
|
|
67
|
+
"@cumulus/types": "11.1.0"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "a846e319e143684ebc408e35f8b51cc1a7195391"
|
|
70
70
|
}
|
package/src/granule.ts
CHANGED
|
@@ -3,9 +3,9 @@ import moment from 'moment';
|
|
|
3
3
|
import { s3 } from '@cumulus/aws-client/services';
|
|
4
4
|
import * as S3 from '@cumulus/aws-client/S3';
|
|
5
5
|
import * as log from '@cumulus/common/log';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
6
|
+
import { ApiFile, DuplicateHandling } from '@cumulus/types';
|
|
7
|
+
import { FilePgModel, translatePostgresFileToApiFile, Knex } from '@cumulus/db';
|
|
8
|
+
import { RecordDoesNotExist } from '@cumulus/errors';
|
|
9
9
|
|
|
10
10
|
export interface EventWithDuplicateHandling {
|
|
11
11
|
config: {
|
|
@@ -26,7 +26,8 @@ export interface File {
|
|
|
26
26
|
key?: string,
|
|
27
27
|
fileName?: string,
|
|
28
28
|
name?: string,
|
|
29
|
-
filename?: string
|
|
29
|
+
filename?: string,
|
|
30
|
+
size?: string
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
export interface MovedGranuleFile {
|
|
@@ -283,8 +284,23 @@ export async function handleDuplicateFile(params: {
|
|
|
283
284
|
return [];
|
|
284
285
|
}
|
|
285
286
|
|
|
286
|
-
|
|
287
|
-
|
|
287
|
+
/**
|
|
288
|
+
* Get the name of the file from the following properties (in order of preference):
|
|
289
|
+
* 1. fileName (e.g. 'granuleFileNamec.md')
|
|
290
|
+
* 2. name (e.g. 'granuleFileName.md')
|
|
291
|
+
* 3. key (e.g. 'stackname/filepath/granuleFileName.md')
|
|
292
|
+
* @param {File} file - file object with the above properties
|
|
293
|
+
* @returns {string | undefined} - The file name as a string or undefined
|
|
294
|
+
*/
|
|
295
|
+
export const getNameOfFile = (file: File): string | undefined => {
|
|
296
|
+
const fileName = file.fileName ?? file.name;
|
|
297
|
+
|
|
298
|
+
if (!fileName) {
|
|
299
|
+
return file.key?.split('/').pop();
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
return fileName;
|
|
303
|
+
};
|
|
288
304
|
|
|
289
305
|
/**
|
|
290
306
|
* For each source file, see if there is a destination and generate the source
|
|
@@ -346,149 +362,74 @@ export function generateMoveFileParams(
|
|
|
346
362
|
}
|
|
347
363
|
|
|
348
364
|
/**
|
|
349
|
-
|
|
350
|
-
*
|
|
351
|
-
* @param {Array<Object>} sourceFiles - array of file objects, they are updated with destination
|
|
352
|
-
* location after the files are moved
|
|
353
|
-
* @param {string} sourceFiles.name - file name
|
|
354
|
-
* @param {string} sourceFiles.bucket - current bucket of file
|
|
355
|
-
* @param {string} sourceFiles.key - current S3 key of file
|
|
356
|
-
* @param {Array<Object>} destinations - array of objects defining the destination of granule files
|
|
357
|
-
* @param {string} destinations.regex - regex for matching filepath of file to new destination
|
|
358
|
-
* @param {string} destinations.bucket - aws bucket of the destination
|
|
359
|
-
* @param {string} destinations.filepath - file path/directory on the bucket for the destination
|
|
360
|
-
* @returns {Promise<Array>} returns array of source files updated with new locations.
|
|
361
|
-
*/
|
|
362
|
-
export async function moveGranuleFiles(
|
|
363
|
-
sourceFiles: File[],
|
|
364
|
-
destinations: {
|
|
365
|
-
regex: string,
|
|
366
|
-
bucket: string,
|
|
367
|
-
filepath: string
|
|
368
|
-
}[]
|
|
369
|
-
): Promise<MovedGranuleFile[]> {
|
|
370
|
-
deprecate(
|
|
371
|
-
'@cumulus/ingest/moveGranuleFiles',
|
|
372
|
-
'9.0.0'
|
|
373
|
-
);
|
|
374
|
-
const moveFileParams = generateMoveFileParams(sourceFiles, destinations);
|
|
375
|
-
const movedGranuleFiles: MovedGranuleFile[] = [];
|
|
376
|
-
const moveFileRequests = moveFileParams.map(
|
|
377
|
-
async (moveFileParam) => {
|
|
378
|
-
const { source, target, file } = moveFileParam;
|
|
379
|
-
|
|
380
|
-
if (source && target) {
|
|
381
|
-
log.debug('moveGranuleFiles', source, target);
|
|
382
|
-
|
|
383
|
-
await S3.moveObject({
|
|
384
|
-
sourceBucket: source.Bucket,
|
|
385
|
-
sourceKey: source.Key,
|
|
386
|
-
destinationBucket: target.Bucket,
|
|
387
|
-
destinationKey: target.Key,
|
|
388
|
-
copyTags: true,
|
|
389
|
-
});
|
|
390
|
-
|
|
391
|
-
movedGranuleFiles.push({
|
|
392
|
-
bucket: target.Bucket,
|
|
393
|
-
key: target.Key,
|
|
394
|
-
name: getNameOfFile(file),
|
|
395
|
-
});
|
|
396
|
-
} else {
|
|
397
|
-
let fileBucket;
|
|
398
|
-
let fileKey;
|
|
399
|
-
if (file.bucket && file.key) {
|
|
400
|
-
fileBucket = file.bucket;
|
|
401
|
-
fileKey = file.key;
|
|
402
|
-
} else if (file.filename) {
|
|
403
|
-
const parsed = S3.parseS3Uri(file.filename);
|
|
404
|
-
fileBucket = parsed.Bucket;
|
|
405
|
-
fileKey = parsed.Key;
|
|
406
|
-
} else {
|
|
407
|
-
throw new Error(`Unable to determine location of file: ${JSON.stringify(file)}`);
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
movedGranuleFiles.push({
|
|
411
|
-
bucket: fileBucket,
|
|
412
|
-
key: fileKey,
|
|
413
|
-
name: getNameOfFile(file),
|
|
414
|
-
});
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
);
|
|
418
|
-
await Promise.all(moveFileRequests);
|
|
419
|
-
return movedGranuleFiles;
|
|
420
|
-
}
|
|
421
|
-
/**
|
|
422
|
-
* Moves a granule file and updates the postgres database accordingly
|
|
365
|
+
* Moves a granule file and updates the datastore accordingly
|
|
423
366
|
* @summary Moves a granule file record according to MoveFileParams and updates database accordingly
|
|
424
367
|
* @param {MoveFileParams} moveFileParam - Parameter object describing the move operation
|
|
425
368
|
* @param {FilePgModel} filesPgModel - FilePgModel instance
|
|
426
369
|
* @param {Knex.Transaction | Knex} trx - Knex transaction or (optionally) Knex object
|
|
427
370
|
* @param {number | undefined } postgresCumulusGranuleId - postgres internal granule id
|
|
428
|
-
* @
|
|
429
|
-
* @returns {ReturnValueDataTypeHere} Brief description of the returning value here.
|
|
371
|
+
* @returns {Promise<Object>} - Returns object of type Omit<ApiFile, 'granuleId>>
|
|
430
372
|
*/
|
|
431
373
|
export async function moveGranuleFile(
|
|
432
374
|
moveFileParam: MoveFileParams,
|
|
433
375
|
filesPgModel: FilePgModel,
|
|
434
376
|
trx: Knex.Transaction | Knex,
|
|
435
|
-
postgresCumulusGranuleId: number | undefined
|
|
436
|
-
|
|
437
|
-
): Promise<MovedGranuleFile> {
|
|
377
|
+
postgresCumulusGranuleId: number | undefined
|
|
378
|
+
): Promise<Omit<ApiFile, 'granuleId'>> {
|
|
438
379
|
const { source, target, file } = moveFileParam;
|
|
439
|
-
|
|
380
|
+
|
|
381
|
+
if (source && target) {
|
|
440
382
|
log.debug('moveGranuleS3Object', source, target);
|
|
441
|
-
if (
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
383
|
+
if (!postgresCumulusGranuleId) {
|
|
384
|
+
throw new Error('postgresCumulusGranuleId must be defined to move granule file if writeToPostgres is true');
|
|
385
|
+
}
|
|
386
|
+
const updatedPgRecords = await filesPgModel.update(
|
|
387
|
+
trx,
|
|
388
|
+
{
|
|
446
389
|
granule_cumulus_id: postgresCumulusGranuleId,
|
|
447
|
-
bucket:
|
|
448
|
-
key:
|
|
449
|
-
});
|
|
450
|
-
await filesPgModel.update(trx, {
|
|
451
|
-
cumulus_id: cumulusId,
|
|
390
|
+
bucket: source.Bucket,
|
|
391
|
+
key: source.Key,
|
|
452
392
|
},
|
|
453
393
|
{
|
|
454
|
-
bucket:
|
|
455
|
-
key:
|
|
394
|
+
bucket: target.Bucket,
|
|
395
|
+
key: target.Key,
|
|
456
396
|
file_name: getNameOfFile(file),
|
|
457
|
-
}
|
|
397
|
+
}, ['*']
|
|
398
|
+
);
|
|
399
|
+
if (updatedPgRecords.length !== 1) {
|
|
400
|
+
throw new RecordDoesNotExist('Attempted to update granule on move, but granule does not exist');
|
|
458
401
|
}
|
|
402
|
+
const updatedPgRecord = updatedPgRecords[0];
|
|
403
|
+
|
|
459
404
|
await S3.moveObject({
|
|
460
|
-
sourceBucket:
|
|
461
|
-
sourceKey:
|
|
462
|
-
destinationBucket:
|
|
463
|
-
destinationKey:
|
|
405
|
+
sourceBucket: source.Bucket,
|
|
406
|
+
sourceKey: source.Key,
|
|
407
|
+
destinationBucket: target.Bucket,
|
|
408
|
+
destinationKey: target.Key,
|
|
464
409
|
copyTags: true,
|
|
465
410
|
});
|
|
466
411
|
|
|
467
412
|
return {
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
413
|
+
...translatePostgresFileToApiFile(updatedPgRecord),
|
|
414
|
+
bucket: target.Bucket,
|
|
415
|
+
key: target.Key,
|
|
416
|
+
fileName: getNameOfFile({ key: target.Key }),
|
|
471
417
|
};
|
|
472
418
|
}
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
if (file.bucket && file.key) {
|
|
476
|
-
fileBucket = file.bucket;
|
|
477
|
-
fileKey = file.key;
|
|
478
|
-
} else if (file.filename) {
|
|
419
|
+
|
|
420
|
+
if (!(file.bucket || file.key) && file.filename) {
|
|
479
421
|
const parsed = S3.parseS3Uri(file.filename);
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
} else {
|
|
483
|
-
throw new Error(
|
|
484
|
-
`Unable to determine location of file: ${JSON.stringify(file)}`
|
|
485
|
-
);
|
|
422
|
+
file.bucket = parsed.Bucket;
|
|
423
|
+
file.key = parsed.Key;
|
|
486
424
|
}
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
425
|
+
|
|
426
|
+
const postgresFileRecord = await filesPgModel.get(trx, {
|
|
427
|
+
granule_cumulus_id: postgresCumulusGranuleId,
|
|
428
|
+
bucket: file.bucket,
|
|
429
|
+
key: file.key,
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
return translatePostgresFileToApiFile(postgresFileRecord);
|
|
492
433
|
}
|
|
493
434
|
|
|
494
435
|
/**
|