@cumulus/ingest 17.0.0 → 18.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lock.d.ts +27 -23
- package/lock.d.ts.map +1 -1
- package/lock.js +39 -34
- package/lock.js.map +1 -1
- package/package.json +13 -13
- package/queue.d.ts +71 -59
- package/queue.d.ts.map +1 -1
- package/queue.js +45 -37
- package/queue.js.map +1 -1
- package/src/lock.ts +43 -36
- package/src/queue.js +45 -37
- package/tsconfig.tsbuildinfo +1 -1
package/lock.d.ts
CHANGED
|
@@ -3,35 +3,39 @@ export interface Lock {
|
|
|
3
3
|
LastModified?: Date;
|
|
4
4
|
}
|
|
5
5
|
/**
|
|
6
|
-
* Checks all locks and removes those older than five minutes. Returns a count
|
|
7
|
-
* of locks that are not older than
|
|
8
|
-
*
|
|
9
|
-
* @param {
|
|
10
|
-
* @param {Array} locks - The list of locks in the bucket
|
|
11
|
-
* @
|
|
12
|
-
|
|
13
|
-
|
|
6
|
+
* Checks all locks and removes those older than five minutes. Returns a count
|
|
7
|
+
* of locks that are not older than configured retention period or 5 minutes.
|
|
8
|
+
*
|
|
9
|
+
* @param {object} bucket - The AWS S3 bucket with the locks to check
|
|
10
|
+
* @param {Array} locks - The list of locks in the bucket
|
|
11
|
+
* @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
|
|
12
|
+
* @returns {integer} - Number of locks remaining in bucket
|
|
13
|
+
*/
|
|
14
|
+
export declare function checkOldLocks(bucket: string, locks?: Lock[], retentionTimeInSeconds?: number): Promise<number>;
|
|
14
15
|
/**
|
|
15
|
-
* Counts the number of locks in a bucket.
|
|
16
|
-
*
|
|
17
|
-
* @param {
|
|
18
|
-
* @param {string} providerName - The provider name
|
|
19
|
-
* @
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
export declare function
|
|
16
|
+
* Counts the number of locks in a bucket.
|
|
17
|
+
*
|
|
18
|
+
* @param {object} bucket - The AWS S3 bucket to check
|
|
19
|
+
* @param {string} providerName - The provider name
|
|
20
|
+
* @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
|
|
21
|
+
* @returns {integer} - Number of current locks in the bucket
|
|
22
|
+
*/
|
|
23
|
+
export declare function countLock(bucket: string, providerName: string, retentionTimeInSeconds?: number): Promise<number>;
|
|
24
|
+
export declare function removeLock(bucket: string, providerName: string, granuleId: string): Promise<void>;
|
|
23
25
|
/**
|
|
24
26
|
*
|
|
25
|
-
* @param {string} bucket
|
|
26
|
-
* @param {
|
|
27
|
-
* @param {string} provider.id
|
|
28
|
-
* @param {number} provider.globalConnectionLimit
|
|
29
|
-
* @param {
|
|
30
|
-
* @param {
|
|
27
|
+
* @param {string} bucket - system bucket to place the lock files
|
|
28
|
+
* @param {object} provider - provider object
|
|
29
|
+
* @param {string} provider.id - provider id
|
|
30
|
+
* @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
|
|
31
|
+
* @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
|
|
32
|
+
* @param {string} granuleId - id of downloading granule
|
|
33
|
+
* @param {number} counter - retry counter
|
|
31
34
|
* @returns {Promise<boolean>}
|
|
32
35
|
*/
|
|
33
36
|
export declare function proceed(bucket: string, provider: {
|
|
34
37
|
id: string;
|
|
35
38
|
globalConnectionLimit?: number;
|
|
36
|
-
|
|
39
|
+
maxDownloadTime?: number;
|
|
40
|
+
}, granuleId: string, counter?: number): Promise<boolean>;
|
|
37
41
|
//# sourceMappingURL=lock.d.ts.map
|
package/lock.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED
|
|
1
|
+
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,IAAI;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,IAAI,CAAA;CACpB;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,IAAI,EAAO,EAClB,sBAAsB,GAAE,MAAY,GACnC,OAAO,CAAC,MAAM,CAAC,CAqBjB;AAED;;;;;;;GAOG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,sBAAsB,CAAC,EAAE,MAAM,GAC9B,OAAO,CAAC,MAAM,CAAC,CAOjB;AAeD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE;IACR,EAAE,EAAE,MAAM,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,EACD,SAAS,EAAE,MAAM,EACjB,OAAO,SAAI,GACV,OAAO,CAAC,OAAO,CAAC,CAwBlB"}
|
package/lock.js
CHANGED
|
@@ -29,85 +29,90 @@ const common_1 = require("@cumulus/common");
|
|
|
29
29
|
const log = __importStar(require("@cumulus/common/log"));
|
|
30
30
|
const lockPrefix = 'lock';
|
|
31
31
|
/**
|
|
32
|
-
* Checks all locks and removes those older than five minutes. Returns a count
|
|
33
|
-
* of locks that are not older than
|
|
34
|
-
*
|
|
35
|
-
* @param {
|
|
36
|
-
* @param {Array} locks - The list of locks in the bucket
|
|
37
|
-
* @
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
32
|
+
* Checks all locks and removes those older than five minutes. Returns a count
|
|
33
|
+
* of locks that are not older than configured retention period or 5 minutes.
|
|
34
|
+
*
|
|
35
|
+
* @param {object} bucket - The AWS S3 bucket with the locks to check
|
|
36
|
+
* @param {Array} locks - The list of locks in the bucket
|
|
37
|
+
* @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
|
|
38
|
+
* @returns {integer} - Number of locks remaining in bucket
|
|
39
|
+
*/
|
|
40
|
+
async function checkOldLocks(bucket, locks = [], retentionTimeInSeconds = 300) {
|
|
41
|
+
const expirationTimestamp = Date.now() - (retentionTimeInSeconds * 1000);
|
|
41
42
|
const expiredLocks = locks.filter((lock) => {
|
|
42
43
|
if (!lock.LastModified) {
|
|
43
44
|
throw new TypeError(`Could not find LastModified on ${JSON.stringify(lock)}`);
|
|
44
45
|
}
|
|
45
|
-
return lock.LastModified.getTime() <
|
|
46
|
+
return lock.LastModified.getTime() < expirationTimestamp;
|
|
46
47
|
});
|
|
47
48
|
await Promise.all(expiredLocks.map((lock) => {
|
|
48
49
|
if (!lock.Key) {
|
|
49
50
|
throw new TypeError(`Could not find Key on ${JSON.stringify(lock)}`);
|
|
50
51
|
}
|
|
52
|
+
log.debug(`Removing expired lock ${JSON.stringify(lock)}`);
|
|
51
53
|
return (0, S3_1.deleteS3Object)(bucket, lock.Key);
|
|
52
54
|
}));
|
|
53
55
|
return locks.length - expiredLocks.length;
|
|
54
56
|
}
|
|
55
57
|
exports.checkOldLocks = checkOldLocks;
|
|
56
58
|
/**
|
|
57
|
-
* Counts the number of locks in a bucket.
|
|
58
|
-
*
|
|
59
|
-
* @param {
|
|
60
|
-
* @param {string} providerName - The provider name
|
|
61
|
-
* @
|
|
62
|
-
|
|
63
|
-
|
|
59
|
+
* Counts the number of locks in a bucket.
|
|
60
|
+
*
|
|
61
|
+
* @param {object} bucket - The AWS S3 bucket to check
|
|
62
|
+
* @param {string} providerName - The provider name
|
|
63
|
+
* @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
|
|
64
|
+
* @returns {integer} - Number of current locks in the bucket
|
|
65
|
+
*/
|
|
66
|
+
async function countLock(bucket, providerName, retentionTimeInSeconds) {
|
|
64
67
|
const locks = await (0, S3_1.listS3ObjectsV2)({
|
|
65
68
|
Bucket: bucket,
|
|
66
69
|
Prefix: `${lockPrefix}/${providerName}`,
|
|
67
70
|
});
|
|
68
|
-
return checkOldLocks(bucket, locks);
|
|
71
|
+
return checkOldLocks(bucket, locks, retentionTimeInSeconds);
|
|
69
72
|
}
|
|
70
73
|
exports.countLock = countLock;
|
|
71
|
-
async function addLock(bucket, providerName,
|
|
72
|
-
const key = `${lockPrefix}/${providerName}/${
|
|
74
|
+
async function addLock(bucket, providerName, granuleId) {
|
|
75
|
+
const key = `${lockPrefix}/${providerName}/${granuleId}`;
|
|
73
76
|
await (0, S3_1.s3PutObject)({
|
|
74
77
|
Bucket: bucket,
|
|
75
78
|
Key: key,
|
|
76
79
|
Body: '',
|
|
77
80
|
});
|
|
78
81
|
}
|
|
79
|
-
async function removeLock(bucket, providerName,
|
|
80
|
-
await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${
|
|
82
|
+
async function removeLock(bucket, providerName, granuleId) {
|
|
83
|
+
await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${granuleId}`);
|
|
81
84
|
}
|
|
82
85
|
exports.removeLock = removeLock;
|
|
83
86
|
/**
|
|
84
87
|
*
|
|
85
|
-
* @param {string} bucket
|
|
86
|
-
* @param {
|
|
87
|
-
* @param {string} provider.id
|
|
88
|
-
* @param {number} provider.globalConnectionLimit
|
|
89
|
-
* @param {
|
|
90
|
-
* @param {
|
|
88
|
+
* @param {string} bucket - system bucket to place the lock files
|
|
89
|
+
* @param {object} provider - provider object
|
|
90
|
+
* @param {string} provider.id - provider id
|
|
91
|
+
* @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
|
|
92
|
+
* @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
|
|
93
|
+
* @param {string} granuleId - id of downloading granule
|
|
94
|
+
* @param {number} counter - retry counter
|
|
91
95
|
* @returns {Promise<boolean>}
|
|
92
96
|
*/
|
|
93
|
-
async function proceed(bucket, provider,
|
|
94
|
-
|
|
97
|
+
async function proceed(bucket, provider, granuleId, counter = 0) {
|
|
98
|
+
const { globalConnectionLimit, maxDownloadTime } = provider;
|
|
99
|
+
if (globalConnectionLimit === undefined) {
|
|
95
100
|
return true;
|
|
96
101
|
}
|
|
97
102
|
// Fail if lock is not removed after 270 tries.
|
|
98
103
|
if (counter > 270) {
|
|
104
|
+
log.debug(`The "${provider.id}" provider has no lock available after ${counter} retries`);
|
|
99
105
|
return false;
|
|
100
106
|
}
|
|
101
|
-
const
|
|
102
|
-
const count = await countLock(bucket, provider.id);
|
|
107
|
+
const count = await countLock(bucket, provider.id, maxDownloadTime);
|
|
103
108
|
if (count >= globalConnectionLimit) {
|
|
104
109
|
log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
|
|
105
110
|
// wait for 5 second and try again
|
|
106
111
|
await (0, common_1.sleep)(5000);
|
|
107
|
-
return proceed(bucket, provider,
|
|
112
|
+
return proceed(bucket, provider, granuleId, counter + 1);
|
|
108
113
|
}
|
|
109
114
|
// add the lock
|
|
110
|
-
await addLock(bucket, provider.id,
|
|
115
|
+
await addLock(bucket, provider.id, granuleId);
|
|
111
116
|
return true;
|
|
112
117
|
}
|
|
113
118
|
exports.proceed = proceed;
|
package/lock.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAIgC;AAChC,4CAAwC;AACxC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B
|
|
1
|
+
{"version":3,"file":"lock.js","sourceRoot":"","sources":["src/lock.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAIgC;AAChC,4CAAwC;AACxC,yDAA2C;AAE3C,MAAM,UAAU,GAAG,MAAM,CAAC;AAO1B;;;;;;;;GAQG;AACI,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,QAAgB,EAAE,EAClB,yBAAiC,GAAG;IAEpC,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;IAEzE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,IAAI,EAAE,EAAE;QACP,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACtB,MAAM,IAAI,SAAS,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SAC/E;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,mBAAmB,CAAC;IAC3D,CAAC,CACF,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,IAAI,SAAS,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACtE;QACD,GAAG,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3D,OAAO,IAAA,mBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC,CAAC;IAEJ,OAAO,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;AAC5C,CAAC;AAzBD,sCAyBC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,YAAoB,EACpB,sBAA+B;IAE/B,MAAM,KAAK,GAAG,MAAM,IAAA,oBAAe,EAAC;QAClC,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,GAAG,UAAU,IAAI,YAAY,EAAE;KACxC,CAAC,CAAC;IAEH,OAAO,aAAa,CAAC,MAAM,EAAE,KAAK,EAAE,sBAAsB,CAAC,CAAC;AAC9D,CAAC;AAXD,8BAWC;AAED,KAAK,UAAU,OAAO,CACpB,MAAc,EACd,YAAoB,EACpB,SAAiB;IAEjB,MAAM,GAAG,GAAG,GAAG,UAAU,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;IACzD,MAAM,IAAA,gBAAW,EAAC;QAChB,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,IAAI,EAAE,EAAE;KACT,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,YAAoB,EACpB,SAAiB;IAEjB,MAAM,IAAA,mBAAc,EAClB,MAAM,EACN,GAAG,UAAU,IAAI,YAAY,IAAI,SAAS,EAAE,CAC7C,CAAC;AACJ,CAAC;AATD,gCASC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,QAIC,EACD,SAAiB,EACjB,OAAO,GAAG,CAAC;IAEX,MAAM,EAAE,qBAAqB,EAAE,eAAe,EAAE,GAAG,QAAQ,CAAC;IAC5D,IAAI,qBAAqB,KAAK,SAAS,EAAE;QACvC,OAAO,IAAI,CAAC;KACb;IAED,+CAA+C;IAC/C,IAAI,OAAO,GAAG,GAAG,EAAE;QACjB,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,OAAO,UAAU,CAAC,CAAC;QAC1F,OAAO,KAAK,CAAC;KACd;IAED,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;IAEpE,IAAI,KAAK,IAAI,qBAAqB,EAAE;QAClC,GAAG,CAAC,KAAK,CAAC,QAAQ,QAAQ,CAAC,EAAE,0CAA0C,QAAQ,CAAC,qBAAqB,qBAAqB,CAAC,CAAC;QAC5H,kCAAkC;QAClC,MAAM,IAAA,cAAK,EAAC,IAAI,CAAC,CAAC;QAClB,OAAO,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;KAC1D;IAED,eAAe;IACf,MAAM,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAjCD,0BAiCC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "18.1.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.19.0"
|
|
@@ -39,13 +39,13 @@
|
|
|
39
39
|
"author": "Cumulus Authors",
|
|
40
40
|
"license": "Apache-2.0",
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@cumulus/aws-client": "
|
|
43
|
-
"@cumulus/common": "
|
|
44
|
-
"@cumulus/db": "
|
|
45
|
-
"@cumulus/errors": "
|
|
46
|
-
"@cumulus/logger": "
|
|
47
|
-
"@cumulus/message": "
|
|
48
|
-
"@cumulus/sftp-client": "
|
|
42
|
+
"@cumulus/aws-client": "18.1.0",
|
|
43
|
+
"@cumulus/common": "18.1.0",
|
|
44
|
+
"@cumulus/db": "18.1.0",
|
|
45
|
+
"@cumulus/errors": "18.1.0",
|
|
46
|
+
"@cumulus/logger": "18.1.0",
|
|
47
|
+
"@cumulus/message": "18.1.0",
|
|
48
|
+
"@cumulus/sftp-client": "18.1.0",
|
|
49
49
|
"aws-sdk": "^2.585.0",
|
|
50
50
|
"cksum": "^1.3.0",
|
|
51
51
|
"encodeurl": "^1.0.2",
|
|
@@ -61,10 +61,10 @@
|
|
|
61
61
|
"tough-cookie": "~4.0.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cumulus/checksum": "
|
|
65
|
-
"@cumulus/cmrjs": "
|
|
66
|
-
"@cumulus/test-data": "
|
|
67
|
-
"@cumulus/types": "
|
|
64
|
+
"@cumulus/checksum": "18.1.0",
|
|
65
|
+
"@cumulus/cmrjs": "18.1.0",
|
|
66
|
+
"@cumulus/test-data": "18.1.0",
|
|
67
|
+
"@cumulus/types": "18.1.0"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "8428a02e0903d8d303c7dd36215aa6a24906e07b"
|
|
70
70
|
}
|
package/queue.d.ts
CHANGED
|
@@ -1,82 +1,94 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Enqueue a PDR to be parsed
|
|
3
3
|
*
|
|
4
|
-
* @param {
|
|
5
|
-
* @param {
|
|
6
|
-
* @param {string} params.queueUrl - the SQS queue to add the message to
|
|
7
|
-
* @param {string} params.parsePdrMessageTemplateUri - the S3 URI of template for
|
|
8
|
-
* a PDR parse message
|
|
9
|
-
* @param {Object} params.provider - the provider config to be attached to the message
|
|
10
|
-
* @param {Object} params.collection - the collection config to be attached to the
|
|
11
|
-
* message
|
|
4
|
+
* @param {object} params
|
|
5
|
+
* @param {object} params.collection - the collection config to be attached to the message
|
|
12
6
|
* @param {string} params.parentExecutionArn - parent workflow execution arn to add to the message
|
|
13
|
-
* @param {string}
|
|
14
|
-
*
|
|
15
|
-
* @param {
|
|
16
|
-
* @
|
|
7
|
+
* @param {string} params.parsePdrWorkflow - the S3 URI of template for a PDR parse message
|
|
8
|
+
* @param {object} params.pdr - the PDR to be enqueued for parsing
|
|
9
|
+
* @param {object} params.provider - the provider config to be attached to the message
|
|
10
|
+
* @param {string} params.stack - stack name
|
|
11
|
+
* @param {string} params.systemBucket - system bucket name
|
|
12
|
+
* @param {string} params.queueUrl - the SQS queue to add the message to
|
|
13
|
+
* @param {string} [params.executionNamePrefix]
|
|
14
|
+
* - the prefix to apply to the name of the enqueued execution
|
|
15
|
+
* @param {object} [params.additionalCustomMeta]
|
|
16
|
+
* - additional object to merge into meta object
|
|
17
|
+
* @returns {Promise<string>}
|
|
18
|
+
* - resolves when the message has been enqueued
|
|
17
19
|
*/
|
|
18
20
|
export function enqueueParsePdrMessage({ collection, parentExecutionArn, parsePdrWorkflow, pdr, provider, stack, systemBucket, queueUrl, executionNamePrefix, additionalCustomMeta, }: {
|
|
19
|
-
|
|
20
|
-
queueUrl: string;
|
|
21
|
-
parsePdrMessageTemplateUri: string;
|
|
22
|
-
provider: Object;
|
|
23
|
-
collection: Object;
|
|
21
|
+
collection: object;
|
|
24
22
|
parentExecutionArn: string;
|
|
23
|
+
parsePdrWorkflow: string;
|
|
24
|
+
pdr: object;
|
|
25
|
+
provider: object;
|
|
26
|
+
stack: string;
|
|
27
|
+
systemBucket: string;
|
|
28
|
+
queueUrl: string;
|
|
25
29
|
executionNamePrefix?: string | undefined;
|
|
26
|
-
additionalCustomMeta?:
|
|
27
|
-
}): Promise<
|
|
30
|
+
additionalCustomMeta?: object | undefined;
|
|
31
|
+
}): Promise<string>;
|
|
28
32
|
/**
|
|
29
33
|
* Enqueue a granule to be ingested
|
|
30
34
|
*
|
|
31
|
-
* @param {
|
|
32
|
-
* @param {
|
|
33
|
-
* @param {
|
|
34
|
-
* @param {
|
|
35
|
-
*
|
|
36
|
-
* @param {
|
|
37
|
-
* @param {
|
|
38
|
-
*
|
|
39
|
-
* @param {
|
|
40
|
-
* @param {
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* @
|
|
35
|
+
* @param {object} params
|
|
36
|
+
* @param {object} params.collection - the collection config to be attached to the message
|
|
37
|
+
* @param {object} params.granules - the granules to be enqueued for ingest
|
|
38
|
+
* @param {string} params.parentExecutionArn
|
|
39
|
+
* - parent workflow execution arn to add to the message
|
|
40
|
+
* @param {object} params.provider - the provider config to be attached to the message
|
|
41
|
+
* @param {object} params.messageTemplate - Message template for the workflow
|
|
42
|
+
* @param {object} params.workflow - workflow name & arn object
|
|
43
|
+
* @param {string} params.queueUrl - the SQS queue to add the message to
|
|
44
|
+
* @param {object | undefined} [params.pdr]
|
|
45
|
+
* - an optional PDR to be configured in the message payload
|
|
46
|
+
* @param {string} [params.executionNamePrefix]
|
|
47
|
+
* - the prefix to apply to the name of the enqueued execution
|
|
48
|
+
* @param {object} [params.additionalCustomMeta]
|
|
49
|
+
* - additional object to merge into meta object
|
|
50
|
+
* @returns {Promise<string>}
|
|
51
|
+
* - resolves when the message has been enqueued
|
|
45
52
|
*/
|
|
46
53
|
export function enqueueGranuleIngestMessage({ collection, granules, parentExecutionArn, pdr, provider, messageTemplate, workflow, queueUrl, executionNamePrefix, additionalCustomMeta, }: {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
messageTemplate: Object;
|
|
50
|
-
workflow: Object;
|
|
51
|
-
provider: Object;
|
|
52
|
-
collection: Object;
|
|
53
|
-
pdr: Object;
|
|
54
|
+
collection: object;
|
|
55
|
+
granules: object;
|
|
54
56
|
parentExecutionArn: string;
|
|
57
|
+
provider: object;
|
|
58
|
+
messageTemplate: object;
|
|
59
|
+
workflow: object;
|
|
60
|
+
queueUrl: string;
|
|
61
|
+
pdr?: object | undefined;
|
|
55
62
|
executionNamePrefix?: string | undefined;
|
|
56
|
-
additionalCustomMeta?:
|
|
57
|
-
}): Promise<
|
|
63
|
+
additionalCustomMeta?: object | undefined;
|
|
64
|
+
}): Promise<string>;
|
|
58
65
|
/**
|
|
59
66
|
* Enqueue a workflow
|
|
60
67
|
*
|
|
61
|
-
* @param {
|
|
62
|
-
* @param {
|
|
63
|
-
*
|
|
64
|
-
* @param {string}
|
|
65
|
-
* @param {
|
|
66
|
-
* @param {
|
|
67
|
-
*
|
|
68
|
-
* @param {
|
|
69
|
-
* @param {
|
|
70
|
-
*
|
|
71
|
-
* @
|
|
68
|
+
* @param {object} params
|
|
69
|
+
* @param {string} params.parentExecutionArn
|
|
70
|
+
* - parent workflow execution arn to add to the message
|
|
71
|
+
* @param {string} params.stack - stack name
|
|
72
|
+
* @param {string} params.systemBucket - system bucket name
|
|
73
|
+
* @param {object} params.collection - the collection config to be attached to the message
|
|
74
|
+
* @param {object} params.provider - the provider config to be attached to the message
|
|
75
|
+
* @param {object} params.workflow - the workflow to be enqueued
|
|
76
|
+
* @param {object} params.workflowInput - the input that should be passed to the queued workflow
|
|
77
|
+
* @param {string} [params.queueUrl] - an optional SQS queue to add the message to
|
|
78
|
+
* @param {string} [params.executionNamePrefix]
|
|
79
|
+
* - the prefix to apply to the name of the enqueued execution
|
|
80
|
+
* @returns {Promise<string>}
|
|
81
|
+
* - resolves when the message has been enqueued
|
|
72
82
|
*/
|
|
73
83
|
export function enqueueWorkflowMessage({ parentExecutionArn, stack, systemBucket, collection, provider, queueUrl, workflow, workflowInput, executionNamePrefix, additionalCustomMeta, }: {
|
|
74
|
-
workflow: Object;
|
|
75
|
-
workflowInput: Object;
|
|
76
|
-
queueUrl?: string | undefined;
|
|
77
|
-
provider: Object;
|
|
78
|
-
collection: Object;
|
|
79
84
|
parentExecutionArn: string;
|
|
85
|
+
stack: string;
|
|
86
|
+
systemBucket: string;
|
|
87
|
+
collection: object;
|
|
88
|
+
provider: object;
|
|
89
|
+
workflow: object;
|
|
90
|
+
workflowInput: object;
|
|
91
|
+
queueUrl?: string | undefined;
|
|
80
92
|
executionNamePrefix?: string | undefined;
|
|
81
|
-
}): Promise<
|
|
93
|
+
}): Promise<string>;
|
|
82
94
|
//# sourceMappingURL=queue.d.ts.map
|
package/queue.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAaA
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;;;;GAkBG;AACH;IAf0B,UAAU,EAAzB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IACS,gBAAgB,EAA/B,MAAM;IACS,GAAG,EAAlB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,KAAK,EAApB,MAAM;IACS,YAAY,EAA3B,MAAM;IACS,QAAQ,EAAvB,MAAM;IACU,mBAAmB;IAEnB,oBAAoB;IAElC,QAAQ,MAAM,CAAC,CA+C3B;AAGD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;IAjB0B,UAAU,EAAzB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,eAAe,EAA9B,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACsB,GAAG,GAA/B,MAAM,GAAG,SAAS;IAEF,mBAAmB;IAEnB,oBAAoB;IAElC,QAAQ,MAAM,CAAC,CAkC3B;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH;IAd0B,kBAAkB,EAAjC,MAAM;IAES,KAAK,EAApB,MAAM;IACS,YAAY,EAA3B,MAAM;IACS,UAAU,EAAzB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,aAAa,EAA5B,MAAM;IACU,QAAQ;IACR,mBAAmB;IAEjC,QAAQ,MAAM,CAAC,CAoD3B"}
|
package/queue.js
CHANGED
|
@@ -7,19 +7,21 @@ const { getWorkflowFileKey, templateKey, } = require('@cumulus/common/workflows'
|
|
|
7
7
|
/**
|
|
8
8
|
* Enqueue a PDR to be parsed
|
|
9
9
|
*
|
|
10
|
-
* @param {
|
|
11
|
-
* @param {
|
|
12
|
-
* @param {string} params.queueUrl - the SQS queue to add the message to
|
|
13
|
-
* @param {string} params.parsePdrMessageTemplateUri - the S3 URI of template for
|
|
14
|
-
* a PDR parse message
|
|
15
|
-
* @param {Object} params.provider - the provider config to be attached to the message
|
|
16
|
-
* @param {Object} params.collection - the collection config to be attached to the
|
|
17
|
-
* message
|
|
10
|
+
* @param {object} params
|
|
11
|
+
* @param {object} params.collection - the collection config to be attached to the message
|
|
18
12
|
* @param {string} params.parentExecutionArn - parent workflow execution arn to add to the message
|
|
19
|
-
* @param {string}
|
|
20
|
-
*
|
|
21
|
-
* @param {
|
|
22
|
-
* @
|
|
13
|
+
* @param {string} params.parsePdrWorkflow - the S3 URI of template for a PDR parse message
|
|
14
|
+
* @param {object} params.pdr - the PDR to be enqueued for parsing
|
|
15
|
+
* @param {object} params.provider - the provider config to be attached to the message
|
|
16
|
+
* @param {string} params.stack - stack name
|
|
17
|
+
* @param {string} params.systemBucket - system bucket name
|
|
18
|
+
* @param {string} params.queueUrl - the SQS queue to add the message to
|
|
19
|
+
* @param {string} [params.executionNamePrefix]
|
|
20
|
+
* - the prefix to apply to the name of the enqueued execution
|
|
21
|
+
* @param {object} [params.additionalCustomMeta]
|
|
22
|
+
* - additional object to merge into meta object
|
|
23
|
+
* @returns {Promise<string>}
|
|
24
|
+
* - resolves when the message has been enqueued
|
|
23
25
|
*/
|
|
24
26
|
async function enqueueParsePdrMessage({ collection, parentExecutionArn, parsePdrWorkflow, pdr, provider, stack, systemBucket, queueUrl, executionNamePrefix, additionalCustomMeta = {}, }) {
|
|
25
27
|
const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
|
|
@@ -49,20 +51,23 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
|
|
|
49
51
|
/**
|
|
50
52
|
* Enqueue a granule to be ingested
|
|
51
53
|
*
|
|
52
|
-
* @param {
|
|
53
|
-
* @param {
|
|
54
|
-
* @param {
|
|
55
|
-
* @param {
|
|
56
|
-
*
|
|
57
|
-
* @param {
|
|
58
|
-
* @param {
|
|
59
|
-
*
|
|
60
|
-
* @param {
|
|
61
|
-
* @param {
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
* @
|
|
54
|
+
* @param {object} params
|
|
55
|
+
* @param {object} params.collection - the collection config to be attached to the message
|
|
56
|
+
* @param {object} params.granules - the granules to be enqueued for ingest
|
|
57
|
+
* @param {string} params.parentExecutionArn
|
|
58
|
+
* - parent workflow execution arn to add to the message
|
|
59
|
+
* @param {object} params.provider - the provider config to be attached to the message
|
|
60
|
+
* @param {object} params.messageTemplate - Message template for the workflow
|
|
61
|
+
* @param {object} params.workflow - workflow name & arn object
|
|
62
|
+
* @param {string} params.queueUrl - the SQS queue to add the message to
|
|
63
|
+
* @param {object | undefined} [params.pdr]
|
|
64
|
+
* - an optional PDR to be configured in the message payload
|
|
65
|
+
* @param {string} [params.executionNamePrefix]
|
|
66
|
+
* - the prefix to apply to the name of the enqueued execution
|
|
67
|
+
* @param {object} [params.additionalCustomMeta]
|
|
68
|
+
* - additional object to merge into meta object
|
|
69
|
+
* @returns {Promise<string>}
|
|
70
|
+
* - resolves when the message has been enqueued
|
|
66
71
|
*/
|
|
67
72
|
async function enqueueGranuleIngestMessage({ collection, granules, parentExecutionArn, pdr, provider, messageTemplate, workflow, queueUrl, executionNamePrefix, additionalCustomMeta = {}, }) {
|
|
68
73
|
const message = buildQueueMessageFromTemplate({
|
|
@@ -85,17 +90,20 @@ exports.enqueueGranuleIngestMessage = enqueueGranuleIngestMessage;
|
|
|
85
90
|
/**
|
|
86
91
|
* Enqueue a workflow
|
|
87
92
|
*
|
|
88
|
-
* @param {
|
|
89
|
-
* @param {
|
|
90
|
-
*
|
|
91
|
-
* @param {string}
|
|
92
|
-
* @param {
|
|
93
|
-
* @param {
|
|
94
|
-
*
|
|
95
|
-
* @param {
|
|
96
|
-
* @param {
|
|
97
|
-
*
|
|
98
|
-
* @
|
|
93
|
+
* @param {object} params
|
|
94
|
+
* @param {string} params.parentExecutionArn
|
|
95
|
+
* - parent workflow execution arn to add to the message
|
|
96
|
+
* @param {string} params.stack - stack name
|
|
97
|
+
* @param {string} params.systemBucket - system bucket name
|
|
98
|
+
* @param {object} params.collection - the collection config to be attached to the message
|
|
99
|
+
* @param {object} params.provider - the provider config to be attached to the message
|
|
100
|
+
* @param {object} params.workflow - the workflow to be enqueued
|
|
101
|
+
* @param {object} params.workflowInput - the input that should be passed to the queued workflow
|
|
102
|
+
* @param {string} [params.queueUrl] - an optional SQS queue to add the message to
|
|
103
|
+
* @param {string} [params.executionNamePrefix]
|
|
104
|
+
* - the prefix to apply to the name of the enqueued execution
|
|
105
|
+
* @returns {Promise<string>}
|
|
106
|
+
* - resolves when the message has been enqueued
|
|
99
107
|
*/
|
|
100
108
|
async function enqueueWorkflowMessage({ parentExecutionArn, stack, systemBucket, collection, provider, queueUrl, workflow, workflowInput, executionNamePrefix, additionalCustomMeta = {}, }) {
|
|
101
109
|
const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
|
package/queue.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;;;GAkBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAChD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,gBAAgB;QACtB,GAAG,EAAE,WAAW;KACjB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,MAAM,CAAC,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,KAAK,UAAU,2BAA2B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,kBAAkB,EAClB,GAAG,EACH,QAAQ,EACR,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO,EAAE,EAAE,QAAQ,EAAE;QACrB,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,iBAAiB,CACtB,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;AACJ,CAAC;AACD,OAAO,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;AAElE;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,kBAAkB,EAClB,KAAK,EACL,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,MAAM,eAAe,CACtD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CACpC,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,GAAG,aAAa;KACjB,CAAC;IAEF,MAAM,wBAAwB,GAAG;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,iBAAiB;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,QAAQ,EAAE,wBAAwB;QAClC,mBAAmB;QACnB,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC"}
|