@cumulus/ingest 9.3.0 → 9.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HttpProviderClient.d.ts +11 -4
- package/HttpProviderClient.d.ts.map +1 -1
- package/HttpProviderClient.js +42 -10
- package/HttpProviderClient.js.map +1 -1
- package/package.json +13 -12
- package/queue.d.ts +1 -1
- package/queue.d.ts.map +1 -1
- package/queue.js +2 -6
- package/queue.js.map +1 -1
- package/sqs.d.ts +5 -3
- package/sqs.d.ts.map +1 -1
- package/sqs.js +24 -9
- package/sqs.js.map +1 -1
- package/src/HttpProviderClient.js +54 -14
- package/src/queue.js +2 -6
- package/src/sqs.ts +31 -9
- package/src/util.ts +6 -1
- package/util.d.ts.map +1 -1
- package/util.js +4 -1
- package/util.js.map +1 -1
package/HttpProviderClient.d.ts
CHANGED
|
@@ -9,6 +9,8 @@ declare class HttpProviderClient {
|
|
|
9
9
|
gotOptions: {};
|
|
10
10
|
certificateUri: any;
|
|
11
11
|
encrypted: any;
|
|
12
|
+
defaultRedirect: any;
|
|
13
|
+
allowedRedirects: any;
|
|
12
14
|
endpoint: string;
|
|
13
15
|
setUpGotOptions(): Promise<void>;
|
|
14
16
|
username: any;
|
|
@@ -37,13 +39,18 @@ declare class HttpProviderClient {
|
|
|
37
39
|
/**
|
|
38
40
|
* Download the remote file to a given s3 location
|
|
39
41
|
*
|
|
40
|
-
* @param {
|
|
41
|
-
* @param {string}
|
|
42
|
-
* @param {string}
|
|
42
|
+
* @param {Object} params
|
|
43
|
+
* @param {string} params.fileRemotePath - the full path to the remote file to be fetched
|
|
44
|
+
* @param {string} params.destinationBucket - destination s3 bucket of the file
|
|
45
|
+
* @param {string} params.destinationKey - destination s3 key of the file
|
|
43
46
|
* @returns {Promise.<{ s3uri: string, etag: string }>} an object containing
|
|
44
47
|
* the S3 URI and ETag of the destination file
|
|
45
48
|
*/
|
|
46
|
-
sync(params:
|
|
49
|
+
sync(params: {
|
|
50
|
+
fileRemotePath: string;
|
|
51
|
+
destinationBucket: string;
|
|
52
|
+
destinationKey: string;
|
|
53
|
+
}): Promise<{
|
|
47
54
|
s3uri: string;
|
|
48
55
|
etag: string;
|
|
49
56
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HttpProviderClient.d.ts","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"HttpProviderClient.d.ts","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":";AAiCA;IACE,iCAsBC;IArBC,oBAAoC;IACpC,cAAuC;IACvC,UAA+B;IAC/B,UAA+B;IAC/B,qBAAqD;IACrD,eAAoB;IACpB,oBAAmD;IAInD,eAAyC;IAEzC,qBAA0E;IAC1E,sBAAkE;IAGlE,iBAIE;IAGJ,iCA2CC;IAzCG,cAA2D;IAC3D,cAA2D;IA0C/D,wCAUC;IANG,gCAAqE;IAQzE;;;;;OAKG;IACH,WAHW,MAAM,GACJ,OAAO,CAAC,KAAK,CAAC,CAoF1B;IAED;;;;;;;OAOG;IACH;QAJ0B,UAAU,EAAzB,MAAM;QACS,SAAS,EAAxB,MAAM;QACJ,OAAO,CAAE,MAAM,CAAC,CA8B5B;IAED;;;;;;;;;OASG;IACH;QAN0B,cAAc,EAA7B,MAAM;QACS,iBAAiB,EAAhC,MAAM;QACS,cAAc,EAA7B,MAAM;;eACc,MAAM;cAAQ,MAAM;OAuClD;IAGD,yBAAkB;IAElB,qBAAc;CAEf"}
|
package/HttpProviderClient.js
CHANGED
|
@@ -3,12 +3,12 @@ const fs = require('fs');
|
|
|
3
3
|
const https = require('https');
|
|
4
4
|
const isIp = require('is-ip');
|
|
5
5
|
const { basename } = require('path');
|
|
6
|
-
const {
|
|
6
|
+
const { pipeline } = require('stream');
|
|
7
7
|
const Crawler = require('simplecrawler');
|
|
8
8
|
const got = require('got');
|
|
9
9
|
const { CookieJar } = require('tough-cookie');
|
|
10
10
|
const { promisify } = require('util');
|
|
11
|
-
const { buildS3Uri, getTextObject, parseS3Uri,
|
|
11
|
+
const { buildS3Uri, getTextObject, parseS3Uri, streamS3Upload, } = require('@cumulus/aws-client/S3');
|
|
12
12
|
const log = require('@cumulus/common/log');
|
|
13
13
|
const isValidHostname = require('is-valid-hostname');
|
|
14
14
|
const { buildURL } = require('@cumulus/common/URLUtils');
|
|
@@ -19,6 +19,7 @@ const validateHost = (host) => {
|
|
|
19
19
|
return;
|
|
20
20
|
throw new TypeError(`provider.host is not a valid hostname or IP: ${host}`);
|
|
21
21
|
};
|
|
22
|
+
const redirectCodes = new Set([300, 301, 302, 303, 304, 307, 308]);
|
|
22
23
|
class HttpProviderClient {
|
|
23
24
|
constructor(providerConfig) {
|
|
24
25
|
this.providerConfig = providerConfig;
|
|
@@ -32,6 +33,9 @@ class HttpProviderClient {
|
|
|
32
33
|
throw new ReferenceError('Found providerConfig.username, but providerConfig.password is not defined');
|
|
33
34
|
}
|
|
34
35
|
this.encrypted = providerConfig.encrypted;
|
|
36
|
+
this.defaultRedirect = this.port ? `${this.host}:${this.port}` : this.host;
|
|
37
|
+
this.allowedRedirects = this.providerConfig.allowedRedirects || [];
|
|
38
|
+
this.allowedRedirects.push(this.defaultRedirect);
|
|
35
39
|
this.endpoint = buildURL({
|
|
36
40
|
protocol: this.protocol,
|
|
37
41
|
host: this.host,
|
|
@@ -52,6 +56,34 @@ class HttpProviderClient {
|
|
|
52
56
|
this.gotOptions.username = this.username;
|
|
53
57
|
if (this.password)
|
|
54
58
|
this.gotOptions.password = this.password;
|
|
59
|
+
const RedirectHandler = {
|
|
60
|
+
// Need to use named function and not fat arrow
|
|
61
|
+
// expression so that we can use the bound value
|
|
62
|
+
// of "this"
|
|
63
|
+
handleBeforeRedirect(options, response) {
|
|
64
|
+
if (!this.allowedRedirects.includes(options.url.host)) {
|
|
65
|
+
log.debug(`
|
|
66
|
+
${options.url.host} does match any of the allowed redirects
|
|
67
|
+
in ${JSON.stringify(this.allowedRedirects)}, so auth credentials
|
|
68
|
+
will not be forwarded. If provider specifies a port number, ensure
|
|
69
|
+
that allowed redirect specifies the port number.
|
|
70
|
+
`);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (redirectCodes.has(response.statusCode)) {
|
|
74
|
+
/* eslint-disable no-param-reassign */
|
|
75
|
+
options.url.username = this.username;
|
|
76
|
+
options.url.password = this.password;
|
|
77
|
+
/* eslint-enable no-param-reassign */
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
const boundHandleBeforeRedirect = RedirectHandler.handleBeforeRedirect.bind(this);
|
|
82
|
+
this.gotOptions.hooks = {
|
|
83
|
+
beforeRedirect: [
|
|
84
|
+
boundHandleBeforeRedirect,
|
|
85
|
+
],
|
|
86
|
+
};
|
|
55
87
|
}
|
|
56
88
|
async downloadTLSCertificate() {
|
|
57
89
|
if (!this.certificateUri || this.certificate !== undefined)
|
|
@@ -59,7 +91,8 @@ class HttpProviderClient {
|
|
|
59
91
|
try {
|
|
60
92
|
const s3Params = parseS3Uri(this.certificateUri);
|
|
61
93
|
this.certificate = await getTextObject(s3Params.Bucket, s3Params.Key);
|
|
62
|
-
this.gotOptions.
|
|
94
|
+
this.gotOptions.https = this.gotOptions.https || {};
|
|
95
|
+
this.gotOptions.https.certificateAuthority = this.certificate;
|
|
63
96
|
}
|
|
64
97
|
catch (error) {
|
|
65
98
|
throw new errors.RemoteResourceError(`Failed to fetch CA certificate: ${error}`);
|
|
@@ -173,9 +206,10 @@ class HttpProviderClient {
|
|
|
173
206
|
/**
|
|
174
207
|
* Download the remote file to a given s3 location
|
|
175
208
|
*
|
|
176
|
-
* @param {
|
|
177
|
-
* @param {string}
|
|
178
|
-
* @param {string}
|
|
209
|
+
* @param {Object} params
|
|
210
|
+
* @param {string} params.fileRemotePath - the full path to the remote file to be fetched
|
|
211
|
+
* @param {string} params.destinationBucket - destination s3 bucket of the file
|
|
212
|
+
* @param {string} params.destinationKey - destination s3 key of the file
|
|
179
213
|
* @returns {Promise.<{ s3uri: string, etag: string }>} an object containing
|
|
180
214
|
* the S3 URI and ETag of the destination file
|
|
181
215
|
*/
|
|
@@ -201,12 +235,10 @@ class HttpProviderClient {
|
|
|
201
235
|
log.info(`HEAD failed for ${remoteUrl} with error: ${error}.`);
|
|
202
236
|
}
|
|
203
237
|
const contentType = headers['content-type'] || lookupMimeType(destinationKey);
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
const { ETag: etag } = await promiseS3Upload({
|
|
238
|
+
const uploadStream = got.stream(remoteUrl, this.gotOptions);
|
|
239
|
+
const { ETag: etag } = await streamS3Upload(uploadStream, {
|
|
207
240
|
Bucket: destinationBucket,
|
|
208
241
|
Key: destinationKey,
|
|
209
|
-
Body: pass,
|
|
210
242
|
ContentType: contentType,
|
|
211
243
|
});
|
|
212
244
|
log.info('Uploading to s3 is complete (http)', s3uri);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HttpProviderClient.js","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACzB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC9B,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AACrC,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"HttpProviderClient.js","sourceRoot":"","sources":["src/HttpProviderClient.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACzB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC9B,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AACrC,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACvC,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AACzC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;AAC3B,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AAC9C,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAEtC,MAAM,EACJ,UAAU,EACV,aAAa,EACb,UAAU,EACV,cAAc,GACf,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AACtC,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC3C,MAAM,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;AACrD,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,0BAA0B,CAAC,CAAC;AACzD,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAE1C,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEtD,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,EAAE;IAC5B,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO;IAEhD,MAAM,IAAI,SAAS,CAAC,gDAAgD,IAAI,EAAE,CAAC,CAAC;AAC9E,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEnE,MAAM,kBAAkB;IACtB,YAAY,cAAc;QACxB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC,eAAe,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC,cAAc,CAAC;QACpD,IAAI,cAAc,CAAC,QAAQ,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE;YACvD,MAAM,IAAI,cAAc,CAAC,2EAA2E,CAAC,CAAC;SACvG;QACD,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC;QAE1C,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3E,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACnE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEjD,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE;YAC3B,IAAI,CAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC5D,IAAI,CAAC,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SAC7D;aAAM;YACL,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;SAC9C;QAED,IAAI,CAAC,UAAU,CAAC,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;QAE5C,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5D,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE5D,MAAM,eAAe,GAAG;YACtB,+CAA+C;YAC/C,gDAAgD;YAChD,YAAY;YACZ,oBAAoB,CAAC,OAAO,EAAE,QAAQ;gBACpC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBACrD,GAAG,CAAC,KAAK,CAAC;cACN,OAAO,CAAC,GAAG,CAAC,IAAI;iBACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC;;;WAG3C,CAAC,CAAC;oBACH,OAAO;iBACR;gBAED,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE;oBAC1C,sCAAsC;oBACtC,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACrC,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oBACrC,qCAAqC;iBACtC;YACH,CAAC;SACF,CAAC;QACF,MAAM,yBAAyB,GAAG,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClF,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG;YACtB,cAAc,EAAE;gBACd,yBAAyB;aAC1B;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO;QACnE,IAAI;YACF,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,WAAW,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YACtE,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC;SAC/D;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,IAAI,MAAM,CAAC,mBAAmB,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;SAClF;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,IAAI;QACb,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEpC,uDAAuD;QACvD,4BAA4B;QAC5B,MAAM,iBAAiB,GAAG,gCAAgC,CAAC;QAC3D,MAAM,0BAA0B,GAAG,MAAM,CAAC;QAE1C,MAAM,CAAC,GAAG,IAAI,OAAO,CACnB,QAAQ,CAAC;YACP,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;SACL,CAAC,CACH,CAAC;QAEF,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE;YAC/D,CAAC,CAAC,UAAU,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;SAC1D;QACD,IAAI,IAAI,CAAC,eAAe,EAAE;YACxB,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;SACzC;QACD,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC,CAAC,cAAc,GAAG,EAAE,CAAC;QACtB,CAAC,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC3B,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;QACxB,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;QACf,MAAM,KAAK,GAAG,EAAE,CAAC;QAEjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,CAAC,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE;gBAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3D,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBACrB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;oBAChC,IAAI,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAEhD,OAAO,KAAK,KAAK,IAAI,EAAE;wBACrB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC5B,yDAAyD;wBACzD,MAAM,IAAI,GAAG,UAAU;6BACpB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;6BACjB,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC;6BACvC,SAAS,EAAE,CAAC;wBACf,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC3B,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBAC7C;gBACH,CAAC,CAAC,CAAC;gBAEH,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE,CACxB,MAAM,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAElE,CAAC,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACzC,IAAI,YAAY,GAAG,EAAE,CAAC;gBACtB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBAC5B,YAAY,IAAI,KAAK,CAAC;gBACxB,CAAC,CAAC,CAAC;gBAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,mBAAmB,CACxC,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,GAAG,6BAA6B,QAAQ,CAAC,UAAU,EAAE,CAC3F,CAAC;oBACF,GAAG,CAAC,OAAO,GAAG,YAAY,CAAC;oBAC3B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,CACxC,MAAM,CAAC,IAAI,MAAM,CAAC,mBAAmB,CAAC,qBAAqB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE5F,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;gBAChC,MAAM,YAAY,GAAG,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAC;gBACnG,YAAY,CAAC,OAAO,GAAG,SAAS,CAAC;gBACjC,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;YAEH,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAM;QACnB,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QACzC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEpC,MAAM,SAAS,GAAG,QAAQ,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,UAAU;SACjB,CAAC,CAAC;QAEH,GAAG,CAAC,IAAI,CAAC,eAAe,SAAS,OAAO,SAAS,EAAE,CAAC,CAAC;QACrD,IAAI;YACF,MAAM,SAAS,CAAC,QAAQ,CAAC,CACvB,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,EACtC,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAChC,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE;gBAC/E,MAAM,OAAO,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,8CAA8C,CAAC;gBACtF,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;aACxC;;gBAAM,MAAM,KAAK,CAAC;SACpB;QACD,GAAG,CAAC,IAAI,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QAE/C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,IAAI,CAAC,MAAM;QACf,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;QACrE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,QAAQ,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QAC5D,GAAG,CAAC,IAAI,CAAC,QAAQ,SAAS,OAAO,KAAK,EAAE,CAAC,CAAC;QAE1C,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI;YACF,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAChE,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;SAChC;QAAC,OAAO,KAAK,EAAE;YACd,GAAG,CAAC,IAAI,CAAC,mBAAmB,SAAS,gBAAgB,KAAK,GAAG,CAAC,CAAC;SAChE;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;QAE9E,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5D,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,cAAc,CACzC,YAAY,EACZ;YACE,MAAM,EAAE,iBAAiB;YACzB,GAAG,EAAE,cAAc;YACnB,WAAW,EAAE,WAAW;SACzB,CACF,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QACtD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,OAAO,KAAI,CAAC;IAElB,KAAK,CAAC,GAAG,KAAI,CAAC;CAEf;AAED,MAAM,CAAC,OAAO,GAAG,kBAAkB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "9.
|
|
3
|
+
"version": "9.7.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=12.18.0"
|
|
@@ -38,12 +38,13 @@
|
|
|
38
38
|
"author": "Cumulus Authors",
|
|
39
39
|
"license": "Apache-2.0",
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@cumulus/aws-client": "9.
|
|
42
|
-
"@cumulus/common": "9.
|
|
43
|
-
"@cumulus/db": "9.
|
|
44
|
-
"@cumulus/errors": "9.
|
|
45
|
-
"@cumulus/
|
|
46
|
-
"@cumulus/
|
|
41
|
+
"@cumulus/aws-client": "9.7.0",
|
|
42
|
+
"@cumulus/common": "9.7.0",
|
|
43
|
+
"@cumulus/db": "9.7.0",
|
|
44
|
+
"@cumulus/errors": "9.7.0",
|
|
45
|
+
"@cumulus/logger": "9.7.0",
|
|
46
|
+
"@cumulus/message": "9.7.0",
|
|
47
|
+
"@cumulus/sftp-client": "9.7.0",
|
|
47
48
|
"aws-sdk": "^2.585.0",
|
|
48
49
|
"cksum": "^1.3.0",
|
|
49
50
|
"delay": "^4.3.0",
|
|
@@ -60,10 +61,10 @@
|
|
|
60
61
|
"tough-cookie": "^4.0.0"
|
|
61
62
|
},
|
|
62
63
|
"devDependencies": {
|
|
63
|
-
"@cumulus/checksum": "9.
|
|
64
|
-
"@cumulus/cmrjs": "9.
|
|
65
|
-
"@cumulus/test-data": "9.
|
|
66
|
-
"@cumulus/types": "9.
|
|
64
|
+
"@cumulus/checksum": "9.7.0",
|
|
65
|
+
"@cumulus/cmrjs": "9.7.0",
|
|
66
|
+
"@cumulus/test-data": "9.7.0",
|
|
67
|
+
"@cumulus/types": "9.7.0"
|
|
67
68
|
},
|
|
68
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "08fb2c1d9113e7be465316f950d4ac9272bd3141"
|
|
69
70
|
}
|
package/queue.d.ts
CHANGED
|
@@ -43,7 +43,7 @@ export function enqueueParsePdrMessage({ collection, parentExecutionArn, parsePd
|
|
|
43
43
|
* @param {Object} [params.additionalCustomMeta] - additional object to merge into meta object
|
|
44
44
|
* @returns {Promise} - resolves when the message has been enqueued
|
|
45
45
|
*/
|
|
46
|
-
export function enqueueGranuleIngestMessage({ collection,
|
|
46
|
+
export function enqueueGranuleIngestMessage({ collection, granules, granuleIngestWorkflow, parentExecutionArn, pdr, provider, stack, systemBucket, queueUrl, executionNamePrefix, additionalCustomMeta, }: {
|
|
47
47
|
granule: Object;
|
|
48
48
|
queueUrl: string;
|
|
49
49
|
granuleIngestMessageTemplateUri: string;
|
package/queue.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;;GAgBG;AACH;IAb0B,GAAG,EAAlB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,0BAA0B,EAAzC,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,kBAAkB,EAAjC,MAAM;IACU,mBAAmB,EAAnC,MAAM;IAEU,oBAAoB,EAApC,MAAM;iBA+ChB;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH;IAd0B,OAAO,EAAtB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,+BAA+B,EAA9C,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,GAAG,EAAlB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IACU,mBAAmB,EAAnC,MAAM;IAEU,oBAAoB,EAApC,MAAM;
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;;GAgBG;AACH;IAb0B,GAAG,EAAlB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,0BAA0B,EAAzC,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,kBAAkB,EAAjC,MAAM;IACU,mBAAmB,EAAnC,MAAM;IAEU,oBAAoB,EAApC,MAAM;iBA+ChB;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH;IAd0B,OAAO,EAAtB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,+BAA+B,EAA9C,MAAM;IAES,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,GAAG,EAAlB,MAAM;IACS,kBAAkB,EAAjC,MAAM;IACU,mBAAmB,EAAnC,MAAM;IAEU,oBAAoB,EAApC,MAAM;iBAoDhB;AAGD;;;;;;;;;;;;;;GAcG;AACH;IAX0B,QAAQ,EAAvB,MAAM;IACS,aAAa,EAA5B,MAAM;IACU,QAAQ,EAAxB,MAAM;IACS,QAAQ,EAAvB,MAAM;IACS,UAAU,EAAzB,MAAM;IAES,kBAAkB,EAAjC,MAAM;IACU,mBAAmB,EAAnC,MAAM;iBAqDhB"}
|
package/queue.js
CHANGED
|
@@ -64,14 +64,10 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
|
|
|
64
64
|
* @param {Object} [params.additionalCustomMeta] - additional object to merge into meta object
|
|
65
65
|
* @returns {Promise} - resolves when the message has been enqueued
|
|
66
66
|
*/
|
|
67
|
-
async function enqueueGranuleIngestMessage({ collection,
|
|
67
|
+
async function enqueueGranuleIngestMessage({ collection, granules, granuleIngestWorkflow, parentExecutionArn, pdr, provider, stack, systemBucket, queueUrl, executionNamePrefix, additionalCustomMeta = {}, }) {
|
|
68
68
|
const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
|
|
69
69
|
const { arn: ingestGranuleArn } = await getJsonS3Object(systemBucket, getWorkflowFileKey(stack, granuleIngestWorkflow));
|
|
70
|
-
const payload = {
|
|
71
|
-
granules: [
|
|
72
|
-
granule,
|
|
73
|
-
],
|
|
74
|
-
};
|
|
70
|
+
const payload = { granules };
|
|
75
71
|
const workflow = {
|
|
76
72
|
name: granuleIngestWorkflow,
|
|
77
73
|
arn: ingestGranuleArn,
|
package/queue.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAChD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,gBAAgB;QACtB,GAAG,EAAE,WAAW;KACjB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,MAAM,CAAC,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;AAE/D;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,2BAA2B,CAAC,EACzC,UAAU,EACV,
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["src/queue.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAE9D,MAAM,EAAE,6BAA6B,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;AAErE,MAAM,EACJ,kBAAkB,EAClB,WAAW,GACZ,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,UAAU,EACV,kBAAkB,EAClB,gBAAgB,EAChB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAChD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAC5C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,gBAAgB;QACtB,GAAG,EAAE,WAAW;KACjB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,MAAM,CAAC,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;AAE/D;;;;;;;;;;;;;;;;;GAiBG;AACH,KAAK,UAAU,2BAA2B,CAAC,EACzC,UAAU,EACV,QAAQ,EACR,qBAAqB,EACrB,kBAAkB,EAClB,GAAG,EACH,QAAQ,EACR,KAAK,EACL,YAAY,EACZ,QAAQ,EACR,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,MAAM,eAAe,CACrD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,qBAAqB,CAAC,CACjD,CAAC;IAEF,MAAM,OAAO,GAAG,EAAE,QAAQ,EAAE,CAAC;IAE7B,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,qBAAqB;QAC3B,GAAG,EAAE,gBAAgB;KACtB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;QACD,mBAAmB;KACpB,CAAC,CAAC;IAEH,IAAI,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IAEhC,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;AAElE;;;;;;;;;;;;;;GAcG;AACH,KAAK,UAAU,sBAAsB,CAAC,EACpC,kBAAkB,EAClB,KAAK,EACL,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,mBAAmB,EACnB,oBAAoB,GAAG,EAAE,GAC1B;IACC,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,MAAM,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,MAAM,eAAe,CACtD,YAAY,EACZ,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC,CACpC,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,GAAG,aAAa;KACjB,CAAC;IAEF,MAAM,wBAAwB,GAAG;QAC/B,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,iBAAiB;KACvB,CAAC;IAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;QAC5C,eAAe;QACf,kBAAkB;QAClB,OAAO;QACP,QAAQ;QACR,QAAQ,EAAE,wBAAwB;QAClC,mBAAmB;QACnB,UAAU,EAAE;YACV,GAAG,oBAAoB;YACvB,UAAU;YACV,QAAQ;SACT;KACF,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,iBAAiB,CAC3B,OAAO,CAAC,YAAY,CAAC,aAAa,EAClC,OAAO,CAAC,YAAY,CAAC,cAAc,CACpC,CAAC;IAEF,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAExC,OAAO,GAAG,CAAC;AACb,CAAC;AACD,OAAO,CAAC,sBAAsB,GAAG,sBAAsB,CAAC"}
|
package/sqs.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { SQSMessage } from '@cumulus/aws-client/SQS';
|
|
2
|
-
export declare function
|
|
2
|
+
export declare function getS3PrefixForArchivedMessage(stackName: string, queueName: string): string;
|
|
3
|
+
export declare function getS3KeyForArchivedMessage(stackName: string, messageId: string, queueName: string): string;
|
|
3
4
|
/**
|
|
4
5
|
* Archives incoming SQS Message into S3
|
|
5
6
|
*
|
|
@@ -11,8 +12,9 @@ export declare function archiveSqsMessageToS3(queueUrl: string, message: SQSMess
|
|
|
11
12
|
/**
|
|
12
13
|
* Deletes archived SQS Message from S3
|
|
13
14
|
*
|
|
14
|
-
* @param {
|
|
15
|
+
* @param {string} messageId - SQS message ID
|
|
16
|
+
* @param {string} queueUrl - SQS queue URL
|
|
15
17
|
* @returns {undefined}
|
|
16
18
|
*/
|
|
17
|
-
export declare function deleteArchivedMessageFromS3(messageId: string): Promise<void>;
|
|
19
|
+
export declare function deleteArchivedMessageFromS3(messageId: string, queueUrl: string): Promise<void>;
|
|
18
20
|
//# sourceMappingURL=sqs.d.ts.map
|
package/sqs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqs.d.ts","sourceRoot":"","sources":["src/sqs.ts"],"names":[],"mappings":"AAGA,OAAO,
|
|
1
|
+
{"version":3,"file":"sqs.d.ts","sourceRoot":"","sources":["src/sqs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAuB,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAM1E,wBAAgB,6BAA6B,CAAC,SAAS,EAAC,MAAM,EAAE,SAAS,EAAE,MAAM,UAGhF;AAED,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,UAIlB;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CAAC,QAAQ,EAAC,MAAM,EAAE,OAAO,EAAE,UAAU,iBA8B/E;AAED;;;;;;GAMG;AACH,wBAAsB,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,iBAiBpF"}
|
package/sqs.js
CHANGED
|
@@ -3,13 +3,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.deleteArchivedMessageFromS3 = exports.archiveSqsMessageToS3 = exports.getS3KeyForArchivedMessage = void 0;
|
|
6
|
+
exports.deleteArchivedMessageFromS3 = exports.archiveSqsMessageToS3 = exports.getS3KeyForArchivedMessage = exports.getS3PrefixForArchivedMessage = void 0;
|
|
7
7
|
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
8
|
+
const SQS_1 = require("@cumulus/aws-client/SQS");
|
|
8
9
|
const common_1 = require("@cumulus/common");
|
|
9
10
|
const S3_1 = require("@cumulus/aws-client/S3");
|
|
10
11
|
const logger = new logger_1.default({ sender: '@cumulus/ingest/sqs' });
|
|
11
|
-
function
|
|
12
|
-
const
|
|
12
|
+
function getS3PrefixForArchivedMessage(stackName, queueName) {
|
|
13
|
+
const prefix = `${stackName}/archived-incoming-messages/${queueName}/`;
|
|
14
|
+
return prefix;
|
|
15
|
+
}
|
|
16
|
+
exports.getS3PrefixForArchivedMessage = getS3PrefixForArchivedMessage;
|
|
17
|
+
function getS3KeyForArchivedMessage(stackName, messageId, queueName) {
|
|
18
|
+
const key = `${getS3PrefixForArchivedMessage(stackName, queueName)}${messageId}`;
|
|
13
19
|
return key;
|
|
14
20
|
}
|
|
15
21
|
exports.getS3KeyForArchivedMessage = getS3KeyForArchivedMessage;
|
|
@@ -24,11 +30,15 @@ async function archiveSqsMessageToS3(queueUrl, message) {
|
|
|
24
30
|
const bucket = common_1.envUtils.getRequiredEnvVar('system_bucket', process.env);
|
|
25
31
|
const stackName = common_1.envUtils.getRequiredEnvVar('stackName', process.env);
|
|
26
32
|
if (!message.MessageId) {
|
|
27
|
-
const error = new Error(`MessageId on message ${message} required but not found.`);
|
|
33
|
+
const error = new Error(`MessageId on message ${JSON.stringify(message)} required but not found.`);
|
|
28
34
|
logger.error(error);
|
|
29
35
|
throw error;
|
|
30
36
|
}
|
|
31
|
-
const
|
|
37
|
+
const queueName = SQS_1.getQueueNameFromUrl(queueUrl);
|
|
38
|
+
if (!queueName) {
|
|
39
|
+
throw new Error(`Unable to determine queueName from ${queueUrl}`);
|
|
40
|
+
}
|
|
41
|
+
const key = getS3KeyForArchivedMessage(stackName, message.MessageId, queueName);
|
|
32
42
|
const body = message.Body;
|
|
33
43
|
logger.info(`Archiving message ${message.MessageId} from queue ${queueUrl}`);
|
|
34
44
|
try {
|
|
@@ -37,7 +47,7 @@ async function archiveSqsMessageToS3(queueUrl, message) {
|
|
|
37
47
|
Key: key,
|
|
38
48
|
Body: body,
|
|
39
49
|
});
|
|
40
|
-
logger.debug(`Archived ${message.MessageId} from queue`);
|
|
50
|
+
logger.debug(`Archived ${message.MessageId} from queue with key ${key}`);
|
|
41
51
|
}
|
|
42
52
|
catch (error) {
|
|
43
53
|
logger.error(`Could not write to bucket. ${error}`);
|
|
@@ -48,13 +58,18 @@ exports.archiveSqsMessageToS3 = archiveSqsMessageToS3;
|
|
|
48
58
|
/**
|
|
49
59
|
* Deletes archived SQS Message from S3
|
|
50
60
|
*
|
|
51
|
-
* @param {
|
|
61
|
+
* @param {string} messageId - SQS message ID
|
|
62
|
+
* @param {string} queueUrl - SQS queue URL
|
|
52
63
|
* @returns {undefined}
|
|
53
64
|
*/
|
|
54
|
-
async function deleteArchivedMessageFromS3(messageId) {
|
|
65
|
+
async function deleteArchivedMessageFromS3(messageId, queueUrl) {
|
|
55
66
|
const bucket = common_1.envUtils.getRequiredEnvVar('system_bucket', process.env);
|
|
56
67
|
const stackName = common_1.envUtils.getRequiredEnvVar('stackName', process.env);
|
|
57
|
-
const
|
|
68
|
+
const queueName = SQS_1.getQueueNameFromUrl(queueUrl);
|
|
69
|
+
if (!queueName) {
|
|
70
|
+
throw new Error(`Unable to determine queueName from ${queueUrl}`);
|
|
71
|
+
}
|
|
72
|
+
const key = getS3KeyForArchivedMessage(stackName, messageId, queueName);
|
|
58
73
|
try {
|
|
59
74
|
await S3_1.deleteS3Object(bucket, key);
|
|
60
75
|
logger.info(`Deleted archived message ${messageId} from S3 at ${bucket}/${key}`);
|
package/sqs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqs.js","sourceRoot":"","sources":["src/sqs.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;;;;;;AAEb,6DAAqC;
|
|
1
|
+
{"version":3,"file":"sqs.js","sourceRoot":"","sources":["src/sqs.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;;;;;;AAEb,6DAAqC;AACrC,iDAA0E;AAC1E,4CAA2C;AAC3C,+CAAqE;AAErE,MAAM,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;AAE7D,SAAgB,6BAA6B,CAAC,SAAgB,EAAE,SAAiB;IAC/E,MAAM,MAAM,GAAG,GAAG,SAAS,+BAA+B,SAAS,GAAG,CAAC;IACvE,OAAO,MAAM,CAAC;AAChB,CAAC;AAHD,sEAGC;AAED,SAAgB,0BAA0B,CACxC,SAAiB,EACjB,SAAiB,EACjB,SAAiB;IAEjB,MAAM,GAAG,GAAG,GAAG,6BAA6B,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,SAAS,EAAE,CAAC;IACjF,OAAO,GAAG,CAAC;AACb,CAAC;AAPD,gEAOC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,qBAAqB,CAAC,QAAe,EAAE,OAAmB;IAC9E,MAAM,MAAM,GAAG,iBAAQ,CAAC,iBAAiB,CAAC,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IACxE,MAAM,SAAS,GAAG,iBAAQ,CAAC,iBAAiB,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAEvE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;QACtB,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;QACnG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpB,MAAM,KAAK,CAAC;KACb;IAED,MAAM,SAAS,GAAG,yBAAmB,CAAC,QAAQ,CAAC,CAAC;IAEhD,IAAI,CAAC,SAAS,EAAE;QACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;KACnE;IAED,MAAM,GAAG,GAAG,0BAA0B,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAChF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,MAAM,CAAC,IAAI,CAAC,qBAAqB,OAAO,CAAC,SAAS,eAAe,QAAQ,EAAE,CAAC,CAAC;IAC7E,IAAI;QACF,MAAM,gBAAW,CAAC;YAChB,MAAM,EAAE,MAAM;YACd,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,SAAS,wBAAwB,GAAG,EAAE,CAAC,CAAC;KAC1E;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,CAAC,KAAK,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;QACpD,MAAM,KAAK,CAAC;KACb;AACH,CAAC;AA9BD,sDA8BC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,2BAA2B,CAAC,SAAiB,EAAE,QAAgB;IACnF,MAAM,MAAM,GAAG,iBAAQ,CAAC,iBAAiB,CAAC,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IACxE,MAAM,SAAS,GAAG,iBAAQ,CAAC,iBAAiB,CAAC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IACvE,MAAM,SAAS,GAAG,yBAAmB,CAAC,QAAQ,CAAC,CAAC;IAEhD,IAAI,CAAC,SAAS,EAAE;QACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;KACnE;IAED,MAAM,GAAG,GAAG,0BAA0B,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACxE,IAAI;QACF,MAAM,mBAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,4BAA4B,SAAS,eAAe,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;KAClF;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,CAAC,KAAK,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;KACb;AACH,CAAC;AAjBD,kEAiBC"}
|
|
@@ -4,7 +4,7 @@ const fs = require('fs');
|
|
|
4
4
|
const https = require('https');
|
|
5
5
|
const isIp = require('is-ip');
|
|
6
6
|
const { basename } = require('path');
|
|
7
|
-
const {
|
|
7
|
+
const { pipeline } = require('stream');
|
|
8
8
|
const Crawler = require('simplecrawler');
|
|
9
9
|
const got = require('got');
|
|
10
10
|
const { CookieJar } = require('tough-cookie');
|
|
@@ -14,7 +14,7 @@ const {
|
|
|
14
14
|
buildS3Uri,
|
|
15
15
|
getTextObject,
|
|
16
16
|
parseS3Uri,
|
|
17
|
-
|
|
17
|
+
streamS3Upload,
|
|
18
18
|
} = require('@cumulus/aws-client/S3');
|
|
19
19
|
const log = require('@cumulus/common/log');
|
|
20
20
|
const isValidHostname = require('is-valid-hostname');
|
|
@@ -29,6 +29,8 @@ const validateHost = (host) => {
|
|
|
29
29
|
throw new TypeError(`provider.host is not a valid hostname or IP: ${host}`);
|
|
30
30
|
};
|
|
31
31
|
|
|
32
|
+
const redirectCodes = new Set([300, 301, 302, 303, 304, 307, 308]);
|
|
33
|
+
|
|
32
34
|
class HttpProviderClient {
|
|
33
35
|
constructor(providerConfig) {
|
|
34
36
|
this.providerConfig = providerConfig;
|
|
@@ -42,6 +44,11 @@ class HttpProviderClient {
|
|
|
42
44
|
throw new ReferenceError('Found providerConfig.username, but providerConfig.password is not defined');
|
|
43
45
|
}
|
|
44
46
|
this.encrypted = providerConfig.encrypted;
|
|
47
|
+
|
|
48
|
+
this.defaultRedirect = this.port ? `${this.host}:${this.port}` : this.host;
|
|
49
|
+
this.allowedRedirects = this.providerConfig.allowedRedirects || [];
|
|
50
|
+
this.allowedRedirects.push(this.defaultRedirect);
|
|
51
|
+
|
|
45
52
|
this.endpoint = buildURL({
|
|
46
53
|
protocol: this.protocol,
|
|
47
54
|
host: this.host,
|
|
@@ -62,6 +69,36 @@ class HttpProviderClient {
|
|
|
62
69
|
|
|
63
70
|
if (this.username) this.gotOptions.username = this.username;
|
|
64
71
|
if (this.password) this.gotOptions.password = this.password;
|
|
72
|
+
|
|
73
|
+
const RedirectHandler = {
|
|
74
|
+
// Need to use named function and not fat arrow
|
|
75
|
+
// expression so that we can use the bound value
|
|
76
|
+
// of "this"
|
|
77
|
+
handleBeforeRedirect(options, response) {
|
|
78
|
+
if (!this.allowedRedirects.includes(options.url.host)) {
|
|
79
|
+
log.debug(`
|
|
80
|
+
${options.url.host} does match any of the allowed redirects
|
|
81
|
+
in ${JSON.stringify(this.allowedRedirects)}, so auth credentials
|
|
82
|
+
will not be forwarded. If provider specifies a port number, ensure
|
|
83
|
+
that allowed redirect specifies the port number.
|
|
84
|
+
`);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (redirectCodes.has(response.statusCode)) {
|
|
89
|
+
/* eslint-disable no-param-reassign */
|
|
90
|
+
options.url.username = this.username;
|
|
91
|
+
options.url.password = this.password;
|
|
92
|
+
/* eslint-enable no-param-reassign */
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
const boundHandleBeforeRedirect = RedirectHandler.handleBeforeRedirect.bind(this);
|
|
97
|
+
this.gotOptions.hooks = {
|
|
98
|
+
beforeRedirect: [
|
|
99
|
+
boundHandleBeforeRedirect,
|
|
100
|
+
],
|
|
101
|
+
};
|
|
65
102
|
}
|
|
66
103
|
|
|
67
104
|
async downloadTLSCertificate() {
|
|
@@ -69,7 +106,8 @@ class HttpProviderClient {
|
|
|
69
106
|
try {
|
|
70
107
|
const s3Params = parseS3Uri(this.certificateUri);
|
|
71
108
|
this.certificate = await getTextObject(s3Params.Bucket, s3Params.Key);
|
|
72
|
-
this.gotOptions.
|
|
109
|
+
this.gotOptions.https = this.gotOptions.https || {};
|
|
110
|
+
this.gotOptions.https.certificateAuthority = this.certificate;
|
|
73
111
|
} catch (error) {
|
|
74
112
|
throw new errors.RemoteResourceError(`Failed to fetch CA certificate: ${error}`);
|
|
75
113
|
}
|
|
@@ -206,9 +244,10 @@ class HttpProviderClient {
|
|
|
206
244
|
/**
|
|
207
245
|
* Download the remote file to a given s3 location
|
|
208
246
|
*
|
|
209
|
-
* @param {
|
|
210
|
-
* @param {string}
|
|
211
|
-
* @param {string}
|
|
247
|
+
* @param {Object} params
|
|
248
|
+
* @param {string} params.fileRemotePath - the full path to the remote file to be fetched
|
|
249
|
+
* @param {string} params.destinationBucket - destination s3 bucket of the file
|
|
250
|
+
* @param {string} params.destinationKey - destination s3 key of the file
|
|
212
251
|
* @returns {Promise.<{ s3uri: string, etag: string }>} an object containing
|
|
213
252
|
* the S3 URI and ETag of the destination file
|
|
214
253
|
*/
|
|
@@ -236,14 +275,15 @@ class HttpProviderClient {
|
|
|
236
275
|
}
|
|
237
276
|
const contentType = headers['content-type'] || lookupMimeType(destinationKey);
|
|
238
277
|
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
278
|
+
const uploadStream = got.stream(remoteUrl, this.gotOptions);
|
|
279
|
+
const { ETag: etag } = await streamS3Upload(
|
|
280
|
+
uploadStream,
|
|
281
|
+
{
|
|
282
|
+
Bucket: destinationBucket,
|
|
283
|
+
Key: destinationKey,
|
|
284
|
+
ContentType: contentType,
|
|
285
|
+
}
|
|
286
|
+
);
|
|
247
287
|
|
|
248
288
|
log.info('Uploading to s3 is complete (http)', s3uri);
|
|
249
289
|
return { s3uri, etag };
|
package/src/queue.js
CHANGED
|
@@ -95,7 +95,7 @@ module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;
|
|
|
95
95
|
*/
|
|
96
96
|
async function enqueueGranuleIngestMessage({
|
|
97
97
|
collection,
|
|
98
|
-
|
|
98
|
+
granules,
|
|
99
99
|
granuleIngestWorkflow,
|
|
100
100
|
parentExecutionArn,
|
|
101
101
|
pdr,
|
|
@@ -112,11 +112,7 @@ async function enqueueGranuleIngestMessage({
|
|
|
112
112
|
getWorkflowFileKey(stack, granuleIngestWorkflow)
|
|
113
113
|
);
|
|
114
114
|
|
|
115
|
-
const payload = {
|
|
116
|
-
granules: [
|
|
117
|
-
granule,
|
|
118
|
-
],
|
|
119
|
-
};
|
|
115
|
+
const payload = { granules };
|
|
120
116
|
|
|
121
117
|
const workflow = {
|
|
122
118
|
name: granuleIngestWorkflow,
|
package/src/sqs.ts
CHANGED
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
import Logger from '@cumulus/logger';
|
|
4
|
-
import { SQSMessage } from '@cumulus/aws-client/SQS';
|
|
4
|
+
import { getQueueNameFromUrl, SQSMessage } from '@cumulus/aws-client/SQS';
|
|
5
5
|
import { envUtils } from '@cumulus/common';
|
|
6
6
|
import { s3PutObject, deleteS3Object } from '@cumulus/aws-client/S3';
|
|
7
7
|
|
|
8
8
|
const logger = new Logger({ sender: '@cumulus/ingest/sqs' });
|
|
9
9
|
|
|
10
|
-
export function
|
|
11
|
-
const
|
|
10
|
+
export function getS3PrefixForArchivedMessage(stackName:string, queueName: string) {
|
|
11
|
+
const prefix = `${stackName}/archived-incoming-messages/${queueName}/`;
|
|
12
|
+
return prefix;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function getS3KeyForArchivedMessage(
|
|
16
|
+
stackName: string,
|
|
17
|
+
messageId: string,
|
|
18
|
+
queueName: string
|
|
19
|
+
) {
|
|
20
|
+
const key = `${getS3PrefixForArchivedMessage(stackName, queueName)}${messageId}`;
|
|
12
21
|
return key;
|
|
13
22
|
}
|
|
14
23
|
|
|
@@ -24,12 +33,18 @@ export async function archiveSqsMessageToS3(queueUrl:string, message: SQSMessage
|
|
|
24
33
|
const stackName = envUtils.getRequiredEnvVar('stackName', process.env);
|
|
25
34
|
|
|
26
35
|
if (!message.MessageId) {
|
|
27
|
-
const error = new Error(`MessageId on message ${message} required but not found.`);
|
|
36
|
+
const error = new Error(`MessageId on message ${JSON.stringify(message)} required but not found.`);
|
|
28
37
|
logger.error(error);
|
|
29
38
|
throw error;
|
|
30
39
|
}
|
|
31
40
|
|
|
32
|
-
const
|
|
41
|
+
const queueName = getQueueNameFromUrl(queueUrl);
|
|
42
|
+
|
|
43
|
+
if (!queueName) {
|
|
44
|
+
throw new Error(`Unable to determine queueName from ${queueUrl}`);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const key = getS3KeyForArchivedMessage(stackName, message.MessageId, queueName);
|
|
33
48
|
const body = message.Body;
|
|
34
49
|
logger.info(`Archiving message ${message.MessageId} from queue ${queueUrl}`);
|
|
35
50
|
try {
|
|
@@ -38,7 +53,7 @@ export async function archiveSqsMessageToS3(queueUrl:string, message: SQSMessage
|
|
|
38
53
|
Key: key,
|
|
39
54
|
Body: body,
|
|
40
55
|
});
|
|
41
|
-
logger.debug(`Archived ${message.MessageId} from queue`);
|
|
56
|
+
logger.debug(`Archived ${message.MessageId} from queue with key ${key}`);
|
|
42
57
|
} catch (error) {
|
|
43
58
|
logger.error(`Could not write to bucket. ${error}`);
|
|
44
59
|
throw error;
|
|
@@ -48,13 +63,20 @@ export async function archiveSqsMessageToS3(queueUrl:string, message: SQSMessage
|
|
|
48
63
|
/**
|
|
49
64
|
* Deletes archived SQS Message from S3
|
|
50
65
|
*
|
|
51
|
-
* @param {
|
|
66
|
+
* @param {string} messageId - SQS message ID
|
|
67
|
+
* @param {string} queueUrl - SQS queue URL
|
|
52
68
|
* @returns {undefined}
|
|
53
69
|
*/
|
|
54
|
-
export async function deleteArchivedMessageFromS3(messageId: string) {
|
|
70
|
+
export async function deleteArchivedMessageFromS3(messageId: string, queueUrl: string) {
|
|
55
71
|
const bucket = envUtils.getRequiredEnvVar('system_bucket', process.env);
|
|
56
72
|
const stackName = envUtils.getRequiredEnvVar('stackName', process.env);
|
|
57
|
-
const
|
|
73
|
+
const queueName = getQueueNameFromUrl(queueUrl);
|
|
74
|
+
|
|
75
|
+
if (!queueName) {
|
|
76
|
+
throw new Error(`Unable to determine queueName from ${queueUrl}`);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const key = getS3KeyForArchivedMessage(stackName, messageId, queueName);
|
|
58
80
|
try {
|
|
59
81
|
await deleteS3Object(bucket, key);
|
|
60
82
|
logger.info(`Deleted archived message ${messageId} from S3 at ${bucket}/${key}`);
|
package/src/util.ts
CHANGED
|
@@ -2,12 +2,17 @@ import mime from 'mime-types';
|
|
|
2
2
|
import * as KMS from '@cumulus/aws-client/KMS';
|
|
3
3
|
import { S3KeyPairProvider } from '@cumulus/common/key-pair-provider';
|
|
4
4
|
|
|
5
|
+
import Logger from '@cumulus/logger';
|
|
6
|
+
|
|
7
|
+
const logger = new Logger({ sender: '@cumulus/ingest/util' });
|
|
8
|
+
|
|
5
9
|
export const decrypt = async (
|
|
6
10
|
ciphertext: string
|
|
7
11
|
): Promise<string | undefined> => {
|
|
8
12
|
try {
|
|
9
13
|
return await KMS.decryptBase64String(ciphertext);
|
|
10
|
-
} catch (
|
|
14
|
+
} catch (error) {
|
|
15
|
+
logger.error('Could not decrypt secret with KMS', error);
|
|
11
16
|
return S3KeyPairProvider.decrypt(ciphertext);
|
|
12
17
|
}
|
|
13
18
|
};
|
package/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["src/util.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["src/util.ts"],"names":[],"mappings":"AAQA,eAAO,MAAM,OAAO,eACN,MAAM,KACjB,OAAO,CAAC,MAAM,GAAG,SAAS,CAO5B,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE9D"}
|
package/util.js
CHANGED
|
@@ -26,11 +26,14 @@ exports.lookupMimeType = exports.decrypt = void 0;
|
|
|
26
26
|
const mime_types_1 = __importDefault(require("mime-types"));
|
|
27
27
|
const KMS = __importStar(require("@cumulus/aws-client/KMS"));
|
|
28
28
|
const key_pair_provider_1 = require("@cumulus/common/key-pair-provider");
|
|
29
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
30
|
+
const logger = new logger_1.default({ sender: '@cumulus/ingest/util' });
|
|
29
31
|
exports.decrypt = async (ciphertext) => {
|
|
30
32
|
try {
|
|
31
33
|
return await KMS.decryptBase64String(ciphertext);
|
|
32
34
|
}
|
|
33
|
-
catch (
|
|
35
|
+
catch (error) {
|
|
36
|
+
logger.error('Could not decrypt secret with KMS', error);
|
|
34
37
|
return key_pair_provider_1.S3KeyPairProvider.decrypt(ciphertext);
|
|
35
38
|
}
|
|
36
39
|
};
|
package/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4DAA8B;AAC9B,6DAA+C;AAC/C,yEAAsE;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4DAA8B;AAC9B,6DAA+C;AAC/C,yEAAsE;AAEtE,6DAAqC;AAErC,MAAM,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;AAEjD,QAAA,OAAO,GAAG,KAAK,EAC1B,UAAkB,EACW,EAAE;IAC/B,IAAI;QACF,OAAO,MAAM,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;KAClD;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;QACzD,OAAO,qCAAiB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;KAC9C;AACH,CAAC,CAAC;AAEF;;;;;GAKG;AACH,SAAgB,cAAc,CAAC,GAAW;IACxC,OAAO,oBAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC;AACvC,CAAC;AAFD,wCAEC"}
|