@cumulus/ingest 19.0.0 → 19.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SftpProviderClient.d.ts +4 -0
- package/SftpProviderClient.d.ts.map +1 -1
- package/SftpProviderClient.js +6 -3
- package/SftpProviderClient.js.map +1 -1
- package/consumer.d.ts.map +1 -1
- package/consumer.js +2 -1
- package/consumer.js.map +1 -1
- package/package.json +14 -14
- package/src/SftpProviderClient.js +6 -3
- package/src/consumer.ts +2 -1
- package/tsconfig.tsbuildinfo +1 -1
package/SftpProviderClient.d.ts
CHANGED
|
@@ -50,11 +50,13 @@ declare class SftpProviderClient {
|
|
|
50
50
|
* @param {Object} params - parameter object
|
|
51
51
|
* @param {string} params.remotePath - the full path to the remote file to be fetched
|
|
52
52
|
* @param {string} params.localPath - the full local destination file path
|
|
53
|
+
* @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
|
|
53
54
|
* @returns {Promise<void>} - the path that the file was saved to
|
|
54
55
|
*/
|
|
55
56
|
download(params: {
|
|
56
57
|
remotePath: string;
|
|
57
58
|
localPath: string;
|
|
59
|
+
fastDownload: boolean;
|
|
58
60
|
}): Promise<void>;
|
|
59
61
|
/**
|
|
60
62
|
* List all files from a given endpoint
|
|
@@ -70,6 +72,7 @@ declare class SftpProviderClient {
|
|
|
70
72
|
* @param {string} params.fileRemotePath - the full path to the remote file to be fetched
|
|
71
73
|
* @param {string} params.destinationBucket - destination s3 bucket of the file
|
|
72
74
|
* @param {string} params.destinationKey - destination s3 key of the file
|
|
75
|
+
* @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
|
|
73
76
|
* @returns {Promise<{ s3uri: string, etag?: string }>} an object containing
|
|
74
77
|
* the S3 URI and ETag of the destination file
|
|
75
78
|
*/
|
|
@@ -77,6 +80,7 @@ declare class SftpProviderClient {
|
|
|
77
80
|
fileRemotePath: string;
|
|
78
81
|
destinationBucket: string;
|
|
79
82
|
destinationKey: string;
|
|
83
|
+
fastDownload: boolean;
|
|
80
84
|
}): Promise<{
|
|
81
85
|
s3uri: string;
|
|
82
86
|
etag?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SftpProviderClient.d.ts","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":";AAeA;;GAEG;AACH;IACE;;;;;;;;;OASG;IACH;QAPkC,QAAQ,EAA/B,MAAM;QACiB,QAAQ,EAA/B,MAAM;QACiB,oBAAoB,EAA3C,MAAM;QACiB,IAAI,EAA3B,MAAM;QACiB,UAAU,EAAjC,MAAM;QACiB,OAAO,EAA9B,MAAM;OAahB;IAVC;kBARS,MAAM;kBACN,MAAM;8BACN,MAAM;cACN,MAAM;oBACN,MAAM;iBACN,MAAM;MAGqB;IAGlC,sCAAgD;IAChD,sCAAgD;IAChD,wCAA8D;IAGhE,mBAAsB;IACtB,0CAAkC;IAGpC,yBAcC;IAED,qBAMC;IAGD,mBAEC;IAGD,2CAQC;IAGD,2CAQC;IAGD,6CAoBC;IAPK,yCAAqE;IAS3E;;OAEG;IACH,sBAMC;IAED
|
|
1
|
+
{"version":3,"file":"SftpProviderClient.d.ts","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":";AAeA;;GAEG;AACH;IACE;;;;;;;;;OASG;IACH;QAPkC,QAAQ,EAA/B,MAAM;QACiB,QAAQ,EAA/B,MAAM;QACiB,oBAAoB,EAA3C,MAAM;QACiB,IAAI,EAA3B,MAAM;QACiB,UAAU,EAAjC,MAAM;QACiB,OAAO,EAA9B,MAAM;OAahB;IAVC;kBARS,MAAM;kBACN,MAAM;8BACN,MAAM;cACN,MAAM;oBACN,MAAM;iBACN,MAAM;MAGqB;IAGlC,sCAAgD;IAChD,sCAAgD;IAChD,wCAA8D;IAGhE,mBAAsB;IACtB,0CAAkC;IAGpC,yBAcC;IAED,qBAMC;IAGD,mBAEC;IAGD,2CAQC;IAGD,2CAQC;IAGD,6CAoBC;IAPK,yCAAqE;IAS3E;;OAEG;IACH,sBAMC;IAED;;;;;;;OAOG;IACH;QAL0B,UAAU,EAAzB,MAAM;QACS,SAAS,EAAxB,MAAM;QACU,YAAY,EAA5B,OAAO;QACL,QAAQ,IAAI,CAAC,CAKzB;IAED;;;;;OAKG;IACH,WAHW,MAAM,GACJ,QAAQ,MAAM,KAAK,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAWvD;IAED;;;;;;;;;;OAUG;IACH;QAP0B,cAAc,EAA7B,MAAM;QACS,iBAAiB,EAAhC,MAAM;QACS,cAAc,EAA7B,MAAM;QACU,YAAY,EAA5B,OAAO;QACL,QAAQ;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CASrD;CACF;;;;;qBA/JY,OAAO,aAAa,EAAE,aAAa"}
|
package/SftpProviderClient.js
CHANGED
|
@@ -107,11 +107,12 @@ class SftpProviderClient {
|
|
|
107
107
|
* @param {Object} params - parameter object
|
|
108
108
|
* @param {string} params.remotePath - the full path to the remote file to be fetched
|
|
109
109
|
* @param {string} params.localPath - the full local destination file path
|
|
110
|
+
* @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
|
|
110
111
|
* @returns {Promise<void>} - the path that the file was saved to
|
|
111
112
|
*/
|
|
112
113
|
async download(params) {
|
|
113
|
-
const { remotePath, localPath } = params;
|
|
114
|
-
return await this.getSftpClient().download(remotePath, localPath);
|
|
114
|
+
const { remotePath, localPath, fastDownload } = params;
|
|
115
|
+
return await this.getSftpClient().download(remotePath, localPath, fastDownload);
|
|
115
116
|
}
|
|
116
117
|
/**
|
|
117
118
|
* List all files from a given endpoint
|
|
@@ -135,6 +136,7 @@ class SftpProviderClient {
|
|
|
135
136
|
* @param {string} params.fileRemotePath - the full path to the remote file to be fetched
|
|
136
137
|
* @param {string} params.destinationBucket - destination s3 bucket of the file
|
|
137
138
|
* @param {string} params.destinationKey - destination s3 key of the file
|
|
139
|
+
* @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
|
|
138
140
|
* @returns {Promise<{ s3uri: string, etag?: string }>} an object containing
|
|
139
141
|
* the S3 URI and ETag of the destination file
|
|
140
142
|
*/
|
|
@@ -142,7 +144,8 @@ class SftpProviderClient {
|
|
|
142
144
|
const remotePath = params.fileRemotePath;
|
|
143
145
|
const bucket = params.destinationBucket;
|
|
144
146
|
const key = params.destinationKey;
|
|
145
|
-
|
|
147
|
+
const syncMethod = params.fastDownload ? 'syncToS3Fast' : 'syncToS3';
|
|
148
|
+
return await this.getSftpClient()[syncMethod](remotePath, bucket, key);
|
|
146
149
|
}
|
|
147
150
|
}
|
|
148
151
|
module.exports = SftpProviderClient;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SftpProviderClient.js","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":"AAAA,WAAW;AAEX,YAAY,CAAC;AAEb,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAClC,MAAM,GAAG,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AACpC,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC7C,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AACvD,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AACtC,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC7D,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAC7C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEtC;;GAEG;AACH,MAAM,kBAAkB;IACtB;;;;;;;;;OASG;IACH,YAAY,cAAc;QACxB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QAErC,IAAI,GAAG,CAAC,cAAc,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,KAAK,EAAE;YACrD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC,oBAAoB,CAAC;SAChE;QAED,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,IAAI,CAAC,iBAAiB,GAAG,IAAI,UAAU,CAAC;YACtC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,CAAC;YAC1C,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,MAAM,IAAI,CAAC,aAAa,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC;QAEvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,EAAE,CAAC;YAEjC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;SACxB;IACH,CAAC;IAED,cAAc;IACd,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;IAClC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,aAAa;QACjB,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC;YAAE,OAAO,SAAS,CAAC;QAE5D,IAAI,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE;YACpC,qEAAqE;YACrE,YAAY;YACZ,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,aAAa,CACvC,iBAAiB,CAAC,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,EAC/C,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,WAAW,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,CACpE,CAAC;YAEF,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;gBAC/B,uDAAuD;gBACvD,IAAI,CAAC,oBAAoB,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;aACvE;iBAAM;gBACL,IAAI,CAAC,oBAAoB,GAAG,UAAU,CAAC;aACxC;SACF;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC9C,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;SACzC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED
|
|
1
|
+
{"version":3,"file":"SftpProviderClient.js","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":"AAAA,WAAW;AAEX,YAAY,CAAC;AAEb,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAClC,MAAM,GAAG,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AACpC,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC7C,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AACvD,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AACtC,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC7D,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAC7C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEtC;;GAEG;AACH,MAAM,kBAAkB;IACtB;;;;;;;;;OASG;IACH,YAAY,cAAc;QACxB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QAErC,IAAI,GAAG,CAAC,cAAc,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,KAAK,EAAE;YACrD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC,oBAAoB,CAAC;SAChE;QAED,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,IAAI,CAAC,iBAAiB,GAAG,IAAI,UAAU,CAAC;YACtC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,CAAC;YAC1C,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,MAAM,IAAI,CAAC,aAAa,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC;QAEvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,EAAE,CAAC;YAEjC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;SACxB;IACH,CAAC;IAED,cAAc;IACd,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;IAClC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,aAAa;QACjB,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC;YAAE,OAAO,SAAS,CAAC;QAE5D,IAAI,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE;YACpC,qEAAqE;YACrE,YAAY;YACZ,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,aAAa,CACvC,iBAAiB,CAAC,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,EAC/C,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,WAAW,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,CACpE,CAAC;YAEF,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;gBAC/B,uDAAuD;gBACvD,IAAI,CAAC,oBAAoB,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;aACvE;iBAAM;gBACL,IAAI,CAAC,oBAAoB,GAAG,UAAU,CAAC;aACxC;SACF;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC9C,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;SACzC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAM;QACnB,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;QACvD,OAAO,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAClF,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,IAAI;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,wBAAwB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAElF,uEAAuE;QACvE,6CAA6C;QAC7C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,IAAI,CAAC,MAAM;QACf,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,cAAc,CAAC;QAClC,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC;QACrE,OAAO,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,UAAU,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IACzE,CAAC;CACF;AAED,MAAM,CAAC,OAAO,GAAG,kBAAkB,CAAC"}
|
package/consumer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,UAAU,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAKzE,oBAAY,uBAAuB,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAG/F,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,OAAO,CAAA;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;IACjD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;gBAE/B,EACV,QAAQ,EACR,YAAgB,EAChB,SAAc,EACd,iBAAiB,EACjB,sBAA6B,GAC9B,EAAE,yBAAyB;YAUd,cAAc;YAsBd,eAAe;IAqBvB,OAAO,CAAC,EAAE,EAAE,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC;CA6B5D"}
|
package/consumer.js
CHANGED
|
@@ -29,6 +29,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
29
29
|
exports.Consumer = void 0;
|
|
30
30
|
const SQS_1 = require("@cumulus/aws-client/SQS");
|
|
31
31
|
const sqs = __importStar(require("@cumulus/aws-client/SQS"));
|
|
32
|
+
const StepFunctions_1 = require("@cumulus/aws-client/StepFunctions");
|
|
32
33
|
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
33
34
|
const log = new logger_1.default({ sender: '@cumulus/ingest/consumer' });
|
|
34
35
|
class Consumer {
|
|
@@ -50,7 +51,7 @@ class Consumer {
|
|
|
50
51
|
return 1;
|
|
51
52
|
}
|
|
52
53
|
catch (error) {
|
|
53
|
-
if (error
|
|
54
|
+
if (error instanceof StepFunctions_1.ExecutionAlreadyExists) {
|
|
54
55
|
log.debug('Deleting message for execution that already exists...');
|
|
55
56
|
await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
|
|
56
57
|
log.debug('Completed deleting message.');
|
package/consumer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consumer.js","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAyE;AACzE,6DAA+C;AAC/C,6DAAqC;AAIrC,MAAM,GAAG,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAS/D,MAAa,QAAQ;IASnB,YAAY,EACV,QAAQ,EACR,YAAY,GAAG,CAAC,EAChB,SAAS,GAAG,EAAE,EACd,iBAAiB,EACjB,sBAAsB,GAAG,IAAI,GACH;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,OAAmB,EACnB,EAA2B;QAE3B,IAAI;YACF,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,sBAAsB,EAAE;gBAC/B,MAAM,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;aAClE;YACD,OAAO,CAAC,CAAC;SACV;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,
|
|
1
|
+
{"version":3,"file":"consumer.js","sourceRoot":"","sources":["src/consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAyE;AACzE,6DAA+C;AAC/C,qEAA2E;AAC3E,6DAAqC;AAIrC,MAAM,GAAG,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAS/D,MAAa,QAAQ;IASnB,YAAY,EACV,QAAQ,EACR,YAAY,GAAG,CAAC,EAChB,SAAS,GAAG,EAAE,EACd,iBAAiB,EACjB,sBAAsB,GAAG,IAAI,GACH;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,OAAmB,EACnB,EAA2B;QAE3B,IAAI;YACF,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,sBAAsB,EAAE;gBAC/B,MAAM,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;aAClE;YACD,OAAO,CAAC,CAAC;SACV;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,KAAK,YAAY,sCAAsB,EAAE;gBAC3C,GAAG,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;gBACnE,MAAM,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;gBACjE,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;gBACzC,OAAO,CAAC,CAAC;aACV;YACD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjB,OAAO,CAAC,CAAC;SACV;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,EAA2B,EAC3B,YAAoB,EACpB,iBAAyB;QAEzB,IAAI,YAAY,GAAG,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,2EAA2E,YAAY,EAAE,CAAC,CAAC;QAElI,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAkB,EACvC,IAAI,CAAC,QAAQ,EACb,EAAE,aAAa,EAAE,YAAY,EAAE,iBAAiB,EAAE,CACnD,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YACvB,GAAG,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;YACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC9E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC7C,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAa,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;SACtE;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAA2B;QACvC,IAAI,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACrC,GAAG,CAAC,IAAI,CAAC,+BAA+B,YAAY,cAAc,CAAC,CAAC;QAEpE,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,qCAAqC;QACrC,8FAA8F;QAC9F,OAAO,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YAC3C,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,YAAY,GAAG,EAAE,EAAE;gBACrB,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBACrE,YAAY,IAAI,EAAE,CAAC;aACpB;iBAAM,IAAI,YAAY,GAAG,CAAC,EAAE;gBAC3B,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBAC/E,YAAY,IAAI,YAAY,CAAC;aAC9B;YACD,GAAG,IAAI,OAAO,CAAC;YACf,oEAAoE;YACpE,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE;gBAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,wCAAwC,CAAC,CAAC;aAC5E;SACF;QACD,oCAAoC;QAEpC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,yCAAyC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzE,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAjGD,4BAiGC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/ingest",
|
|
3
|
-
"version": "19.
|
|
3
|
+
"version": "19.1.0",
|
|
4
4
|
"description": "Ingest utilities",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=20.12.2"
|
|
@@ -41,13 +41,13 @@
|
|
|
41
41
|
"author": "Cumulus Authors",
|
|
42
42
|
"license": "Apache-2.0",
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@cumulus/aws-client": "19.
|
|
45
|
-
"@cumulus/common": "19.
|
|
46
|
-
"@cumulus/db": "19.
|
|
47
|
-
"@cumulus/errors": "19.
|
|
48
|
-
"@cumulus/logger": "19.
|
|
49
|
-
"@cumulus/message": "19.
|
|
50
|
-
"@cumulus/sftp-client": "19.
|
|
44
|
+
"@cumulus/aws-client": "19.1.0",
|
|
45
|
+
"@cumulus/common": "19.1.0",
|
|
46
|
+
"@cumulus/db": "19.1.0",
|
|
47
|
+
"@cumulus/errors": "19.1.0",
|
|
48
|
+
"@cumulus/logger": "19.1.0",
|
|
49
|
+
"@cumulus/message": "19.1.0",
|
|
50
|
+
"@cumulus/sftp-client": "19.1.0",
|
|
51
51
|
"cksum": "^1.3.0",
|
|
52
52
|
"encodeurl": "^1.0.2",
|
|
53
53
|
"fs-extra": "^5.0.0",
|
|
@@ -56,15 +56,15 @@
|
|
|
56
56
|
"jsftp": "https://github.com/jkovarik/jsftp.git#add_288",
|
|
57
57
|
"lodash": "^4.17.21",
|
|
58
58
|
"mime-types": "^2.1.22",
|
|
59
|
-
"moment": "2.
|
|
59
|
+
"moment": "^2.30.1",
|
|
60
60
|
"simplecrawler": "^1.1.9",
|
|
61
61
|
"tough-cookie": "~4.0.0"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
64
|
-
"@cumulus/checksum": "19.
|
|
65
|
-
"@cumulus/cmrjs": "19.
|
|
66
|
-
"@cumulus/test-data": "19.
|
|
67
|
-
"@cumulus/types": "19.
|
|
64
|
+
"@cumulus/checksum": "19.1.0",
|
|
65
|
+
"@cumulus/cmrjs": "19.1.0",
|
|
66
|
+
"@cumulus/test-data": "19.1.0",
|
|
67
|
+
"@cumulus/types": "19.1.0"
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "803acf8212df5203ec502c5b16dde49a20265b7e"
|
|
70
70
|
}
|
|
@@ -130,11 +130,12 @@ class SftpProviderClient {
|
|
|
130
130
|
* @param {Object} params - parameter object
|
|
131
131
|
* @param {string} params.remotePath - the full path to the remote file to be fetched
|
|
132
132
|
* @param {string} params.localPath - the full local destination file path
|
|
133
|
+
* @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
|
|
133
134
|
* @returns {Promise<void>} - the path that the file was saved to
|
|
134
135
|
*/
|
|
135
136
|
async download(params) {
|
|
136
|
-
const { remotePath, localPath } = params;
|
|
137
|
-
return await this.getSftpClient().download(remotePath, localPath);
|
|
137
|
+
const { remotePath, localPath, fastDownload } = params;
|
|
138
|
+
return await this.getSftpClient().download(remotePath, localPath, fastDownload);
|
|
138
139
|
}
|
|
139
140
|
|
|
140
141
|
/**
|
|
@@ -161,6 +162,7 @@ class SftpProviderClient {
|
|
|
161
162
|
* @param {string} params.fileRemotePath - the full path to the remote file to be fetched
|
|
162
163
|
* @param {string} params.destinationBucket - destination s3 bucket of the file
|
|
163
164
|
* @param {string} params.destinationKey - destination s3 key of the file
|
|
165
|
+
* @param {boolean} params.fastDownload - whether fast download is performed using parallel reads
|
|
164
166
|
* @returns {Promise<{ s3uri: string, etag?: string }>} an object containing
|
|
165
167
|
* the S3 URI and ETag of the destination file
|
|
166
168
|
*/
|
|
@@ -168,7 +170,8 @@ class SftpProviderClient {
|
|
|
168
170
|
const remotePath = params.fileRemotePath;
|
|
169
171
|
const bucket = params.destinationBucket;
|
|
170
172
|
const key = params.destinationKey;
|
|
171
|
-
|
|
173
|
+
const syncMethod = params.fastDownload ? 'syncToS3Fast' : 'syncToS3';
|
|
174
|
+
return await this.getSftpClient()[syncMethod](remotePath, bucket, key);
|
|
172
175
|
}
|
|
173
176
|
}
|
|
174
177
|
|
package/src/consumer.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { receiveSQSMessages, SQSMessage } from '@cumulus/aws-client/SQS';
|
|
2
2
|
import * as sqs from '@cumulus/aws-client/SQS';
|
|
3
|
+
import { ExecutionAlreadyExists } from '@cumulus/aws-client/StepFunctions';
|
|
3
4
|
import Logger from '@cumulus/logger';
|
|
4
5
|
|
|
5
6
|
export type MessageConsumerFunction = (queueUrl: string, message: SQSMessage) => Promise<void>;
|
|
@@ -49,7 +50,7 @@ export class Consumer {
|
|
|
49
50
|
}
|
|
50
51
|
return 1;
|
|
51
52
|
} catch (error) {
|
|
52
|
-
if (error
|
|
53
|
+
if (error instanceof ExecutionAlreadyExists) {
|
|
53
54
|
log.debug('Deleting message for execution that already exists...');
|
|
54
55
|
await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
|
|
55
56
|
log.debug('Completed deleting message.');
|