@cumulus/ingest 18.3.1 → 18.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/SftpProviderClient.d.ts +59 -20
  2. package/SftpProviderClient.d.ts.map +1 -1
  3. package/SftpProviderClient.js +30 -12
  4. package/SftpProviderClient.js.map +1 -1
  5. package/package.json +13 -13
  6. package/recursion.d.ts +1 -1
  7. package/recursion.d.ts.map +1 -1
  8. package/src/SftpProviderClient.js +31 -12
  9. package/src/recursion.ts +1 -1
  10. package/tsconfig.tsbuildinfo +1 -1
package/SftpProviderClient.d.ts CHANGED
@@ -1,51 +1,90 @@
1
1
  export = SftpProviderClient;
2
+ /**
3
+ * @typedef {import('./recursion').RecursionFile} RecursionFile
4
+ */
2
5
  declare class SftpProviderClient {
3
- constructor(providerConfig: any);
4
- providerConfig: any;
5
- plaintextUsername: any;
6
- plaintextPassword: any;
7
- plaintextPrivateKey: any;
6
+ /**
7
+ *
8
+ * @param {Object} providerConfig
9
+ * @param {string} providerConfig.username
10
+ * @param {string} providerConfig.password
11
+ * @param {string} providerConfig.plaintextProviderKey
12
+ * @param {string} providerConfig.host
13
+ * @param {string} providerConfig.privateKey
14
+ * @param {string} providerConfig.cmKeyId
15
+ */
16
+ constructor(providerConfig: {
17
+ username: string;
18
+ password: string;
19
+ plaintextProviderKey: string;
20
+ host: string;
21
+ privateKey: string;
22
+ cmKeyId: string;
23
+ });
24
+ providerConfig: {
25
+ username: string;
26
+ password: string;
27
+ plaintextProviderKey: string;
28
+ host: string;
29
+ privateKey: string;
30
+ cmKeyId: string;
31
+ };
32
+ plaintextUsername: string | undefined;
33
+ plaintextPassword: string | undefined;
34
+ plaintextPrivateKey: string | undefined;
8
35
  connected: boolean;
9
36
  privateSftpClient: SftpClient | undefined;
10
37
  connect(): Promise<void>;
11
38
  end(): Promise<void>;
12
- get host(): any;
13
- getUsername(): Promise<any>;
14
- getPassword(): Promise<any>;
39
+ get host(): string;
40
+ getUsername(): Promise<string | undefined>;
41
+ getPassword(): Promise<string | undefined>;
15
42
  getPrivateKey(): Promise<string | undefined>;
16
- plaintextProviderKey: string | Promise<string | undefined> | undefined;
17
- getSftpClient(): SftpClient | undefined;
43
+ plaintextProviderKey: string | undefined;
44
+ /** @private
45
+ * @returns {SftpClient}
46
+ */
47
+ private getSftpClient;
18
48
  /**
19
49
  * Download a remote file to disk
20
50
  * @param {Object} params - parameter object
21
51
  * @param {string} params.remotePath - the full path to the remote file to be fetched
22
52
  * @param {string} params.localPath - the full local destination file path
23
- * @returns {Promise<string>} - the path that the file was saved to
53
+ * @returns {Promise<void>} - the path that the file was saved to
24
54
  */
25
55
  download(params: {
26
56
  remotePath: string;
27
57
  localPath: string;
28
- }): Promise<string>;
58
+ }): Promise<void>;
29
59
  /**
30
60
  * List all files from a given endpoint
31
61
  *
32
62
  * @param {string} path - the path to list
33
- * @returns {Promise}
63
+ * @returns {Promise<Array<Pick<RecursionFile, "name">>>}
34
64
  */
35
- list(path: string): Promise<any>;
65
+ list(path: string): Promise<Array<Pick<RecursionFile, "name">>>;
36
66
  /**
37
67
  * Transfer the remote file to a given s3 location
38
68
  *
39
- * @param {string} remotePath - the full path to the remote file to be fetched
40
- * @param {string} bucket - destination s3 bucket of the file
41
- * @param {string} key - destination s3 key of the file
42
- * @returns {Promise.<{ s3uri: string, etag: string }>} an object containing
69
+ * @param {Object} params
70
+ * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
71
+ * @param {string} params.destinationBucket - destination s3 bucket of the file
72
+ * @param {string} params.destinationKey - destination s3 key of the file
73
+ * @returns {Promise<{ s3uri: string, etag?: string }>} an object containing
43
74
  * the S3 URI and ETag of the destination file
44
75
  */
45
- sync(params: any): Promise<{
76
+ sync(params: {
77
+ fileRemotePath: string;
78
+ destinationBucket: string;
79
+ destinationKey: string;
80
+ }): Promise<{
46
81
  s3uri: string;
47
- etag: string;
82
+ etag?: string;
48
83
  }>;
49
84
  }
85
+ declare namespace SftpProviderClient {
86
+ export { RecursionFile };
87
+ }
50
88
  import { SftpClient } from "@cumulus/sftp-client";
89
+ type RecursionFile = import('./recursion').RecursionFile;
51
90
  //# sourceMappingURL=SftpProviderClient.d.ts.map
package/SftpProviderClient.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"SftpProviderClient.d.ts","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":";AAYA;IACE,iCAWC;IAVC,oBAAoC;IAGlC,uBAAgD;IAChD,uBAAgD;IAChD,yBAA8D;IAGhE,mBAAsB;IACtB,0CAAkC;IAGpC,yBAcC;IAED,qBAMC;IAGD,gBAEC;IAGD,4BAQC;IAGD,4BAQC;IAGD,6CAoBC;IAPK,uEAA+D;IAUrE,wCAMC;IAED;;;;;;OAMG;IACH;QAJ0B,UAAU,EAAzB,MAAM;QACS,SAAS,EAAxB,MAAM;QACJ,QAAQ,MAAM,CAAC,CAK3B;IAED;;;;;OAKG;IACH,WAHW,MAAM,gBAYhB;IAED;;;;;;;;OAQG;IACH,mBAHa,QAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAQrD;CACF"}
1
+ {"version":3,"file":"SftpProviderClient.d.ts","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":";AAeA;;GAEG;AACH;IACE;;;;;;;;;OASG;IACH;QAPkC,QAAQ,EAA/B,MAAM;QACiB,QAAQ,EAA/B,MAAM;QACiB,oBAAoB,EAA3C,MAAM;QACiB,IAAI,EAA3B,MAAM;QACiB,UAAU,EAAjC,MAAM;QACiB,OAAO,EAA9B,MAAM;OAahB;IAVC;kBARS,MAAM;kBACN,MAAM;8BACN,MAAM;cACN,MAAM;oBACN,MAAM;iBACN,MAAM;MAGqB;IAGlC,sCAAgD;IAChD,sCAAgD;IAChD,wCAA8D;IAGhE,mBAAsB;IACtB,0CAAkC;IAGpC,yBAcC;IAED,qBAMC;IAGD,mBAEC;IAGD,2CAQC;IAGD,2CAQC;IAGD,6CAoBC;IAPK,yCAAqE;IAS3E;;OAEG;IACH,sBAMC;IAED;;;;;;OAMG;IACH;QAJ0B,UAAU,EAAzB,MAAM;QACS,SAAS,EAAxB,MAAM;QACJ,QAAQ,IAAI,CAAC,CAKzB;IAED;;;;;OAKG;IACH,WAHW,MAAM,GACJ,QAAQ,MAAM,KAAK,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAWvD;IAED;;;;;;;;;OASG;IACH;QAN0B,cAAc,EAA7B,MAAM;QACS,iBAAiB,EAAhC,MAAM;QACS,cAAc,EAA7B,MAAM;QACJ,QAAQ;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAQrD;CACF;;;;;qBA5JY,OAAO,aAAa,EAAE,aAAa"}
package/SftpProviderClient.js CHANGED
@@ -1,3 +1,4 @@
1
+ //@ts-check
1
2
  'use strict';
2
3
  const get = require('lodash/get');
3
4
  const KMS = require('@cumulus/aws-client/KMS');
@@ -6,9 +7,23 @@ const omit = require('lodash/omit');
6
7
  const S3 = require('@cumulus/aws-client/S3');
7
8
  const { SftpClient } = require('@cumulus/sftp-client');
8
9
  const isNil = require('lodash/isNil');
10
+ const { getRequiredEnvVar } = require('@cumulus/common/env');
9
11
  const { recursion } = require('./recursion');
10
12
  const { decrypt } = require('./util');
13
+ /**
14
+ * @typedef {import('./recursion').RecursionFile} RecursionFile
15
+ */
11
16
  class SftpProviderClient {
17
+ /**
18
+ *
19
+ * @param {Object} providerConfig
20
+ * @param {string} providerConfig.username
21
+ * @param {string} providerConfig.password
22
+ * @param {string} providerConfig.plaintextProviderKey
23
+ * @param {string} providerConfig.host
24
+ * @param {string} providerConfig.privateKey
25
+ * @param {string} providerConfig.cmKeyId
26
+ */
12
27
  constructor(providerConfig) {
13
28
  this.providerConfig = providerConfig;
14
29
  if (get(providerConfig, 'encrypted', false) === false) {
@@ -29,12 +44,12 @@ class SftpProviderClient {
29
44
  password: await this.getPassword(),
30
45
  privateKey: await this.getPrivateKey(),
31
46
  });
32
- await this.privateSftpClient.connect(this.clientOptions);
47
+ await this.privateSftpClient.connect();
33
48
  this.connected = true;
34
49
  }
35
50
  async end() {
36
51
  if (this.connected) {
37
- await this.privateSftpClient.end();
52
+ await this.getSftpClient().end();
38
53
  this.connected = false;
39
54
  }
40
55
  }
@@ -67,10 +82,10 @@ class SftpProviderClient {
67
82
  if (isNil(this.plaintextProviderKey)) {
68
83
  // we are assuming that the specified private key is in the S3 crypto
69
84
  // directory
70
- const fetchedKey = await S3.getTextObject(process.env.system_bucket, `${process.env.stackName}/crypto/${this.providerConfig.privateKey}`);
85
+ const fetchedKey = await S3.getTextObject(getRequiredEnvVar('system_bucket', process.env), `${process.env.stackName}/crypto/${this.providerConfig.privateKey}`);
71
86
  if (this.providerConfig.cmKeyId) {
72
87
  // we are using AWS KMS and the privateKey is encrypted
73
- this.plaintextProviderKey = KMS.decryptBase64String(fetchedKey);
88
+ this.plaintextProviderKey = await KMS.decryptBase64String(fetchedKey);
74
89
  }
75
90
  else {
76
91
  this.plaintextProviderKey = fetchedKey;
@@ -78,9 +93,11 @@ class SftpProviderClient {
78
93
  }
79
94
  return this.plaintextProviderKey;
80
95
  }
81
- /* @private */
96
+ /** @private
97
+ * @returns {SftpClient}
98
+ */
82
99
  getSftpClient() {
83
- if (!this.connected) {
100
+ if (!this.connected || !this.privateSftpClient) {
84
101
  throw new Error('Client not connected');
85
102
  }
86
103
  return this.privateSftpClient;
@@ -90,7 +107,7 @@ class SftpProviderClient {
90
107
  * @param {Object} params - parameter object
91
108
  * @param {string} params.remotePath - the full path to the remote file to be fetched
92
109
  * @param {string} params.localPath - the full local destination file path
93
- * @returns {Promise<string>} - the path that the file was saved to
110
+ * @returns {Promise<void>} - the path that the file was saved to
94
111
  */
95
112
  async download(params) {
96
113
  const { remotePath, localPath } = params;
@@ -100,7 +117,7 @@ class SftpProviderClient {
100
117
  * List all files from a given endpoint
101
118
  *
102
119
  * @param {string} path - the path to list
103
- * @returns {Promise}
120
+ * @returns {Promise<Array<Pick<RecursionFile, "name">>>}
104
121
  */
105
122
  async list(path) {
106
123
  const sftpClient = this.getSftpClient();
@@ -114,10 +131,11 @@ class SftpProviderClient {
114
131
  /**
115
132
  * Transfer the remote file to a given s3 location
116
133
  *
117
- * @param {string} remotePath - the full path to the remote file to be fetched
118
- * @param {string} bucket - destination s3 bucket of the file
119
- * @param {string} key - destination s3 key of the file
120
- * @returns {Promise.<{ s3uri: string, etag: string }>} an object containing
134
+ * @param {Object} params
135
+ * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
136
+ * @param {string} params.destinationBucket - destination s3 bucket of the file
137
+ * @param {string} params.destinationKey - destination s3 key of the file
138
+ * @returns {Promise<{ s3uri: string, etag?: string }>} an object containing
121
139
  * the S3 URI and ETag of the destination file
122
140
  */
123
141
  async sync(params) {
package/SftpProviderClient.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"SftpProviderClient.js","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAClC,MAAM,GAAG,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AACpC,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC7C,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AACvD,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AACtC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAC7C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEtC,MAAM,kBAAkB;IACtB,YAAY,cAAc;QACxB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QAErC,IAAI,GAAG,CAAC,cAAc,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,KAAK,EAAE;YACrD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC,oBAAoB,CAAC;SAChE;QAED,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,IAAI,CAAC,iBAAiB,GAAG,IAAI,UAAU,CAAC;YACtC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,CAAC;YAC1C,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,MAAM,IAAI,CAAC,aAAa,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC;YAEnC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;SACxB;IACH,CAAC;IAED,cAAc;IACd,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;IAClC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,aAAa;QACjB,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC;YAAE,OAAO,SAAS,CAAC;QAE5D,IAAI,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE;YACpC,qEAAqE;YACrE,YAAY;YACZ,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,aAAa,CACvC,OAAO,CAAC,GAAG,CAAC,aAAa,EACzB,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,WAAW,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,CACpE,CAAC;YAEF,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;gBAC/B,uDAAuD;gBACvD,IAAI,CAAC,oBAAoB,GAAG,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;aACjE;iBAAM;gBACL,IAAI,CAAC,oBAAoB,GAAG,UAAU,CAAC;aACxC;SACF;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAED,cAAc;IACd,aAAa;QACX,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;SACzC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAM;QACnB,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QACzC,OAAO,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,IAAI;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,wBAAwB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAElF,uEAAuE;QACvE,6CAA6C;QAC7C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,IAAI,CAAC,MAAM;QACf,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,cAAc,CAAC;QAClC,OAAO,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IACtE,CAAC;CACF;AAED,MAAM,CAAC,OAAO,GAAG,kBAAkB,CAAC"}
1
+ {"version":3,"file":"SftpProviderClient.js","sourceRoot":"","sources":["src/SftpProviderClient.js"],"names":[],"mappings":"AAAA,WAAW;AAEX,YAAY,CAAC;AAEb,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAClC,MAAM,GAAG,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;AAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AACpC,MAAM,EAAE,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAC7C,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AACvD,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;AACtC,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC7D,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAC7C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEtC;;GAEG;AACH,MAAM,kBAAkB;IACtB;;;;;;;;;OASG;IACH,YAAY,cAAc;QACxB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QAErC,IAAI,GAAG,CAAC,cAAc,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,KAAK,EAAE;YACrD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,iBAAiB,GAAG,cAAc,CAAC,QAAQ,CAAC;YACjD,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC,oBAAoB,CAAC;SAChE;QAED,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,IAAI,CAAC,iBAAiB,GAAG,IAAI,UAAU,CAAC;YACtC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,CAAC;YAC1C,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;YAClC,UAAU,EAAE,MAAM,IAAI,CAAC,aAAa,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC;QAEvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,EAAE,CAAC;YAEjC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;SACxB;IACH,CAAC;IAED,cAAc;IACd,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;IAClC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,WAAW;QACf,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1D,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACjC,IAAI,CAAC,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;SACtE;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED,cAAc;IACd,KAAK,CAAC,aAAa;QACjB,IAAI,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC;YAAE,OAAO,SAAS,CAAC;QAE5D,IAAI,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE;YACpC,qEAAqE;YACrE,YAAY;YACZ,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,aAAa,CACvC,iBAAiB,CAAC,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,EAC/C,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,WAAW,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,CACpE,CAAC;YAEF,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE;gBAC/B,uDAAuD;gBACvD,IAAI,CAAC,oBAAoB,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;aACvE;iBAAM;gBACL,IAAI,CAAC,oBAAoB,GAAG,UAAU,CAAC;aACxC;SACF;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC9C,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;SACzC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAM;QACnB,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QACzC,OAAO,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,IAAI;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,wBAAwB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAElF,uEAAuE;QACvE,6CAA6C;QAC7C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,IAAI,CAAC,MAAM;QACf,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,cAAc,CAAC;QAClC,OAAO,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IACtE,CAAC;CACF;AAED,MAAM,CAAC,OAAO,GAAG,kBAAkB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cumulus/ingest",
3
- "version": "18.3.1",
3
+ "version": "18.3.3",
4
4
  "description": "Ingest utilities",
5
5
  "engines": {
6
6
  "node": ">=20.12.2"
@@ -41,13 +41,13 @@
41
41
  "author": "Cumulus Authors",
42
42
  "license": "Apache-2.0",
43
43
  "dependencies": {
44
- "@cumulus/aws-client": "18.3.1",
45
- "@cumulus/common": "18.3.1",
46
- "@cumulus/db": "18.3.1",
47
- "@cumulus/errors": "18.3.1",
48
- "@cumulus/logger": "18.3.1",
49
- "@cumulus/message": "18.3.1",
50
- "@cumulus/sftp-client": "18.3.1",
44
+ "@cumulus/aws-client": "18.3.3",
45
+ "@cumulus/common": "18.3.3",
46
+ "@cumulus/db": "18.3.3",
47
+ "@cumulus/errors": "18.3.3",
48
+ "@cumulus/logger": "18.3.3",
49
+ "@cumulus/message": "18.3.3",
50
+ "@cumulus/sftp-client": "18.3.3",
51
51
  "cksum": "^1.3.0",
52
52
  "encodeurl": "^1.0.2",
53
53
  "fs-extra": "^5.0.0",
@@ -61,10 +61,10 @@
61
61
  "tough-cookie": "~4.0.0"
62
62
  },
63
63
  "devDependencies": {
64
- "@cumulus/checksum": "18.3.1",
65
- "@cumulus/cmrjs": "18.3.1",
66
- "@cumulus/test-data": "18.3.1",
67
- "@cumulus/types": "18.3.1"
64
+ "@cumulus/checksum": "18.3.3",
65
+ "@cumulus/cmrjs": "18.3.3",
66
+ "@cumulus/test-data": "18.3.3",
67
+ "@cumulus/types": "18.3.3"
68
68
  },
69
- "gitHead": "0393f90c6401ef0c524068e4636c1dcc389020b8"
69
+ "gitHead": "2750792b9913f7bbad429d4b942ef4e59c5ef3d2"
70
70
  }
package/recursion.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  export interface RecursionFile {
2
- type: number;
2
+ type: number | string;
3
3
  name: string;
4
4
  }
5
5
  /**
package/recursion.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"recursion.d.ts","sourceRoot":"","sources":["src/recursion.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAA;CACb;AA2CD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,SAAS,CAAC,CAAC,SAAS,aAAa,EACrD,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,EAClC,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,CAAC,EAAE,CAAC,CAcd"}
1
+ {"version":3,"file":"recursion.d.ts","sourceRoot":"","sources":["src/recursion.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAA;CACb;AA2CD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,SAAS,CAAC,CAAC,SAAS,aAAa,EACrD,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC,EAClC,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,CAAC,EAAE,CAAC,CAcd"}
package/src/SftpProviderClient.js CHANGED
@@ -1,3 +1,5 @@
1
+ //@ts-check
2
+
1
3
  'use strict';
2
4
 
3
5
  const get = require('lodash/get');
@@ -7,10 +9,24 @@ const omit = require('lodash/omit');
7
9
  const S3 = require('@cumulus/aws-client/S3');
8
10
  const { SftpClient } = require('@cumulus/sftp-client');
9
11
  const isNil = require('lodash/isNil');
12
+ const { getRequiredEnvVar } = require('@cumulus/common/env');
10
13
  const { recursion } = require('./recursion');
11
14
  const { decrypt } = require('./util');
12
15
 
16
+ /**
17
+ * @typedef {import('./recursion').RecursionFile} RecursionFile
18
+ */
13
19
  class SftpProviderClient {
20
+ /**
21
+ *
22
+ * @param {Object} providerConfig
23
+ * @param {string} providerConfig.username
24
+ * @param {string} providerConfig.password
25
+ * @param {string} providerConfig.plaintextProviderKey
26
+ * @param {string} providerConfig.host
27
+ * @param {string} providerConfig.privateKey
28
+ * @param {string} providerConfig.cmKeyId
29
+ */
14
30
  constructor(providerConfig) {
15
31
  this.providerConfig = providerConfig;
16
32
 
@@ -35,14 +51,14 @@ class SftpProviderClient {
35
51
  privateKey: await this.getPrivateKey(),
36
52
  });
37
53
 
38
- await this.privateSftpClient.connect(this.clientOptions);
54
+ await this.privateSftpClient.connect();
39
55
 
40
56
  this.connected = true;
41
57
  }
42
58
 
43
59
  async end() {
44
60
  if (this.connected) {
45
- await this.privateSftpClient.end();
61
+ await this.getSftpClient().end();
46
62
 
47
63
  this.connected = false;
48
64
  }
@@ -83,13 +99,13 @@ class SftpProviderClient {
83
99
  // we are assuming that the specified private key is in the S3 crypto
84
100
  // directory
85
101
  const fetchedKey = await S3.getTextObject(
86
- process.env.system_bucket,
102
+ getRequiredEnvVar('system_bucket', process.env),
87
103
  `${process.env.stackName}/crypto/${this.providerConfig.privateKey}`
88
104
  );
89
105
 
90
106
  if (this.providerConfig.cmKeyId) {
91
107
  // we are using AWS KMS and the privateKey is encrypted
92
- this.plaintextProviderKey = KMS.decryptBase64String(fetchedKey);
108
+ this.plaintextProviderKey = await KMS.decryptBase64String(fetchedKey);
93
109
  } else {
94
110
  this.plaintextProviderKey = fetchedKey;
95
111
  }
@@ -98,9 +114,11 @@ class SftpProviderClient {
98
114
  return this.plaintextProviderKey;
99
115
  }
100
116
 
101
- /* @private */
117
+ /** @private
118
+ * @returns {SftpClient}
119
+ */
102
120
  getSftpClient() {
103
- if (!this.connected) {
121
+ if (!this.connected || !this.privateSftpClient) {
104
122
  throw new Error('Client not connected');
105
123
  }
106
124
 
@@ -112,7 +130,7 @@ class SftpProviderClient {
112
130
  * @param {Object} params - parameter object
113
131
  * @param {string} params.remotePath - the full path to the remote file to be fetched
114
132
  * @param {string} params.localPath - the full local destination file path
115
- * @returns {Promise<string>} - the path that the file was saved to
133
+ * @returns {Promise<void>} - the path that the file was saved to
116
134
  */
117
135
  async download(params) {
118
136
  const { remotePath, localPath } = params;
@@ -123,7 +141,7 @@ class SftpProviderClient {
123
141
  * List all files from a given endpoint
124
142
  *
125
143
  * @param {string} path - the path to list
126
- * @returns {Promise}
144
+ * @returns {Promise<Array<Pick<RecursionFile, "name">>>}
127
145
  */
128
146
  async list(path) {
129
147
  const sftpClient = this.getSftpClient();
@@ -139,10 +157,11 @@ class SftpProviderClient {
139
157
  /**
140
158
  * Transfer the remote file to a given s3 location
141
159
  *
142
- * @param {string} remotePath - the full path to the remote file to be fetched
143
- * @param {string} bucket - destination s3 bucket of the file
144
- * @param {string} key - destination s3 key of the file
145
- * @returns {Promise.<{ s3uri: string, etag: string }>} an object containing
160
+ * @param {Object} params
161
+ * @param {string} params.fileRemotePath - the full path to the remote file to be fetched
162
+ * @param {string} params.destinationBucket - destination s3 bucket of the file
163
+ * @param {string} params.destinationKey - destination s3 key of the file
164
+ * @returns {Promise<{ s3uri: string, etag?: string }>} an object containing
146
165
  * the S3 URI and ETag of the destination file
147
166
  */
148
167
  async sync(params) {
package/src/recursion.ts CHANGED
@@ -2,7 +2,7 @@ import path from 'path';
2
2
  import * as log from '@cumulus/common/log';
3
3
 
4
4
  export interface RecursionFile {
5
- type: number,
5
+ type: number | string,
6
6
  name: string
7
7
  }
8
8