@cubejs-backend/base-driver 1.6.60 → 1.6.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/storage-fs/aws.fs.d.ts +17 -0
- package/dist/src/storage-fs/aws.fs.d.ts.map +1 -1
- package/dist/src/storage-fs/aws.fs.js +33 -2
- package/dist/src/storage-fs/aws.fs.js.map +1 -1
- package/dist/src/storage-fs/gcs.fs.d.ts +8 -0
- package/dist/src/storage-fs/gcs.fs.d.ts.map +1 -1
- package/dist/src/storage-fs/gcs.fs.js +19 -2
- package/dist/src/storage-fs/gcs.fs.js.map +1 -1
- package/package.json +4 -4
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
import { S3ClientConfig } from '@aws-sdk/client-s3';
|
|
2
2
|
export type S3StorageClientConfig = S3ClientConfig;
|
|
3
|
+
/**
|
|
4
|
+
* Returns a copy of the S3 client config with unusable static credentials and
|
|
5
|
+
* a blank region removed, so the AWS SDK falls back to its default resolution.
|
|
6
|
+
*
|
|
7
|
+
* When an export bucket is configured for OIDC / workload identity there are no
|
|
8
|
+
* static keys, yet some drivers still pass a `{ accessKeyId: '', secretAccessKey: '' }`
|
|
9
|
+
* object. The SDK would send those empty strings verbatim and the request fails
|
|
10
|
+
* with `AuthorizationHeaderMalformed: ... a non-empty Access Key (AKID) must be
|
|
11
|
+
* provided`. Dropping the empty credentials lets the SDK use its default provider
|
|
12
|
+
* chain (env vars, web identity token file `AWS_WEB_IDENTITY_TOKEN_FILE`, IRSA,
|
|
13
|
+
* etc.). Likewise a blank region would short-circuit region resolution from
|
|
14
|
+
* `AWS_REGION` / `AWS_DEFAULT_REGION`.
|
|
15
|
+
*
|
|
16
|
+
* Credential *providers* (functions, e.g. `fromTemporaryCredentials`) and
|
|
17
|
+
* fully-populated static credentials are left untouched.
|
|
18
|
+
*/
|
|
19
|
+
export declare function normalizeS3ClientConfig(clientOptions: S3StorageClientConfig): S3StorageClientConfig;
|
|
3
20
|
/**
|
|
4
21
|
* Returns an array of signed AWS S3 URLs of the unloaded csv files.
|
|
5
22
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"aws.fs.d.ts","sourceRoot":"","sources":["../../../src/storage-fs/aws.fs.ts"],"names":[],"mappings":"AACA,OAAO,EAAwB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAE1E,MAAM,MAAM,qBAAqB,GAAG,cAAc,CAAC;AAEnD;;GAEG;AACH,wBAAsB,0BAA0B,CAC9C,aAAa,EAAE,qBAAqB,EACpC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,EAAE,CAAC,CA6BnB"}
|
|
1
|
+
{"version":3,"file":"aws.fs.d.ts","sourceRoot":"","sources":["../../../src/storage-fs/aws.fs.ts"],"names":[],"mappings":"AACA,OAAO,EAAwB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAE1E,MAAM,MAAM,qBAAqB,GAAG,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CAAC,aAAa,EAAE,qBAAqB,GAAG,qBAAqB,CAkBnG;AAED;;GAEG;AACH,wBAAsB,0BAA0B,CAC9C,aAAa,EAAE,qBAAqB,EACpC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,EAAE,CAAC,CA6BnB"}
|
|
@@ -1,13 +1,44 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.extractUnloadedFilesFromS3 = void 0;
|
|
3
|
+
exports.extractUnloadedFilesFromS3 = exports.normalizeS3ClientConfig = void 0;
|
|
4
4
|
const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
|
|
5
5
|
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
6
|
+
/**
|
|
7
|
+
* Returns a copy of the S3 client config with unusable static credentials and
|
|
8
|
+
* a blank region removed, so the AWS SDK falls back to its default resolution.
|
|
9
|
+
*
|
|
10
|
+
* When an export bucket is configured for OIDC / workload identity there are no
|
|
11
|
+
* static keys, yet some drivers still pass a `{ accessKeyId: '', secretAccessKey: '' }`
|
|
12
|
+
* object. The SDK would send those empty strings verbatim and the request fails
|
|
13
|
+
* with `AuthorizationHeaderMalformed: ... a non-empty Access Key (AKID) must be
|
|
14
|
+
* provided`. Dropping the empty credentials lets the SDK use its default provider
|
|
15
|
+
* chain (env vars, web identity token file `AWS_WEB_IDENTITY_TOKEN_FILE`, IRSA,
|
|
16
|
+
* etc.). Likewise a blank region would short-circuit region resolution from
|
|
17
|
+
* `AWS_REGION` / `AWS_DEFAULT_REGION`.
|
|
18
|
+
*
|
|
19
|
+
* Credential *providers* (functions, e.g. `fromTemporaryCredentials`) and
|
|
20
|
+
* fully-populated static credentials are left untouched.
|
|
21
|
+
*/
|
|
22
|
+
function normalizeS3ClientConfig(clientOptions) {
|
|
23
|
+
const config = { ...clientOptions };
|
|
24
|
+
const { credentials } = config;
|
|
25
|
+
if (credentials &&
|
|
26
|
+
typeof credentials === 'object' &&
|
|
27
|
+
'accessKeyId' in credentials &&
|
|
28
|
+
(!credentials.accessKeyId || !credentials.secretAccessKey)) {
|
|
29
|
+
delete config.credentials;
|
|
30
|
+
}
|
|
31
|
+
if (typeof config.region === 'string' && config.region.trim() === '') {
|
|
32
|
+
delete config.region;
|
|
33
|
+
}
|
|
34
|
+
return config;
|
|
35
|
+
}
|
|
36
|
+
exports.normalizeS3ClientConfig = normalizeS3ClientConfig;
|
|
6
37
|
/**
|
|
7
38
|
* Returns an array of signed AWS S3 URLs of the unloaded csv files.
|
|
8
39
|
*/
|
|
9
40
|
async function extractUnloadedFilesFromS3(clientOptions, bucketName, prefix) {
|
|
10
|
-
const storage = new client_s3_1.S3(clientOptions);
|
|
41
|
+
const storage = new client_s3_1.S3(normalizeS3ClientConfig(clientOptions));
|
|
11
42
|
// It looks that different driver configurations use different formats
|
|
12
43
|
// for the bucket - some expect only names, some - full url-like names.
|
|
13
44
|
// So we unify this.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"aws.fs.js","sourceRoot":"","sources":["../../../src/storage-fs/aws.fs.ts"],"names":[],"mappings":";;;AAAA,wEAA6D;AAC7D,kDAA0E;AAI1E;;GAEG;AACI,KAAK,UAAU,0BAA0B,CAC9C,aAAoC,EACpC,UAAkB,EAClB,MAAc;IAEd,MAAM,OAAO,GAAG,IAAI,cAAE,CAAC,aAAa,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"aws.fs.js","sourceRoot":"","sources":["../../../src/storage-fs/aws.fs.ts"],"names":[],"mappings":";;;AAAA,wEAA6D;AAC7D,kDAA0E;AAI1E;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,uBAAuB,CAAC,aAAoC;IAC1E,MAAM,MAAM,GAA0B,EAAE,GAAG,aAAa,EAAE,CAAC;IAC3D,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAE/B,IACE,WAAW;QACX,OAAO,WAAW,KAAK,QAAQ;QAC/B,aAAa,IAAI,WAAW;QAC5B,CAAC,CAAC,WAAW,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,EAC1D;QACA,OAAO,MAAM,CAAC,WAAW,CAAC;KAC3B;IAED,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACpE,OAAO,MAAM,CAAC,MAAM,CAAC;KACtB;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAlBD,0DAkBC;AAED;;GAEG;AACI,KAAK,UAAU,0BAA0B,CAC9C,aAAoC,EACpC,UAAkB,EAClB,MAAc;IAEd,MAAM,OAAO,GAAG,IAAI,cAAE,CAAC,uBAAuB,CAAC,aAAa,CAAC,CAAC,CAAC;IAC/D,sEAAsE;IACtE,uEAAuE;IACvE,oBAAoB;IACpB,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IAEvD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC;QACvC,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,MAAM;KACf,CAAC,CAAC;IACH,IAAI,IAAI,EAAE;QACR,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;YAClB,OAAO,EAAE,CAAC;SACX;aAAM;YACL,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBAC/B,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;oBACnC,MAAM,EAAE,UAAU;oBAClB,GAAG,EAAE,IAAI,CAAC,GAAG;iBACd,CAAC,CAAC;gBACH,OAAO,IAAA,mCAAY,EAAC,OAAO,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7D,CAAC,CAAC,CACH,CAAC;YACF,OAAO,QAAQ,CAAC;SACjB;KACF;IAED,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;AACvF,CAAC;AAjCD,gEAiCC"}
|
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
export type GoogleStorageClientConfig = {
|
|
2
2
|
credentials: any;
|
|
3
3
|
};
|
|
4
|
+
/**
|
|
5
|
+
* Whether the provided GCS credentials are usable. Empty string / undefined
|
|
6
|
+
* (no credentials configured, e.g. OIDC / workload identity) and an empty
|
|
7
|
+
* object are treated as absent so the Google SDK falls back to Application
|
|
8
|
+
* Default Credentials (which honors `GOOGLE_APPLICATION_CREDENTIALS`, including
|
|
9
|
+
* workload-identity-federation `external_account` config files).
|
|
10
|
+
*/
|
|
11
|
+
export declare function hasGCSCredentials(credentials: any): boolean;
|
|
4
12
|
/**
|
|
5
13
|
* Returns an array of signed GCS URLs of the unloaded csv files.
|
|
6
14
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gcs.fs.d.ts","sourceRoot":"","sources":["../../../src/storage-fs/gcs.fs.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,GAAG,CAAC;CAClB,CAAC;AAEF;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,yBAAyB,EACpC,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CAuBnB"}
|
|
1
|
+
{"version":3,"file":"gcs.fs.d.ts","sourceRoot":"","sources":["../../../src/storage-fs/gcs.fs.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,yBAAyB,GAAG;IACtC,WAAW,EAAE,GAAG,CAAC;CAClB,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,GAAG,GAAG,OAAO,CAQ3D;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,yBAAyB,EACpC,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CAuBnB"}
|
|
@@ -1,12 +1,29 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.extractFilesFromGCS = void 0;
|
|
3
|
+
exports.extractFilesFromGCS = exports.hasGCSCredentials = void 0;
|
|
4
4
|
const storage_1 = require("@google-cloud/storage");
|
|
5
|
+
/**
|
|
6
|
+
* Whether the provided GCS credentials are usable. Empty string / undefined
|
|
7
|
+
* (no credentials configured, e.g. OIDC / workload identity) and an empty
|
|
8
|
+
* object are treated as absent so the Google SDK falls back to Application
|
|
9
|
+
* Default Credentials (which honors `GOOGLE_APPLICATION_CREDENTIALS`, including
|
|
10
|
+
* workload-identity-federation `external_account` config files).
|
|
11
|
+
*/
|
|
12
|
+
function hasGCSCredentials(credentials) {
|
|
13
|
+
if (!credentials) {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
if (typeof credentials === 'object' && Object.keys(credentials).length === 0) {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
exports.hasGCSCredentials = hasGCSCredentials;
|
|
5
22
|
/**
|
|
6
23
|
* Returns an array of signed GCS URLs of the unloaded csv files.
|
|
7
24
|
*/
|
|
8
25
|
async function extractFilesFromGCS(gcsConfig, bucketName, tableName) {
|
|
9
|
-
const storage = new storage_1.Storage(gcsConfig.credentials
|
|
26
|
+
const storage = new storage_1.Storage(hasGCSCredentials(gcsConfig.credentials)
|
|
10
27
|
? { credentials: gcsConfig.credentials, projectId: gcsConfig.credentials.project_id }
|
|
11
28
|
: undefined);
|
|
12
29
|
const bucket = storage.bucket(bucketName);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gcs.fs.js","sourceRoot":"","sources":["../../../src/storage-fs/gcs.fs.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAMhD;;GAEG;AACI,KAAK,UAAU,mBAAmB,CACvC,SAAoC,EACpC,UAAkB,EAClB,SAAiB;IAEjB,MAAM,OAAO,GAAG,IAAI,iBAAO,CACzB,SAAS,CAAC,WAAW;
|
|
1
|
+
{"version":3,"file":"gcs.fs.js","sourceRoot":"","sources":["../../../src/storage-fs/gcs.fs.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAMhD;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,WAAgB;IAChD,IAAI,CAAC,WAAW,EAAE;QAChB,OAAO,KAAK,CAAC;KACd;IACD,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5E,OAAO,KAAK,CAAC;KACd;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AARD,8CAQC;AAED;;GAEG;AACI,KAAK,UAAU,mBAAmB,CACvC,SAAoC,EACpC,UAAkB,EAClB,SAAiB;IAEjB,MAAM,OAAO,GAAG,IAAI,iBAAO,CACzB,iBAAiB,CAAC,SAAS,CAAC,WAAW,CAAC;QACtC,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE;QACrF,CAAC,CAAC,SAAS,CACd,CAAC;IAEF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC1C,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,GAAG,EAAE,CAAC,CAAC;IAEnE,IAAI,KAAK,CAAC,MAAM,EAAE;QAChB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1D,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;gBACpC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;aACzD,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC,CAAC;QAEJ,OAAO,QAAQ,CAAC;KACjB;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;KAC/D;AACH,CAAC;AA3BD,kDA2BC"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@cubejs-backend/base-driver",
|
|
3
3
|
"description": "Cube.js Base Driver",
|
|
4
4
|
"author": "Cube Dev, Inc.",
|
|
5
|
-
"version": "1.6.
|
|
5
|
+
"version": "1.6.62",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "https://github.com/cube-js/cube.git",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"@aws-sdk/s3-request-presigner": "^3.49.0",
|
|
34
34
|
"@azure/identity": "^4.4.1",
|
|
35
35
|
"@azure/storage-blob": "^12.9.0",
|
|
36
|
-
"@cubejs-backend/shared": "1.6.
|
|
36
|
+
"@cubejs-backend/shared": "1.6.62",
|
|
37
37
|
"@google-cloud/storage": "^7.13.0"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
|
-
"@cubejs-backend/linter": "1.6.
|
|
40
|
+
"@cubejs-backend/linter": "1.6.62",
|
|
41
41
|
"@types/jest": "^29",
|
|
42
42
|
"@types/node": "^20",
|
|
43
43
|
"jest": "^29",
|
|
@@ -51,5 +51,5 @@
|
|
|
51
51
|
"publishConfig": {
|
|
52
52
|
"access": "public"
|
|
53
53
|
},
|
|
54
|
-
"gitHead": "
|
|
54
|
+
"gitHead": "e317395654cc5e80abd155c6b5a4fcad5f387af7"
|
|
55
55
|
}
|