@deepnote/database-integrations 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +385 -241
- package/dist/index.d.cts +1784 -70
- package/dist/index.d.ts +1784 -70
- package/dist/index.js +381 -243
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -60,6 +60,332 @@ function isFederatedAuthMetadata(metadata) {
|
|
|
60
60
|
return isFederatedAuthMethod(metadata.authMethod);
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
+
//#endregion
|
|
64
|
+
//#region src/database-integration-metadata-schemas.ts
|
|
65
|
+
const athenaMetadataSchema = zod.z.object({
|
|
66
|
+
access_key_id: zod.z.string(),
|
|
67
|
+
region: zod.z.string(),
|
|
68
|
+
s3_output_path: zod.z.string(),
|
|
69
|
+
secret_access_key: zod.z.string(),
|
|
70
|
+
workgroup: zod.z.string().optional()
|
|
71
|
+
});
|
|
72
|
+
const bigqueryMetadataSchema = zod.z.discriminatedUnion("authMethod", [zod.z.object({
|
|
73
|
+
authMethod: zod.z.literal(BigQueryAuthMethods.ServiceAccount),
|
|
74
|
+
service_account: zod.z.string()
|
|
75
|
+
}), zod.z.object({
|
|
76
|
+
authMethod: zod.z.literal(BigQueryAuthMethods.GoogleOauth),
|
|
77
|
+
project: zod.z.string(),
|
|
78
|
+
clientId: zod.z.string(),
|
|
79
|
+
clientSecret: zod.z.string()
|
|
80
|
+
})]);
|
|
81
|
+
const clickhouseMetadataSchema = zod.z.object({
|
|
82
|
+
host: zod.z.string(),
|
|
83
|
+
user: zod.z.string(),
|
|
84
|
+
password: zod.z.string().optional(),
|
|
85
|
+
database: zod.z.string(),
|
|
86
|
+
port: zod.z.string().optional(),
|
|
87
|
+
sshEnabled: zod.z.boolean().optional(),
|
|
88
|
+
sshHost: zod.z.string().optional(),
|
|
89
|
+
sshPort: zod.z.string().optional(),
|
|
90
|
+
sshUser: zod.z.string().optional(),
|
|
91
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
92
|
+
caCertificateName: zod.z.string().optional(),
|
|
93
|
+
caCertificateText: zod.z.string().optional()
|
|
94
|
+
});
|
|
95
|
+
const databricksMetadataSchema = zod.z.object({
|
|
96
|
+
host: zod.z.string(),
|
|
97
|
+
httpPath: zod.z.string(),
|
|
98
|
+
token: zod.z.string(),
|
|
99
|
+
port: zod.z.string(),
|
|
100
|
+
schema: zod.z.string().optional(),
|
|
101
|
+
catalog: zod.z.string().optional(),
|
|
102
|
+
sshEnabled: zod.z.boolean().optional(),
|
|
103
|
+
sshHost: zod.z.string().optional(),
|
|
104
|
+
sshPort: zod.z.string().optional(),
|
|
105
|
+
sshUser: zod.z.string().optional()
|
|
106
|
+
});
|
|
107
|
+
const dremioMetadataSchema = zod.z.object({
|
|
108
|
+
schema: zod.z.string(),
|
|
109
|
+
host: zod.z.string(),
|
|
110
|
+
port: zod.z.string(),
|
|
111
|
+
token: zod.z.string(),
|
|
112
|
+
sshEnabled: zod.z.boolean().optional(),
|
|
113
|
+
sshHost: zod.z.string().optional(),
|
|
114
|
+
sshPort: zod.z.string().optional(),
|
|
115
|
+
sshUser: zod.z.string().optional()
|
|
116
|
+
});
|
|
117
|
+
const mongodbMetadataSchema = zod.z.object({
|
|
118
|
+
connection_string: zod.z.string(),
|
|
119
|
+
rawConnectionString: zod.z.string().optional(),
|
|
120
|
+
prefix: zod.z.string().optional(),
|
|
121
|
+
host: zod.z.string().optional(),
|
|
122
|
+
port: zod.z.string().optional(),
|
|
123
|
+
user: zod.z.string().optional(),
|
|
124
|
+
password: zod.z.string().optional(),
|
|
125
|
+
database: zod.z.string().optional(),
|
|
126
|
+
options: zod.z.string().optional(),
|
|
127
|
+
sshEnabled: zod.z.boolean().optional(),
|
|
128
|
+
sshHost: zod.z.string().optional(),
|
|
129
|
+
sshPort: zod.z.string().optional(),
|
|
130
|
+
sshUser: zod.z.string().optional(),
|
|
131
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
132
|
+
caCertificateName: zod.z.string().optional(),
|
|
133
|
+
caCertificateText: zod.z.string().optional()
|
|
134
|
+
});
|
|
135
|
+
const pandasDataframeMetadataSchema = zod.z.object({});
|
|
136
|
+
const commonRedshiftMetadataSchema = zod.z.object({
|
|
137
|
+
database: zod.z.string(),
|
|
138
|
+
host: zod.z.string(),
|
|
139
|
+
port: zod.z.string().optional(),
|
|
140
|
+
sshEnabled: zod.z.boolean().optional(),
|
|
141
|
+
sshHost: zod.z.string().optional(),
|
|
142
|
+
sshPort: zod.z.string().optional(),
|
|
143
|
+
sshUser: zod.z.string().optional(),
|
|
144
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
145
|
+
caCertificateName: zod.z.string().optional(),
|
|
146
|
+
caCertificateText: zod.z.string().optional()
|
|
147
|
+
});
|
|
148
|
+
const redshiftMetadataSchema = zod.z.discriminatedUnion("authMethod", [
|
|
149
|
+
commonRedshiftMetadataSchema.extend({
|
|
150
|
+
authMethod: zod.z.literal(DatabaseAuthMethods.UsernameAndPassword),
|
|
151
|
+
user: zod.z.string(),
|
|
152
|
+
password: zod.z.string()
|
|
153
|
+
}),
|
|
154
|
+
commonRedshiftMetadataSchema.extend({
|
|
155
|
+
authMethod: zod.z.literal(AwsAuthMethods.IamRole),
|
|
156
|
+
roleArn: zod.z.string(),
|
|
157
|
+
roleExternalId: zod.z.string(),
|
|
158
|
+
roleNonce: zod.z.string()
|
|
159
|
+
}),
|
|
160
|
+
commonRedshiftMetadataSchema.extend({ authMethod: zod.z.literal(DatabaseAuthMethods.IndividualCredentials) })
|
|
161
|
+
]);
|
|
162
|
+
const commonSnowflakeMetadataSchema = zod.z.object({
|
|
163
|
+
accountName: zod.z.string(),
|
|
164
|
+
warehouse: zod.z.string().optional(),
|
|
165
|
+
database: zod.z.string().optional(),
|
|
166
|
+
role: zod.z.string().optional()
|
|
167
|
+
});
|
|
168
|
+
const snowflakeMetadataSchema = zod.z.discriminatedUnion("authMethod", [
|
|
169
|
+
commonSnowflakeMetadataSchema.extend({
|
|
170
|
+
authMethod: zod.z.literal(SnowflakeAuthMethods.Password),
|
|
171
|
+
username: zod.z.string(),
|
|
172
|
+
password: zod.z.string()
|
|
173
|
+
}),
|
|
174
|
+
commonSnowflakeMetadataSchema.extend({
|
|
175
|
+
authMethod: zod.z.literal(SnowflakeAuthMethods.Okta),
|
|
176
|
+
clientId: zod.z.string(),
|
|
177
|
+
clientSecret: zod.z.string(),
|
|
178
|
+
oktaSubdomain: zod.z.string(),
|
|
179
|
+
identityProvider: zod.z.string(),
|
|
180
|
+
authorizationServer: zod.z.string()
|
|
181
|
+
}),
|
|
182
|
+
commonSnowflakeMetadataSchema.extend({
|
|
183
|
+
authMethod: zod.z.literal(SnowflakeAuthMethods.NativeSnowflake),
|
|
184
|
+
clientId: zod.z.string(),
|
|
185
|
+
clientSecret: zod.z.string()
|
|
186
|
+
}),
|
|
187
|
+
commonSnowflakeMetadataSchema.extend({
|
|
188
|
+
authMethod: zod.z.literal(SnowflakeAuthMethods.AzureAd),
|
|
189
|
+
clientId: zod.z.string(),
|
|
190
|
+
clientSecret: zod.z.string(),
|
|
191
|
+
resource: zod.z.string(),
|
|
192
|
+
tenant: zod.z.string()
|
|
193
|
+
}),
|
|
194
|
+
commonSnowflakeMetadataSchema.extend({ authMethod: zod.z.literal(SnowflakeAuthMethods.KeyPair) }),
|
|
195
|
+
commonSnowflakeMetadataSchema.extend({
|
|
196
|
+
authMethod: zod.z.literal(SnowflakeAuthMethods.ServiceAccountKeyPair),
|
|
197
|
+
username: zod.z.string(),
|
|
198
|
+
privateKey: zod.z.string(),
|
|
199
|
+
privateKeyPassphrase: zod.z.string().optional()
|
|
200
|
+
})
|
|
201
|
+
]);
|
|
202
|
+
const spannerMetadataSchema = zod.z.object({
|
|
203
|
+
service_account: zod.z.string(),
|
|
204
|
+
dataBoostEnabled: zod.z.boolean(),
|
|
205
|
+
instance: zod.z.string(),
|
|
206
|
+
database: zod.z.string()
|
|
207
|
+
});
|
|
208
|
+
const trinoMetadataSchema = zod.z.object({
|
|
209
|
+
host: zod.z.string(),
|
|
210
|
+
user: zod.z.string(),
|
|
211
|
+
password: zod.z.string(),
|
|
212
|
+
database: zod.z.string(),
|
|
213
|
+
port: zod.z.string(),
|
|
214
|
+
sshEnabled: zod.z.boolean().optional(),
|
|
215
|
+
sshHost: zod.z.string().optional(),
|
|
216
|
+
sshPort: zod.z.string().optional(),
|
|
217
|
+
sshUser: zod.z.string().optional(),
|
|
218
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
219
|
+
caCertificateName: zod.z.string().optional(),
|
|
220
|
+
caCertificateText: zod.z.string().optional()
|
|
221
|
+
});
|
|
222
|
+
const commonDatabaseSchema = zod.z.object({
|
|
223
|
+
host: zod.z.string(),
|
|
224
|
+
user: zod.z.string(),
|
|
225
|
+
password: zod.z.string(),
|
|
226
|
+
database: zod.z.string(),
|
|
227
|
+
sshEnabled: zod.z.boolean().optional(),
|
|
228
|
+
sshHost: zod.z.string().optional(),
|
|
229
|
+
sshPort: zod.z.string().optional(),
|
|
230
|
+
sshUser: zod.z.string().optional()
|
|
231
|
+
});
|
|
232
|
+
const alloydbMetadataSchema = commonDatabaseSchema.extend({
|
|
233
|
+
port: zod.z.string().optional(),
|
|
234
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
235
|
+
caCertificateName: zod.z.string().optional(),
|
|
236
|
+
caCertificateText: zod.z.string().optional()
|
|
237
|
+
});
|
|
238
|
+
const mariadbMetadataSchema = commonDatabaseSchema.extend({
|
|
239
|
+
port: zod.z.string().optional(),
|
|
240
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
241
|
+
caCertificateName: zod.z.string().optional(),
|
|
242
|
+
caCertificateText: zod.z.string().optional()
|
|
243
|
+
});
|
|
244
|
+
const mindsdbMetadataSchema = commonDatabaseSchema.extend({
|
|
245
|
+
port: zod.z.string().optional(),
|
|
246
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
247
|
+
caCertificateName: zod.z.string().optional(),
|
|
248
|
+
caCertificateText: zod.z.string().optional()
|
|
249
|
+
});
|
|
250
|
+
const mysqlMetadataSchema = commonDatabaseSchema.extend({
|
|
251
|
+
port: zod.z.string().optional(),
|
|
252
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
253
|
+
caCertificateName: zod.z.string().optional(),
|
|
254
|
+
caCertificateText: zod.z.string().optional()
|
|
255
|
+
});
|
|
256
|
+
const pgsqlMetadataSchema = commonDatabaseSchema.extend({
|
|
257
|
+
port: zod.z.string().optional(),
|
|
258
|
+
sslEnabled: zod.z.boolean().optional(),
|
|
259
|
+
caCertificateName: zod.z.string().optional(),
|
|
260
|
+
caCertificateText: zod.z.string().optional()
|
|
261
|
+
});
|
|
262
|
+
const materializeMetadataSchema = pgsqlMetadataSchema.extend({ cluster: zod.z.string() });
|
|
263
|
+
const sqlServerMetadataSchema = commonDatabaseSchema.extend({ port: zod.z.string() });
|
|
264
|
+
const databaseMetadataSchemasByType = {
|
|
265
|
+
"alloydb": alloydbMetadataSchema,
|
|
266
|
+
"athena": athenaMetadataSchema,
|
|
267
|
+
"big-query": bigqueryMetadataSchema,
|
|
268
|
+
"clickhouse": clickhouseMetadataSchema,
|
|
269
|
+
"databricks": databricksMetadataSchema,
|
|
270
|
+
"dremio": dremioMetadataSchema,
|
|
271
|
+
"mariadb": mariadbMetadataSchema,
|
|
272
|
+
"materialize": materializeMetadataSchema,
|
|
273
|
+
"mindsdb": mindsdbMetadataSchema,
|
|
274
|
+
"mongodb": mongodbMetadataSchema,
|
|
275
|
+
"mysql": mysqlMetadataSchema,
|
|
276
|
+
"pandas-dataframe": pandasDataframeMetadataSchema,
|
|
277
|
+
"pgsql": pgsqlMetadataSchema,
|
|
278
|
+
"redshift": redshiftMetadataSchema,
|
|
279
|
+
"snowflake": snowflakeMetadataSchema,
|
|
280
|
+
"spanner": spannerMetadataSchema,
|
|
281
|
+
"sql-server": sqlServerMetadataSchema,
|
|
282
|
+
"trino": trinoMetadataSchema
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
//#endregion
|
|
286
|
+
//#region src/database-integration-config.ts
|
|
287
|
+
const commonIntegrationConfig = zod.default.object({
|
|
288
|
+
id: zod.default.string(),
|
|
289
|
+
name: zod.default.string()
|
|
290
|
+
});
|
|
291
|
+
const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
|
|
292
|
+
commonIntegrationConfig.extend({
|
|
293
|
+
type: zod.default.literal("alloydb"),
|
|
294
|
+
metadata: databaseMetadataSchemasByType["alloydb"],
|
|
295
|
+
federated_auth_method: zod.default.null().optional()
|
|
296
|
+
}),
|
|
297
|
+
commonIntegrationConfig.extend({
|
|
298
|
+
type: zod.default.literal("athena"),
|
|
299
|
+
metadata: databaseMetadataSchemasByType["athena"],
|
|
300
|
+
federated_auth_method: zod.default.null().optional()
|
|
301
|
+
}),
|
|
302
|
+
commonIntegrationConfig.extend({
|
|
303
|
+
type: zod.default.literal("big-query"),
|
|
304
|
+
metadata: databaseMetadataSchemasByType["big-query"],
|
|
305
|
+
federated_auth_method: zod.default.enum(["google-oauth"]).optional().nullable()
|
|
306
|
+
}),
|
|
307
|
+
commonIntegrationConfig.extend({
|
|
308
|
+
type: zod.default.literal("clickhouse"),
|
|
309
|
+
metadata: databaseMetadataSchemasByType["clickhouse"],
|
|
310
|
+
federated_auth_method: zod.default.null().optional()
|
|
311
|
+
}),
|
|
312
|
+
commonIntegrationConfig.extend({
|
|
313
|
+
type: zod.default.literal("databricks"),
|
|
314
|
+
metadata: databaseMetadataSchemasByType["databricks"],
|
|
315
|
+
federated_auth_method: zod.default.null().optional()
|
|
316
|
+
}),
|
|
317
|
+
commonIntegrationConfig.extend({
|
|
318
|
+
type: zod.default.literal("dremio"),
|
|
319
|
+
metadata: databaseMetadataSchemasByType["dremio"],
|
|
320
|
+
federated_auth_method: zod.default.null().optional()
|
|
321
|
+
}),
|
|
322
|
+
commonIntegrationConfig.extend({
|
|
323
|
+
type: zod.default.literal("mariadb"),
|
|
324
|
+
metadata: databaseMetadataSchemasByType["mariadb"],
|
|
325
|
+
federated_auth_method: zod.default.null().optional()
|
|
326
|
+
}),
|
|
327
|
+
commonIntegrationConfig.extend({
|
|
328
|
+
type: zod.default.literal("materialize"),
|
|
329
|
+
metadata: databaseMetadataSchemasByType["materialize"],
|
|
330
|
+
federated_auth_method: zod.default.null().optional()
|
|
331
|
+
}),
|
|
332
|
+
commonIntegrationConfig.extend({
|
|
333
|
+
type: zod.default.literal("mindsdb"),
|
|
334
|
+
metadata: databaseMetadataSchemasByType["mindsdb"],
|
|
335
|
+
federated_auth_method: zod.default.null().optional()
|
|
336
|
+
}),
|
|
337
|
+
commonIntegrationConfig.extend({
|
|
338
|
+
type: zod.default.literal("mongodb"),
|
|
339
|
+
metadata: databaseMetadataSchemasByType["mongodb"],
|
|
340
|
+
federated_auth_method: zod.default.null().optional()
|
|
341
|
+
}),
|
|
342
|
+
commonIntegrationConfig.extend({
|
|
343
|
+
type: zod.default.literal("mysql"),
|
|
344
|
+
metadata: databaseMetadataSchemasByType["mysql"],
|
|
345
|
+
federated_auth_method: zod.default.null().optional()
|
|
346
|
+
}),
|
|
347
|
+
commonIntegrationConfig.extend({
|
|
348
|
+
type: zod.default.literal("pandas-dataframe"),
|
|
349
|
+
metadata: databaseMetadataSchemasByType["pandas-dataframe"],
|
|
350
|
+
federated_auth_method: zod.default.null().optional()
|
|
351
|
+
}),
|
|
352
|
+
commonIntegrationConfig.extend({
|
|
353
|
+
type: zod.default.literal("pgsql"),
|
|
354
|
+
metadata: databaseMetadataSchemasByType["pgsql"],
|
|
355
|
+
federated_auth_method: zod.default.null().optional()
|
|
356
|
+
}),
|
|
357
|
+
commonIntegrationConfig.extend({
|
|
358
|
+
type: zod.default.literal("redshift"),
|
|
359
|
+
metadata: databaseMetadataSchemasByType["redshift"],
|
|
360
|
+
federated_auth_method: zod.default.enum(["individual-credentials"]).optional().nullable()
|
|
361
|
+
}),
|
|
362
|
+
commonIntegrationConfig.extend({
|
|
363
|
+
type: zod.default.literal("snowflake"),
|
|
364
|
+
metadata: databaseMetadataSchemasByType["snowflake"],
|
|
365
|
+
federated_auth_method: zod.default.enum([
|
|
366
|
+
"okta",
|
|
367
|
+
"snowflake",
|
|
368
|
+
"azure",
|
|
369
|
+
"key-pair"
|
|
370
|
+
]).optional().nullable()
|
|
371
|
+
}),
|
|
372
|
+
commonIntegrationConfig.extend({
|
|
373
|
+
type: zod.default.literal("spanner"),
|
|
374
|
+
metadata: databaseMetadataSchemasByType["spanner"],
|
|
375
|
+
federated_auth_method: zod.default.null().optional()
|
|
376
|
+
}),
|
|
377
|
+
commonIntegrationConfig.extend({
|
|
378
|
+
type: zod.default.literal("sql-server"),
|
|
379
|
+
metadata: databaseMetadataSchemasByType["sql-server"],
|
|
380
|
+
federated_auth_method: zod.default.null().optional()
|
|
381
|
+
}),
|
|
382
|
+
commonIntegrationConfig.extend({
|
|
383
|
+
type: zod.default.literal("trino"),
|
|
384
|
+
metadata: databaseMetadataSchemasByType["trino"],
|
|
385
|
+
federated_auth_method: zod.default.null().optional()
|
|
386
|
+
})
|
|
387
|
+
]);
|
|
388
|
+
|
|
63
389
|
//#endregion
|
|
64
390
|
//#region src/snowflake-integration-env-vars.ts
|
|
65
391
|
/**
|
|
@@ -92,6 +418,22 @@ function getSnowflakeSqlAlchemyInput(metadata, params) {
|
|
|
92
418
|
queryParams: { ...createBaseQueryParams(metadata, params.snowflakePartnerIdentifier) }
|
|
93
419
|
}));
|
|
94
420
|
}
|
|
421
|
+
function getSnowflakeFederatedAuthSqlAlchemyInput(metadata, options = {}) {
|
|
422
|
+
return createSqlAlchemyInputFromUrl(buildSnowflakeUrlForFederatedAuth(metadata, options), options.keyPair);
|
|
423
|
+
}
|
|
424
|
+
function buildSnowflakeUrlForFederatedAuth(metadata, options) {
|
|
425
|
+
const queryParams = { ...createBaseQueryParams(metadata, options.snowflakePartnerIdentifier) };
|
|
426
|
+
if (options.role) queryParams.role = options.role;
|
|
427
|
+
if (options.accessToken) {
|
|
428
|
+
queryParams.token = options.accessToken;
|
|
429
|
+
queryParams.authenticator = "oauth";
|
|
430
|
+
}
|
|
431
|
+
const accountIdentifier = metadata.accountName.replace(".privatelink", "");
|
|
432
|
+
return buildUrl(options.keyPair?.username ? `snowflake://${encodeURIComponent(options.keyPair.username)}@${encodeURIComponent(accountIdentifier)}` : `snowflake://:@${encodeURIComponent(accountIdentifier)}`, {
|
|
433
|
+
path: metadata.database,
|
|
434
|
+
queryParams
|
|
435
|
+
});
|
|
436
|
+
}
|
|
95
437
|
function createBaseQueryParams(metadata, snowflakePartnerIdentifier) {
|
|
96
438
|
return {
|
|
97
439
|
warehouse: metadata.warehouse,
|
|
@@ -117,7 +459,9 @@ function getEnvironmentVariablesForIntegrations(integrations, params) {
|
|
|
117
459
|
const errors = [];
|
|
118
460
|
integrations.forEach((integration) => {
|
|
119
461
|
const namePrefix = convertToEnvironmentVariableName(integration.name);
|
|
120
|
-
const envVarsForThisIntegration = Object.entries(integration.metadata).
|
|
462
|
+
const envVarsForThisIntegration = Object.entries(integration.metadata).filter(([key]) => {
|
|
463
|
+
return key !== "caCertificateText";
|
|
464
|
+
}).map(([key, rawValue]) => {
|
|
121
465
|
const name = `${namePrefix}_${key.toUpperCase()}`;
|
|
122
466
|
const value = String(rawValue);
|
|
123
467
|
if (integration.type === "mongodb" && integration.metadata.sslEnabled && key === "connection_string") return {
|
|
@@ -213,12 +557,12 @@ const getDatabricksSqlAlchemyInput = ({ httpPath, host, token, port, schema, cat
|
|
|
213
557
|
catalog
|
|
214
558
|
} },
|
|
215
559
|
param_style: "pyformat",
|
|
216
|
-
ssh_options: {
|
|
217
|
-
enabled:
|
|
560
|
+
ssh_options: sshEnabled ? {
|
|
561
|
+
enabled: true,
|
|
218
562
|
host: sshHost,
|
|
219
563
|
port: sshPort,
|
|
220
564
|
user: sshUser
|
|
221
|
-
}
|
|
565
|
+
} : {}
|
|
222
566
|
};
|
|
223
567
|
};
|
|
224
568
|
const getDremioSqlAlchemyInput = ({ port, host, token, schema, sshEnabled, sshHost, sshPort, sshUser }) => {
|
|
@@ -229,12 +573,12 @@ const getDremioSqlAlchemyInput = ({ port, host, token, schema, sshEnabled, sshHo
|
|
|
229
573
|
url: url.href,
|
|
230
574
|
params: {},
|
|
231
575
|
param_style: "pyformat",
|
|
232
|
-
ssh_options: {
|
|
233
|
-
enabled:
|
|
576
|
+
ssh_options: sshEnabled ? {
|
|
577
|
+
enabled: true,
|
|
234
578
|
host: sshHost,
|
|
235
579
|
port: sshPort,
|
|
236
580
|
user: sshUser
|
|
237
|
-
}
|
|
581
|
+
} : {}
|
|
238
582
|
};
|
|
239
583
|
};
|
|
240
584
|
const getSQLServerVar = ({ user, password, host, port, database, sshEnabled, sshHost, sshPort, sshUser }) => {
|
|
@@ -243,12 +587,12 @@ const getSQLServerVar = ({ user, password, host, port, database, sshEnabled, ssh
|
|
|
243
587
|
url: `mssql+pymssql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}${portSuffix}/${database}`,
|
|
244
588
|
params: {},
|
|
245
589
|
param_style: "pyformat",
|
|
246
|
-
ssh_options: {
|
|
247
|
-
enabled:
|
|
590
|
+
ssh_options: sshEnabled ? {
|
|
591
|
+
enabled: true,
|
|
248
592
|
host: sshHost,
|
|
249
593
|
port: sshPort,
|
|
250
594
|
user: sshUser
|
|
251
|
-
}
|
|
595
|
+
} : {}
|
|
252
596
|
};
|
|
253
597
|
};
|
|
254
598
|
const getPostgresSqlAlchemyInput = (integrationId, projectRootDirectory, { user, password, host, port, database, sshEnabled, sshHost, sshPort, sshUser, sslEnabled, caCertificateName }) => {
|
|
@@ -269,12 +613,12 @@ const getPostgresSqlAlchemyInput = (integrationId, projectRootDirectory, { user,
|
|
|
269
613
|
}) : void 0
|
|
270
614
|
} },
|
|
271
615
|
param_style: "pyformat",
|
|
272
|
-
ssh_options: {
|
|
273
|
-
enabled:
|
|
616
|
+
ssh_options: sshEnabled ? {
|
|
617
|
+
enabled: true,
|
|
274
618
|
host: sshHost,
|
|
275
619
|
port: sshPort,
|
|
276
620
|
user: sshUser
|
|
277
|
-
}
|
|
621
|
+
} : {}
|
|
278
622
|
};
|
|
279
623
|
};
|
|
280
624
|
const getRedshiftSqlAlchemyInput = (integrationId, projectRootDirectory, metadata) => {
|
|
@@ -297,12 +641,12 @@ const getRedshiftSqlAlchemyInput = (integrationId, projectRootDirectory, metadat
|
|
|
297
641
|
}) : void 0
|
|
298
642
|
} },
|
|
299
643
|
param_style: "pyformat",
|
|
300
|
-
ssh_options: {
|
|
301
|
-
enabled:
|
|
644
|
+
ssh_options: metadata.sshEnabled ? {
|
|
645
|
+
enabled: true,
|
|
302
646
|
host: metadata.sshHost,
|
|
303
647
|
port: metadata.sshPort,
|
|
304
648
|
user: metadata.sshUser
|
|
305
|
-
}
|
|
649
|
+
} : {}
|
|
306
650
|
};
|
|
307
651
|
if (metadata.authMethod === AwsAuthMethods.IamRole && metadata.roleArn && metadata.roleExternalId) vars.iamParams = {
|
|
308
652
|
integrationId,
|
|
@@ -329,12 +673,12 @@ const getMaterializePostgresSqlAlchemyInput = (integrationId, projectRootDirecto
|
|
|
329
673
|
}) : void 0
|
|
330
674
|
} },
|
|
331
675
|
param_style: "pyformat",
|
|
332
|
-
ssh_options: {
|
|
333
|
-
enabled:
|
|
676
|
+
ssh_options: sshEnabled ? {
|
|
677
|
+
enabled: true,
|
|
334
678
|
host: sshHost,
|
|
335
679
|
port: sshPort,
|
|
336
680
|
user: sshUser
|
|
337
|
-
}
|
|
681
|
+
} : {}
|
|
338
682
|
};
|
|
339
683
|
};
|
|
340
684
|
const getTrinoEnvVars = (integrationId, metadata, projectRootDirectory) => {
|
|
@@ -382,12 +726,12 @@ const getClickHouseSqlAlchemyInput = (integrationId, projectRootDirectory, { use
|
|
|
382
726
|
url: `clickhouse://${password ? `${encodeURIComponent(user)}:${encodeURIComponent(password)}` : encodeURIComponent(user)}@${host}${portSuffix}/${database}?${params.toString()}`,
|
|
383
727
|
params: {},
|
|
384
728
|
param_style: "pyformat",
|
|
385
|
-
ssh_options: {
|
|
386
|
-
enabled:
|
|
729
|
+
ssh_options: sshEnabled ? {
|
|
730
|
+
enabled: true,
|
|
387
731
|
host: sshHost,
|
|
388
732
|
port: sshPort,
|
|
389
733
|
user: sshUser
|
|
390
|
-
}
|
|
734
|
+
} : {}
|
|
391
735
|
};
|
|
392
736
|
};
|
|
393
737
|
var BigQueryServiceAccountParseError = class extends Error {
|
|
@@ -448,12 +792,12 @@ const getMySqlSqlAlchemyInput = (integrationId, projectRootDirectory, { user, pa
|
|
|
448
792
|
url: `mysql+pymysql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}${portSuffix}/${database}`,
|
|
449
793
|
params: { connect_args: { ssl } },
|
|
450
794
|
param_style: "pyformat",
|
|
451
|
-
ssh_options: {
|
|
452
|
-
enabled:
|
|
795
|
+
ssh_options: sshEnabled ? {
|
|
796
|
+
enabled: true,
|
|
453
797
|
host: sshHost,
|
|
454
798
|
port: sshPort,
|
|
455
799
|
user: sshUser
|
|
456
|
-
}
|
|
800
|
+
} : {}
|
|
457
801
|
};
|
|
458
802
|
};
|
|
459
803
|
function getCACertificatePath({ projectRoot, integrationId, caCertificateName }) {
|
|
@@ -469,222 +813,6 @@ function assertNever(value) {
|
|
|
469
813
|
throw new Error(`Unexpected value: ${value}`);
|
|
470
814
|
}
|
|
471
815
|
|
|
472
|
-
//#endregion
|
|
473
|
-
//#region src/database-integration-metadata-schemas.ts
|
|
474
|
-
const athenaMetadataSchema = zod.z.object({
|
|
475
|
-
access_key_id: zod.z.string(),
|
|
476
|
-
region: zod.z.string(),
|
|
477
|
-
s3_output_path: zod.z.string(),
|
|
478
|
-
secret_access_key: zod.z.string(),
|
|
479
|
-
workgroup: zod.z.string().optional()
|
|
480
|
-
});
|
|
481
|
-
const bigqueryMetadataSchema = zod.z.discriminatedUnion("authMethod", [zod.z.object({
|
|
482
|
-
authMethod: zod.z.literal(BigQueryAuthMethods.ServiceAccount),
|
|
483
|
-
service_account: zod.z.string()
|
|
484
|
-
}), zod.z.object({
|
|
485
|
-
authMethod: zod.z.literal(BigQueryAuthMethods.GoogleOauth),
|
|
486
|
-
project: zod.z.string(),
|
|
487
|
-
clientId: zod.z.string(),
|
|
488
|
-
clientSecret: zod.z.string()
|
|
489
|
-
})]);
|
|
490
|
-
const clickhouseMetadataSchema = zod.z.object({
|
|
491
|
-
host: zod.z.string(),
|
|
492
|
-
user: zod.z.string(),
|
|
493
|
-
password: zod.z.string().optional(),
|
|
494
|
-
database: zod.z.string(),
|
|
495
|
-
port: zod.z.string().optional(),
|
|
496
|
-
sshEnabled: zod.z.boolean().optional(),
|
|
497
|
-
sshHost: zod.z.string().optional(),
|
|
498
|
-
sshPort: zod.z.string().optional(),
|
|
499
|
-
sshUser: zod.z.string().optional(),
|
|
500
|
-
sslEnabled: zod.z.boolean().optional(),
|
|
501
|
-
caCertificateName: zod.z.string().optional(),
|
|
502
|
-
caCertificateText: zod.z.string().optional()
|
|
503
|
-
});
|
|
504
|
-
const databricksMetadataSchema = zod.z.object({
|
|
505
|
-
host: zod.z.string(),
|
|
506
|
-
httpPath: zod.z.string(),
|
|
507
|
-
token: zod.z.string(),
|
|
508
|
-
port: zod.z.string(),
|
|
509
|
-
schema: zod.z.string().optional(),
|
|
510
|
-
catalog: zod.z.string().optional(),
|
|
511
|
-
sshEnabled: zod.z.boolean().optional(),
|
|
512
|
-
sshHost: zod.z.string().optional(),
|
|
513
|
-
sshPort: zod.z.string().optional(),
|
|
514
|
-
sshUser: zod.z.string().optional()
|
|
515
|
-
});
|
|
516
|
-
const dremioMetadataSchema = zod.z.object({
|
|
517
|
-
schema: zod.z.string(),
|
|
518
|
-
host: zod.z.string(),
|
|
519
|
-
port: zod.z.string(),
|
|
520
|
-
token: zod.z.string(),
|
|
521
|
-
sshEnabled: zod.z.boolean().optional(),
|
|
522
|
-
sshHost: zod.z.string().optional(),
|
|
523
|
-
sshPort: zod.z.string().optional(),
|
|
524
|
-
sshUser: zod.z.string().optional()
|
|
525
|
-
});
|
|
526
|
-
const mongodbMetadataSchema = zod.z.object({
|
|
527
|
-
connection_string: zod.z.string(),
|
|
528
|
-
rawConnectionString: zod.z.string().optional(),
|
|
529
|
-
prefix: zod.z.string().optional(),
|
|
530
|
-
host: zod.z.string().optional(),
|
|
531
|
-
port: zod.z.string().optional(),
|
|
532
|
-
user: zod.z.string().optional(),
|
|
533
|
-
password: zod.z.string().optional(),
|
|
534
|
-
database: zod.z.string().optional(),
|
|
535
|
-
options: zod.z.string().optional(),
|
|
536
|
-
sshEnabled: zod.z.boolean().optional(),
|
|
537
|
-
sshHost: zod.z.string().optional(),
|
|
538
|
-
sshPort: zod.z.string().optional(),
|
|
539
|
-
sshUser: zod.z.string().optional(),
|
|
540
|
-
sslEnabled: zod.z.boolean().optional(),
|
|
541
|
-
caCertificateName: zod.z.string().optional(),
|
|
542
|
-
caCertificateText: zod.z.string().optional()
|
|
543
|
-
});
|
|
544
|
-
const pandasDataframeMetadataSchema = zod.z.object({});
|
|
545
|
-
const commonRedshiftMetadataSchema = zod.z.object({
|
|
546
|
-
database: zod.z.string(),
|
|
547
|
-
host: zod.z.string(),
|
|
548
|
-
port: zod.z.string().optional(),
|
|
549
|
-
sshEnabled: zod.z.boolean().optional(),
|
|
550
|
-
sshHost: zod.z.string().optional(),
|
|
551
|
-
sshPort: zod.z.string().optional(),
|
|
552
|
-
sshUser: zod.z.string().optional(),
|
|
553
|
-
sslEnabled: zod.z.boolean().optional(),
|
|
554
|
-
caCertificateName: zod.z.string().optional(),
|
|
555
|
-
caCertificateText: zod.z.string().optional()
|
|
556
|
-
});
|
|
557
|
-
const redshiftMetadataSchema = zod.z.discriminatedUnion("authMethod", [
|
|
558
|
-
commonRedshiftMetadataSchema.extend({
|
|
559
|
-
authMethod: zod.z.literal(DatabaseAuthMethods.UsernameAndPassword),
|
|
560
|
-
user: zod.z.string(),
|
|
561
|
-
password: zod.z.string()
|
|
562
|
-
}),
|
|
563
|
-
commonRedshiftMetadataSchema.extend({
|
|
564
|
-
authMethod: zod.z.literal(AwsAuthMethods.IamRole),
|
|
565
|
-
roleArn: zod.z.string(),
|
|
566
|
-
roleExternalId: zod.z.string(),
|
|
567
|
-
roleNonce: zod.z.string()
|
|
568
|
-
}),
|
|
569
|
-
commonRedshiftMetadataSchema.extend({ authMethod: zod.z.literal(DatabaseAuthMethods.IndividualCredentials) })
|
|
570
|
-
]);
|
|
571
|
-
const commonSnowflakeMetadataSchema = zod.z.object({
|
|
572
|
-
accountName: zod.z.string(),
|
|
573
|
-
warehouse: zod.z.string().optional(),
|
|
574
|
-
database: zod.z.string().optional(),
|
|
575
|
-
role: zod.z.string().optional()
|
|
576
|
-
});
|
|
577
|
-
const snowflakeMetadataSchema = zod.z.discriminatedUnion("authMethod", [
|
|
578
|
-
commonSnowflakeMetadataSchema.extend({
|
|
579
|
-
authMethod: zod.z.literal(SnowflakeAuthMethods.Password),
|
|
580
|
-
username: zod.z.string(),
|
|
581
|
-
password: zod.z.string()
|
|
582
|
-
}),
|
|
583
|
-
commonSnowflakeMetadataSchema.extend({
|
|
584
|
-
authMethod: zod.z.literal(SnowflakeAuthMethods.Okta),
|
|
585
|
-
clientId: zod.z.string(),
|
|
586
|
-
clientSecret: zod.z.string(),
|
|
587
|
-
oktaSubdomain: zod.z.string(),
|
|
588
|
-
identityProvider: zod.z.string(),
|
|
589
|
-
authorizationServer: zod.z.string()
|
|
590
|
-
}),
|
|
591
|
-
commonSnowflakeMetadataSchema.extend({
|
|
592
|
-
authMethod: zod.z.literal(SnowflakeAuthMethods.NativeSnowflake),
|
|
593
|
-
clientId: zod.z.string(),
|
|
594
|
-
clientSecret: zod.z.string()
|
|
595
|
-
}),
|
|
596
|
-
commonSnowflakeMetadataSchema.extend({
|
|
597
|
-
authMethod: zod.z.literal(SnowflakeAuthMethods.AzureAd),
|
|
598
|
-
clientId: zod.z.string(),
|
|
599
|
-
clientSecret: zod.z.string(),
|
|
600
|
-
resource: zod.z.string(),
|
|
601
|
-
tenant: zod.z.string()
|
|
602
|
-
}),
|
|
603
|
-
commonSnowflakeMetadataSchema.extend({ authMethod: zod.z.literal(SnowflakeAuthMethods.KeyPair) }),
|
|
604
|
-
commonSnowflakeMetadataSchema.extend({
|
|
605
|
-
authMethod: zod.z.literal(SnowflakeAuthMethods.ServiceAccountKeyPair),
|
|
606
|
-
username: zod.z.string(),
|
|
607
|
-
privateKey: zod.z.string(),
|
|
608
|
-
privateKeyPassphrase: zod.z.string().optional()
|
|
609
|
-
})
|
|
610
|
-
]);
|
|
611
|
-
const spannerMetadataSchema = zod.z.object({
|
|
612
|
-
service_account: zod.z.string(),
|
|
613
|
-
dataBoostEnabled: zod.z.boolean(),
|
|
614
|
-
instance: zod.z.string(),
|
|
615
|
-
database: zod.z.string()
|
|
616
|
-
});
|
|
617
|
-
const trinoMetadataSchema = zod.z.object({
|
|
618
|
-
host: zod.z.string(),
|
|
619
|
-
user: zod.z.string(),
|
|
620
|
-
password: zod.z.string(),
|
|
621
|
-
database: zod.z.string(),
|
|
622
|
-
port: zod.z.string(),
|
|
623
|
-
sshEnabled: zod.z.boolean().optional(),
|
|
624
|
-
sshHost: zod.z.string().optional(),
|
|
625
|
-
sshPort: zod.z.string().optional(),
|
|
626
|
-
sshUser: zod.z.string().optional(),
|
|
627
|
-
sslEnabled: zod.z.boolean().optional(),
|
|
628
|
-
caCertificateName: zod.z.string().optional(),
|
|
629
|
-
caCertificateText: zod.z.string().optional()
|
|
630
|
-
});
|
|
631
|
-
const commonDatabaseSchema = zod.z.object({
|
|
632
|
-
host: zod.z.string(),
|
|
633
|
-
user: zod.z.string(),
|
|
634
|
-
password: zod.z.string(),
|
|
635
|
-
database: zod.z.string(),
|
|
636
|
-
sshEnabled: zod.z.boolean().optional(),
|
|
637
|
-
sshHost: zod.z.string().optional(),
|
|
638
|
-
sshPort: zod.z.string().optional(),
|
|
639
|
-
sshUser: zod.z.string().optional()
|
|
640
|
-
});
|
|
641
|
-
const alloydbMetadataSchema = commonDatabaseSchema.extend({
|
|
642
|
-
port: zod.z.string().optional(),
|
|
643
|
-
sslEnabled: zod.z.boolean().optional(),
|
|
644
|
-
caCertificateName: zod.z.string().optional(),
|
|
645
|
-
caCertificateText: zod.z.string().optional()
|
|
646
|
-
});
|
|
647
|
-
const mariadbMetadataSchema = commonDatabaseSchema.extend({
|
|
648
|
-
port: zod.z.string().optional(),
|
|
649
|
-
caCertificateName: zod.z.string().optional()
|
|
650
|
-
});
|
|
651
|
-
const mindsdbMetadataSchema = commonDatabaseSchema.extend({
|
|
652
|
-
port: zod.z.string().optional(),
|
|
653
|
-
caCertificateName: zod.z.string().optional()
|
|
654
|
-
});
|
|
655
|
-
const mysqlMetadataSchema = commonDatabaseSchema.extend({
|
|
656
|
-
port: zod.z.string().optional(),
|
|
657
|
-
caCertificateName: zod.z.string().optional()
|
|
658
|
-
});
|
|
659
|
-
const pgsqlMetadataSchema = commonDatabaseSchema.extend({
|
|
660
|
-
port: zod.z.string().optional(),
|
|
661
|
-
sslEnabled: zod.z.boolean().optional(),
|
|
662
|
-
caCertificateName: zod.z.string().optional(),
|
|
663
|
-
caCertificateText: zod.z.string().optional()
|
|
664
|
-
});
|
|
665
|
-
const materializeMetadataSchema = pgsqlMetadataSchema.extend({ cluster: zod.z.string() });
|
|
666
|
-
const sqlServerMetadataSchema = commonDatabaseSchema.extend({ port: zod.z.string() });
|
|
667
|
-
const databaseMetadataSchemasByType = {
|
|
668
|
-
"alloydb": alloydbMetadataSchema,
|
|
669
|
-
"athena": athenaMetadataSchema,
|
|
670
|
-
"big-query": bigqueryMetadataSchema,
|
|
671
|
-
"clickhouse": clickhouseMetadataSchema,
|
|
672
|
-
"databricks": databricksMetadataSchema,
|
|
673
|
-
"dremio": dremioMetadataSchema,
|
|
674
|
-
"mariadb": mariadbMetadataSchema,
|
|
675
|
-
"materialize": materializeMetadataSchema,
|
|
676
|
-
"mindsdb": mindsdbMetadataSchema,
|
|
677
|
-
"mongodb": mongodbMetadataSchema,
|
|
678
|
-
"mysql": mysqlMetadataSchema,
|
|
679
|
-
"pandas-dataframe": pandasDataframeMetadataSchema,
|
|
680
|
-
"pgsql": pgsqlMetadataSchema,
|
|
681
|
-
"redshift": redshiftMetadataSchema,
|
|
682
|
-
"snowflake": snowflakeMetadataSchema,
|
|
683
|
-
"spanner": spannerMetadataSchema,
|
|
684
|
-
"sql-server": sqlServerMetadataSchema,
|
|
685
|
-
"trino": trinoMetadataSchema
|
|
686
|
-
};
|
|
687
|
-
|
|
688
816
|
//#endregion
|
|
689
817
|
//#region src/database-integration-types.ts
|
|
690
818
|
const sqlIntegrationTypes = [
|
|
@@ -708,6 +836,7 @@ const sqlIntegrationTypes = [
|
|
|
708
836
|
];
|
|
709
837
|
const databaseIntegrationTypes = [...sqlIntegrationTypes, "mongodb"];
|
|
710
838
|
const databaseIntegrationTypesWithSslSupport = [
|
|
839
|
+
"alloydb",
|
|
711
840
|
"clickhouse",
|
|
712
841
|
"dremio",
|
|
713
842
|
"mariadb",
|
|
@@ -718,6 +847,15 @@ const databaseIntegrationTypesWithSslSupport = [
|
|
|
718
847
|
"redshift",
|
|
719
848
|
"trino"
|
|
720
849
|
];
|
|
850
|
+
function isDatabaseIntegrationType(type) {
|
|
851
|
+
return databaseIntegrationTypes.includes(type);
|
|
852
|
+
}
|
|
853
|
+
function isSqlIntegrationType(type) {
|
|
854
|
+
return sqlIntegrationTypes.includes(type);
|
|
855
|
+
}
|
|
856
|
+
function isDatabaseIntegrationTypeWithSslSupport(type) {
|
|
857
|
+
return databaseIntegrationTypesWithSslSupport.includes(type);
|
|
858
|
+
}
|
|
721
859
|
|
|
722
860
|
//#endregion
|
|
723
861
|
exports.AwsAuthMethods = AwsAuthMethods;
|
|
@@ -726,11 +864,17 @@ exports.BigQueryServiceAccountParseError = BigQueryServiceAccountParseError;
|
|
|
726
864
|
exports.DatabaseAuthMethods = DatabaseAuthMethods;
|
|
727
865
|
exports.SnowflakeAuthMethods = SnowflakeAuthMethods;
|
|
728
866
|
exports.SpannerServiceAccountParseError = SpannerServiceAccountParseError;
|
|
867
|
+
exports.databaseIntegrationConfigSchema = databaseIntegrationConfigSchema;
|
|
729
868
|
exports.databaseIntegrationTypes = databaseIntegrationTypes;
|
|
730
869
|
exports.databaseIntegrationTypesWithSslSupport = databaseIntegrationTypesWithSslSupport;
|
|
731
870
|
exports.databaseMetadataSchemasByType = databaseMetadataSchemasByType;
|
|
871
|
+
exports.federatedAuthMethods = federatedAuthMethods;
|
|
732
872
|
exports.getEnvironmentVariablesForIntegrations = getEnvironmentVariablesForIntegrations;
|
|
873
|
+
exports.getSnowflakeFederatedAuthSqlAlchemyInput = getSnowflakeFederatedAuthSqlAlchemyInput;
|
|
733
874
|
exports.getSqlAlchemyInput = getSqlAlchemyInput;
|
|
875
|
+
exports.isDatabaseIntegrationType = isDatabaseIntegrationType;
|
|
876
|
+
exports.isDatabaseIntegrationTypeWithSslSupport = isDatabaseIntegrationTypeWithSslSupport;
|
|
734
877
|
exports.isFederatedAuthMetadata = isFederatedAuthMetadata;
|
|
735
878
|
exports.isFederatedAuthMethod = isFederatedAuthMethod;
|
|
879
|
+
exports.isSqlIntegrationType = isSqlIntegrationType;
|
|
736
880
|
exports.sqlIntegrationTypes = sqlIntegrationTypes;
|