@deepnote/database-integrations 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +221 -0
- package/dist/index.cjs +734 -0
- package/dist/index.d.cts +398 -0
- package/dist/index.d.ts +398 -0
- package/dist/index.js +696 -0
- package/package.json +42 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,696 @@
|
|
|
1
|
+
import buildUrl from "build-url-ts";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
//#region src/sql-integration-auth-methods.ts
|
|
5
|
+
const AwsAuthMethods = {
|
|
6
|
+
AccessKeys: "access-keys",
|
|
7
|
+
IamRole: "iam-role"
|
|
8
|
+
};
|
|
9
|
+
const SnowflakeAuthMethods = {
|
|
10
|
+
AzureAd: "azure",
|
|
11
|
+
KeyPair: "key-pair",
|
|
12
|
+
ServiceAccountKeyPair: "service-account-key-pair",
|
|
13
|
+
NativeSnowflake: "snowflake",
|
|
14
|
+
Okta: "okta",
|
|
15
|
+
Password: "password"
|
|
16
|
+
};
|
|
17
|
+
const BigQueryAuthMethods = {
|
|
18
|
+
ServiceAccount: "service-account",
|
|
19
|
+
GoogleOauth: "google-oauth"
|
|
20
|
+
};
|
|
21
|
+
const DatabaseAuthMethods = {
|
|
22
|
+
UsernameAndPassword: "username-and-password",
|
|
23
|
+
IndividualCredentials: "individual-credentials"
|
|
24
|
+
};
|
|
25
|
+
const federatedAuthMethods = [
|
|
26
|
+
SnowflakeAuthMethods.Okta,
|
|
27
|
+
SnowflakeAuthMethods.NativeSnowflake,
|
|
28
|
+
SnowflakeAuthMethods.AzureAd,
|
|
29
|
+
SnowflakeAuthMethods.KeyPair,
|
|
30
|
+
BigQueryAuthMethods.GoogleOauth,
|
|
31
|
+
DatabaseAuthMethods.IndividualCredentials
|
|
32
|
+
];
|
|
33
|
+
function isFederatedAuthMethod(authMethod) {
|
|
34
|
+
return federatedAuthMethods.includes(authMethod);
|
|
35
|
+
}
|
|
36
|
+
function isFederatedAuthMetadata(metadata) {
|
|
37
|
+
return isFederatedAuthMethod(metadata.authMethod);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
//#endregion
|
|
41
|
+
//#region src/snowflake-integration-env-vars.ts
|
|
42
|
+
function getSnowflakeSqlAlchemyInput(metadata, params) {
|
|
43
|
+
if (isFederatedAuthMetadata(metadata)) return null;
|
|
44
|
+
const { username, accountName, database, authMethod } = metadata;
|
|
45
|
+
if (authMethod === SnowflakeAuthMethods.ServiceAccountKeyPair) {
|
|
46
|
+
const url$1 = buildUrl(`snowflake://${encodeURIComponent(username)}@${encodeURIComponent(accountName)}`, {
|
|
47
|
+
path: database,
|
|
48
|
+
queryParams: {
|
|
49
|
+
...createBaseQueryParams(metadata, params.snowflakePartnerIdentifier),
|
|
50
|
+
authenticator: "snowflake_jwt"
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
if (!url$1) throw new Error("getSnowflakeEnvVar URL must be defined");
|
|
54
|
+
return createSqlAlchemyInputFromUrl(url$1, {
|
|
55
|
+
privateKey: metadata.privateKey,
|
|
56
|
+
password: metadata.privateKeyPassphrase,
|
|
57
|
+
username: metadata.username
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
const url = buildUrl(`snowflake://${encodeURIComponent(username)}:${encodeURIComponent(metadata.password)}@${encodeURIComponent(accountName)}`, {
|
|
61
|
+
path: database,
|
|
62
|
+
queryParams: { ...createBaseQueryParams(metadata, params.snowflakePartnerIdentifier) }
|
|
63
|
+
});
|
|
64
|
+
if (!url) throw new Error("getSnowflakeEnvVar URL must be defined");
|
|
65
|
+
return createSqlAlchemyInputFromUrl(url);
|
|
66
|
+
}
|
|
67
|
+
function createBaseQueryParams(metadata, snowflakePartnerIdentifier) {
|
|
68
|
+
return {
|
|
69
|
+
warehouse: metadata.warehouse,
|
|
70
|
+
role: metadata.role,
|
|
71
|
+
application: snowflakePartnerIdentifier
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function createSqlAlchemyInputFromUrl(url, keyPair) {
|
|
75
|
+
return {
|
|
76
|
+
url,
|
|
77
|
+
params: keyPair ? {
|
|
78
|
+
snowflake_private_key: Buffer.from(keyPair.privateKey).toString("base64"),
|
|
79
|
+
snowflake_private_key_passphrase: keyPair.password || void 0
|
|
80
|
+
} : {},
|
|
81
|
+
param_style: "pyformat"
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
//#endregion
|
|
86
|
+
//#region src/database-integration-env-vars.ts
|
|
87
|
+
function getIntegrationListEnv(sqlIntegrations, params) {
|
|
88
|
+
const envVars = [];
|
|
89
|
+
const errors = [];
|
|
90
|
+
sqlIntegrations.forEach((integration) => {
|
|
91
|
+
const namePrefix = convertToEnvironmentVariableName(integration.name);
|
|
92
|
+
const envVarsForThisIntegration = Object.entries(integration.metadata).map(([key, rawValue]) => {
|
|
93
|
+
const name = `${namePrefix}_${key.toUpperCase()}`;
|
|
94
|
+
const value = String(rawValue);
|
|
95
|
+
if (integration.type === "mongodb" && integration.metadata.sslEnabled && key === "connection_string") return {
|
|
96
|
+
name,
|
|
97
|
+
value: addSslOptionsToMongoConnectionString(params.projectRootDirectory, value, integration.id, integration.metadata)
|
|
98
|
+
};
|
|
99
|
+
return {
|
|
100
|
+
name,
|
|
101
|
+
value
|
|
102
|
+
};
|
|
103
|
+
});
|
|
104
|
+
if (integration.type !== "mongodb") try {
|
|
105
|
+
const envVar = getEnvVarForSqlCells(integration, params);
|
|
106
|
+
if (envVar) envVarsForThisIntegration.push(envVar);
|
|
107
|
+
} catch (error) {
|
|
108
|
+
errors.push(error);
|
|
109
|
+
}
|
|
110
|
+
envVars.push(...envVarsForThisIntegration.filter((envVar) => !!envVar.name));
|
|
111
|
+
});
|
|
112
|
+
return {
|
|
113
|
+
envVars,
|
|
114
|
+
errors
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
const duckDbEnvVar = {
|
|
118
|
+
url: `deepnote+duckdb:///:memory:`,
|
|
119
|
+
params: {},
|
|
120
|
+
param_style: "qmark"
|
|
121
|
+
};
|
|
122
|
+
function addSslOptionsToMongoConnectionString(projectRoot, connectionString, integrationId, metadata) {
|
|
123
|
+
const [base, paramString] = connectionString.split("?");
|
|
124
|
+
const params = new URLSearchParams(paramString);
|
|
125
|
+
params.set("tls", "true");
|
|
126
|
+
if (metadata.caCertificateName) {
|
|
127
|
+
const path = getCACertificatePath({
|
|
128
|
+
projectRoot,
|
|
129
|
+
integrationId,
|
|
130
|
+
caCertificateName: metadata.caCertificateName
|
|
131
|
+
});
|
|
132
|
+
params.set("tlsCAFile", path);
|
|
133
|
+
}
|
|
134
|
+
return `${base}?${params.toString()}`;
|
|
135
|
+
}
|
|
136
|
+
function getEnvVarForSqlCells(integration, params) {
|
|
137
|
+
const sqlAlchemyInput = getSqlAlchemyInput(integration, params);
|
|
138
|
+
if (!sqlAlchemyInput) return null;
|
|
139
|
+
return {
|
|
140
|
+
name: convertToEnvironmentVariableName(getSqlEnvVarName(integration.id)),
|
|
141
|
+
value: JSON.stringify({
|
|
142
|
+
integration_id: integration.id,
|
|
143
|
+
...sqlAlchemyInput
|
|
144
|
+
})
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* @returns sql alchemy input, or undefined if there is an error.
|
|
149
|
+
*
|
|
150
|
+
* The URL and the params are passed to SQL Alchemy's create_engine function as arguments. The params are
|
|
151
|
+
* passed as keyword arguments to the function.
|
|
152
|
+
* https://docs.sqlalchemy.org/en/20/core/engines.html#sqlalchemy.create_engine
|
|
153
|
+
*
|
|
154
|
+
* See the `query_data_source` method in Deepnote Toolkit for more details:
|
|
155
|
+
* https://github.com/deepnote/deepnote-toolkit/blob/main/deepnote_toolkit/sql/sql_execution.py#L321
|
|
156
|
+
*/
|
|
157
|
+
function getSqlAlchemyInput(integration, params) {
|
|
158
|
+
if (integration.federated_auth_method) return null;
|
|
159
|
+
switch (integration.type) {
|
|
160
|
+
case "alloydb":
|
|
161
|
+
case "pgsql": return getPostgresSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
|
|
162
|
+
case "athena": return getAthenaSqlAlchemyInput(integration.metadata);
|
|
163
|
+
case "big-query": return getBigQuerySqlAlchemyInput(integration.metadata);
|
|
164
|
+
case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
|
|
165
|
+
case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata);
|
|
166
|
+
case "dremio": return getDremioSqlAlchemyInput(integration.metadata);
|
|
167
|
+
case "mariadb":
|
|
168
|
+
case "mindsdb":
|
|
169
|
+
case "mysql": return getMySqlSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
|
|
170
|
+
case "materialize": return getMaterializePostgresSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
|
|
171
|
+
case "pandas-dataframe": return { ...duckDbEnvVar };
|
|
172
|
+
case "redshift": return getRedshiftSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
|
|
173
|
+
case "snowflake": return getSnowflakeSqlAlchemyInput(integration.metadata, { snowflakePartnerIdentifier: params.snowflakePartnerIdentifier });
|
|
174
|
+
case "spanner": return getSpannerSqlAlchemyInput(integration.metadata);
|
|
175
|
+
case "sql-server": return getSQLServerVar(integration.metadata);
|
|
176
|
+
case "trino": return getTrinoEnvVars(integration.id, integration.metadata, params.projectRootDirectory);
|
|
177
|
+
default: assertNever(integration);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const getDatabricksSqlAlchemyInput = ({ httpPath, host, token, port, schema, catalog, sshEnabled, sshHost, sshPort, sshUser }) => {
|
|
181
|
+
return {
|
|
182
|
+
url: `databricks+connector://token:${encodeURIComponent(token)}@${host}:${port}`,
|
|
183
|
+
params: { connect_args: {
|
|
184
|
+
http_path: httpPath,
|
|
185
|
+
schema,
|
|
186
|
+
catalog
|
|
187
|
+
} },
|
|
188
|
+
param_style: "pyformat",
|
|
189
|
+
ssh_options: {
|
|
190
|
+
enabled: String(sshEnabled),
|
|
191
|
+
host: sshHost,
|
|
192
|
+
port: sshPort,
|
|
193
|
+
user: sshUser
|
|
194
|
+
}
|
|
195
|
+
};
|
|
196
|
+
};
|
|
197
|
+
const getDremioSqlAlchemyInput = ({ port, host, token, schema, sshEnabled, sshHost, sshPort, sshUser }) => {
|
|
198
|
+
const url = new URL(`dremio+flight://${host}:${port}${schema ? "/" + schema : ""}`);
|
|
199
|
+
url.searchParams.set("UseEncryption", "true");
|
|
200
|
+
url.searchParams.set("Token", token);
|
|
201
|
+
return {
|
|
202
|
+
url: url.href,
|
|
203
|
+
params: {},
|
|
204
|
+
param_style: "pyformat",
|
|
205
|
+
ssh_options: {
|
|
206
|
+
enabled: String(sshEnabled),
|
|
207
|
+
host: sshHost,
|
|
208
|
+
port: sshPort,
|
|
209
|
+
user: sshUser
|
|
210
|
+
}
|
|
211
|
+
};
|
|
212
|
+
};
|
|
213
|
+
const getSQLServerVar = ({ user, password, host, port, database, sshEnabled, sshHost, sshPort, sshUser }) => {
|
|
214
|
+
const portSuffix = port ? `:${port}` : "";
|
|
215
|
+
return {
|
|
216
|
+
url: `mssql+pymssql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}${portSuffix}/${database}`,
|
|
217
|
+
params: {},
|
|
218
|
+
param_style: "pyformat",
|
|
219
|
+
ssh_options: {
|
|
220
|
+
enabled: String(sshEnabled),
|
|
221
|
+
host: sshHost,
|
|
222
|
+
port: sshPort,
|
|
223
|
+
user: sshUser
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
};
|
|
227
|
+
const getPostgresSqlAlchemyInput = (integrationId, projectRootDirectory, { user, password, host, port, database, sshEnabled, sshHost, sshPort, sshUser, sslEnabled, caCertificateName }) => {
|
|
228
|
+
const portSuffix = port ? `:${port}` : "";
|
|
229
|
+
const mode = getPostgresStyleMode(caCertificateName, sslEnabled);
|
|
230
|
+
return {
|
|
231
|
+
url: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}${portSuffix}/${database}`,
|
|
232
|
+
params: { connect_args: {
|
|
233
|
+
keepalives: 1,
|
|
234
|
+
keepalives_idle: 30,
|
|
235
|
+
keepalives_interval: 10,
|
|
236
|
+
keepalives_count: 5,
|
|
237
|
+
sslmode: mode,
|
|
238
|
+
sslrootcert: caCertificateName ? getCACertificatePath({
|
|
239
|
+
projectRoot: projectRootDirectory,
|
|
240
|
+
integrationId,
|
|
241
|
+
caCertificateName
|
|
242
|
+
}) : void 0
|
|
243
|
+
} },
|
|
244
|
+
param_style: "pyformat",
|
|
245
|
+
ssh_options: {
|
|
246
|
+
enabled: String(sshEnabled),
|
|
247
|
+
host: sshHost,
|
|
248
|
+
port: sshPort,
|
|
249
|
+
user: sshUser
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
};
|
|
253
|
+
const getRedshiftSqlAlchemyInput = (integrationId, projectRootDirectory, metadata) => {
|
|
254
|
+
if (isFederatedAuthMethod(metadata.authMethod)) return null;
|
|
255
|
+
const portSuffix = metadata.port ? `:${metadata.port}` : "";
|
|
256
|
+
const mode = getPostgresStyleMode(metadata.caCertificateName, metadata.sslEnabled);
|
|
257
|
+
const credentials = metadata.authMethod === DatabaseAuthMethods.UsernameAndPassword ? `${encodeURIComponent(metadata.user)}:${encodeURIComponent(metadata.password)}` : "";
|
|
258
|
+
const vars = {
|
|
259
|
+
url: `redshift+psycopg2://${credentials ? `${credentials}@` : ""}${metadata.host}${portSuffix}/${metadata.database}`,
|
|
260
|
+
params: { connect_args: {
|
|
261
|
+
keepalives: 1,
|
|
262
|
+
keepalives_idle: 30,
|
|
263
|
+
keepalives_interval: 10,
|
|
264
|
+
keepalives_count: 5,
|
|
265
|
+
sslmode: mode,
|
|
266
|
+
sslrootcert: metadata.caCertificateName ? getCACertificatePath({
|
|
267
|
+
projectRoot: projectRootDirectory,
|
|
268
|
+
integrationId,
|
|
269
|
+
caCertificateName: metadata.caCertificateName
|
|
270
|
+
}) : void 0
|
|
271
|
+
} },
|
|
272
|
+
param_style: "pyformat",
|
|
273
|
+
ssh_options: {
|
|
274
|
+
enabled: String(metadata.sshEnabled),
|
|
275
|
+
host: metadata.sshHost,
|
|
276
|
+
port: metadata.sshPort,
|
|
277
|
+
user: metadata.sshUser
|
|
278
|
+
}
|
|
279
|
+
};
|
|
280
|
+
if (metadata.authMethod === AwsAuthMethods.IamRole && metadata.roleArn && metadata.roleExternalId) vars.iamParams = {
|
|
281
|
+
integrationId,
|
|
282
|
+
type: "redshift"
|
|
283
|
+
};
|
|
284
|
+
return vars;
|
|
285
|
+
};
|
|
286
|
+
function getPostgresStyleMode(caCertificateName, sslEnabled) {
|
|
287
|
+
if (caCertificateName) return "verify-ca";
|
|
288
|
+
if (sslEnabled) return "require";
|
|
289
|
+
return "prefer";
|
|
290
|
+
}
|
|
291
|
+
const getMaterializePostgresSqlAlchemyInput = (integrationId, projectRootDirectory, { user, password, host, port, database, sshEnabled, sshHost, sshPort, sshUser, sslEnabled, caCertificateName, cluster }) => {
|
|
292
|
+
const portSuffix = port ? `:${port}` : "";
|
|
293
|
+
const mode = getPostgresStyleMode(caCertificateName, sslEnabled);
|
|
294
|
+
return {
|
|
295
|
+
url: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}${portSuffix}/${database}?options=--cluster%3D${cluster}`,
|
|
296
|
+
params: { connect_args: {
|
|
297
|
+
sslmode: mode,
|
|
298
|
+
sslrootcert: caCertificateName ? getCACertificatePath({
|
|
299
|
+
projectRoot: projectRootDirectory,
|
|
300
|
+
integrationId,
|
|
301
|
+
caCertificateName
|
|
302
|
+
}) : void 0
|
|
303
|
+
} },
|
|
304
|
+
param_style: "pyformat",
|
|
305
|
+
ssh_options: {
|
|
306
|
+
enabled: String(sshEnabled),
|
|
307
|
+
host: sshHost,
|
|
308
|
+
port: sshPort,
|
|
309
|
+
user: sshUser
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
};
|
|
313
|
+
const getTrinoEnvVars = (integrationId, metadata, projectRootDirectory) => {
|
|
314
|
+
return {
|
|
315
|
+
url: `trino://${encodeURIComponent(metadata.user)}:${encodeURIComponent(metadata.password)}@${metadata.host}:${metadata.port}/${encodeURIComponent(metadata.database)}`,
|
|
316
|
+
params: { connect_args: {
|
|
317
|
+
...metadata.sslEnabled ? { http_scheme: "https" } : {},
|
|
318
|
+
...metadata.caCertificateName ? { verify: getCACertificatePath({
|
|
319
|
+
projectRoot: projectRootDirectory,
|
|
320
|
+
integrationId,
|
|
321
|
+
caCertificateName: metadata.caCertificateName
|
|
322
|
+
}) } : {}
|
|
323
|
+
} },
|
|
324
|
+
param_style: "pyformat"
|
|
325
|
+
};
|
|
326
|
+
};
|
|
327
|
+
const getAthenaSqlAlchemyInput = ({ access_key_id, region, s3_output_path, secret_access_key, workgroup }) => {
|
|
328
|
+
const baseUrl = `awsathena+rest://${encodeURIComponent(access_key_id)}:${encodeURIComponent(secret_access_key)}@athena.${region}.amazonaws.com:443/`;
|
|
329
|
+
const params = new URLSearchParams();
|
|
330
|
+
params.set("s3_staging_dir", s3_output_path);
|
|
331
|
+
if (workgroup) params.set("work_group", workgroup);
|
|
332
|
+
return {
|
|
333
|
+
url: `${baseUrl}?${params.toString()}`,
|
|
334
|
+
params: {},
|
|
335
|
+
param_style: "pyformat"
|
|
336
|
+
};
|
|
337
|
+
};
|
|
338
|
+
const getClickHouseSqlAlchemyInput = (integrationId, projectRootDirectory, { user, password, host, port, database, sshEnabled, sshHost, sshPort, sshUser, sslEnabled, caCertificateName }) => {
|
|
339
|
+
const portSuffix = port ? `:${port}` : "";
|
|
340
|
+
const params = new URLSearchParams();
|
|
341
|
+
params.set("protocol", "https");
|
|
342
|
+
if (sslEnabled) {
|
|
343
|
+
params.set("secure", "true");
|
|
344
|
+
params.set("tls_mode", "strict");
|
|
345
|
+
}
|
|
346
|
+
if (caCertificateName) {
|
|
347
|
+
const certificatePath = getCACertificatePath({
|
|
348
|
+
projectRoot: projectRootDirectory,
|
|
349
|
+
integrationId,
|
|
350
|
+
caCertificateName
|
|
351
|
+
});
|
|
352
|
+
params.set("verify", certificatePath);
|
|
353
|
+
}
|
|
354
|
+
return {
|
|
355
|
+
url: `clickhouse://${password ? `${encodeURIComponent(user)}:${encodeURIComponent(password)}` : encodeURIComponent(user)}@${host}${portSuffix}/${database}?${params.toString()}`,
|
|
356
|
+
params: {},
|
|
357
|
+
param_style: "pyformat",
|
|
358
|
+
ssh_options: {
|
|
359
|
+
enabled: String(sshEnabled),
|
|
360
|
+
host: sshHost,
|
|
361
|
+
port: sshPort,
|
|
362
|
+
user: sshUser
|
|
363
|
+
}
|
|
364
|
+
};
|
|
365
|
+
};
|
|
366
|
+
var BigQueryServiceAccountParseError = class extends Error {
|
|
367
|
+
cause;
|
|
368
|
+
constructor(message, params) {
|
|
369
|
+
super(message);
|
|
370
|
+
this.name = "BigQueryServiceAccountParseError";
|
|
371
|
+
this.cause = params.cause;
|
|
372
|
+
}
|
|
373
|
+
};
|
|
374
|
+
const getBigQuerySqlAlchemyInput = (metadata) => {
|
|
375
|
+
if (isFederatedAuthMetadata(metadata)) return null;
|
|
376
|
+
try {
|
|
377
|
+
return {
|
|
378
|
+
url: "bigquery://",
|
|
379
|
+
params: { credentials_info: JSON.parse(metadata.service_account) },
|
|
380
|
+
param_style: "pyformat"
|
|
381
|
+
};
|
|
382
|
+
} catch (error) {
|
|
383
|
+
throw new BigQueryServiceAccountParseError("Failed to parse bigquery service account, returning empty environment variable.", { cause: error instanceof Error ? error : new Error(String(error)) });
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
var SpannerServiceAccountParseError = class extends Error {
|
|
387
|
+
cause;
|
|
388
|
+
constructor(message, params) {
|
|
389
|
+
super(message);
|
|
390
|
+
this.name = "SpannerServiceAccountParseError";
|
|
391
|
+
this.cause = params.cause;
|
|
392
|
+
}
|
|
393
|
+
};
|
|
394
|
+
const getSpannerSqlAlchemyInput = (metadata) => {
|
|
395
|
+
try {
|
|
396
|
+
const projectId = JSON.parse(metadata.service_account).project_id;
|
|
397
|
+
if (!projectId || typeof projectId !== "string") throw new SpannerServiceAccountParseError("Failed to parse spanner service account, returning empty environment variable.", { cause: /* @__PURE__ */ new Error("project_id is missing in service account") });
|
|
398
|
+
return {
|
|
399
|
+
url: `spanner+spanner:///projects/${projectId}/instances/${metadata.instance}/databases/${metadata.database}`,
|
|
400
|
+
params: {},
|
|
401
|
+
param_style: "pyformat"
|
|
402
|
+
};
|
|
403
|
+
} catch (error) {
|
|
404
|
+
if (error instanceof SpannerServiceAccountParseError) throw error;
|
|
405
|
+
throw new SpannerServiceAccountParseError("Failed to parse spanner service account, returning empty environment variable.", { cause: error instanceof Error ? error : new Error(String(error)) });
|
|
406
|
+
}
|
|
407
|
+
};
|
|
408
|
+
const getMySqlSqlAlchemyInput = (integrationId, projectRootDirectory, { user, password, host, port, database, sshEnabled, sshHost, sshPort, sshUser, caCertificateName }) => {
|
|
409
|
+
const portSuffix = port ? `:${port}` : "";
|
|
410
|
+
let ssl;
|
|
411
|
+
if (caCertificateName) ssl = {
|
|
412
|
+
ca: getCACertificatePath({
|
|
413
|
+
projectRoot: projectRootDirectory,
|
|
414
|
+
integrationId,
|
|
415
|
+
caCertificateName
|
|
416
|
+
}),
|
|
417
|
+
check_hostname: false
|
|
418
|
+
};
|
|
419
|
+
else ssl = { enable: true };
|
|
420
|
+
return {
|
|
421
|
+
url: `mysql+pymysql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}${portSuffix}/${database}`,
|
|
422
|
+
params: { connect_args: { ssl } },
|
|
423
|
+
param_style: "pyformat",
|
|
424
|
+
ssh_options: {
|
|
425
|
+
enabled: String(sshEnabled),
|
|
426
|
+
host: sshHost,
|
|
427
|
+
port: sshPort,
|
|
428
|
+
user: sshUser
|
|
429
|
+
}
|
|
430
|
+
};
|
|
431
|
+
};
|
|
432
|
+
function getCACertificatePath({ projectRoot, integrationId, caCertificateName }) {
|
|
433
|
+
return `${projectRoot ?? ""}/.deepnote/${integrationId}/${caCertificateName}`;
|
|
434
|
+
}
|
|
435
|
+
function convertToEnvironmentVariableName(str) {
|
|
436
|
+
return (/^\d/.test(str) ? `_${str}` : str).toUpperCase().replace(/[^\w]/g, "_");
|
|
437
|
+
}
|
|
438
|
+
function getSqlEnvVarName(integrationId) {
|
|
439
|
+
return `SQL_${integrationId}`;
|
|
440
|
+
}
|
|
441
|
+
function assertNever(value) {
|
|
442
|
+
throw new Error(`Unexpected value: ${value}`);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
//#endregion
|
|
446
|
+
//#region src/database-integration-metadata-schemas.ts
|
|
447
|
+
const athenaMetadataSchema = z.object({
|
|
448
|
+
access_key_id: z.string(),
|
|
449
|
+
region: z.string(),
|
|
450
|
+
s3_output_path: z.string(),
|
|
451
|
+
secret_access_key: z.string(),
|
|
452
|
+
workgroup: z.string().optional()
|
|
453
|
+
});
|
|
454
|
+
const bigqueryMetadataSchema = z.discriminatedUnion("authMethod", [z.object({
|
|
455
|
+
authMethod: z.literal(BigQueryAuthMethods.ServiceAccount),
|
|
456
|
+
service_account: z.string()
|
|
457
|
+
}), z.object({
|
|
458
|
+
authMethod: z.literal(BigQueryAuthMethods.GoogleOauth),
|
|
459
|
+
project: z.string(),
|
|
460
|
+
clientId: z.string(),
|
|
461
|
+
clientSecret: z.string()
|
|
462
|
+
})]);
|
|
463
|
+
const clickhouseMetadataSchema = z.object({
|
|
464
|
+
host: z.string(),
|
|
465
|
+
user: z.string(),
|
|
466
|
+
password: z.string().optional(),
|
|
467
|
+
database: z.string(),
|
|
468
|
+
port: z.string().optional(),
|
|
469
|
+
sshEnabled: z.boolean().optional(),
|
|
470
|
+
sshHost: z.string().optional(),
|
|
471
|
+
sshPort: z.string().optional(),
|
|
472
|
+
sshUser: z.string().optional(),
|
|
473
|
+
sslEnabled: z.boolean().optional(),
|
|
474
|
+
caCertificateName: z.string().optional(),
|
|
475
|
+
caCertificateText: z.string().optional()
|
|
476
|
+
});
|
|
477
|
+
const databricksMetadataSchema = z.object({
|
|
478
|
+
host: z.string(),
|
|
479
|
+
httpPath: z.string(),
|
|
480
|
+
token: z.string(),
|
|
481
|
+
port: z.string(),
|
|
482
|
+
schema: z.string().optional(),
|
|
483
|
+
catalog: z.string().optional(),
|
|
484
|
+
sshEnabled: z.boolean().optional(),
|
|
485
|
+
sshHost: z.string().optional(),
|
|
486
|
+
sshPort: z.string().optional(),
|
|
487
|
+
sshUser: z.string().optional()
|
|
488
|
+
});
|
|
489
|
+
const dremioMetadataSchema = z.object({
|
|
490
|
+
schema: z.string(),
|
|
491
|
+
host: z.string(),
|
|
492
|
+
port: z.string(),
|
|
493
|
+
token: z.string(),
|
|
494
|
+
sshEnabled: z.boolean().optional(),
|
|
495
|
+
sshHost: z.string().optional(),
|
|
496
|
+
sshPort: z.string().optional(),
|
|
497
|
+
sshUser: z.string().optional()
|
|
498
|
+
});
|
|
499
|
+
const mongodbMetadataSchema = z.object({
|
|
500
|
+
connection_string: z.string(),
|
|
501
|
+
rawConnectionString: z.string().optional(),
|
|
502
|
+
prefix: z.string().optional(),
|
|
503
|
+
host: z.string().optional(),
|
|
504
|
+
port: z.string().optional(),
|
|
505
|
+
user: z.string().optional(),
|
|
506
|
+
password: z.string().optional(),
|
|
507
|
+
database: z.string().optional(),
|
|
508
|
+
options: z.string().optional(),
|
|
509
|
+
sshEnabled: z.boolean().optional(),
|
|
510
|
+
sshHost: z.string().optional(),
|
|
511
|
+
sshPort: z.string().optional(),
|
|
512
|
+
sshUser: z.string().optional(),
|
|
513
|
+
sslEnabled: z.boolean().optional(),
|
|
514
|
+
caCertificateName: z.string().optional(),
|
|
515
|
+
caCertificateText: z.string().optional()
|
|
516
|
+
});
|
|
517
|
+
const pandasDataframeMetadataSchema = z.object({});
|
|
518
|
+
const commonRedshiftMetadataSchema = z.object({
|
|
519
|
+
database: z.string(),
|
|
520
|
+
host: z.string(),
|
|
521
|
+
port: z.string().optional(),
|
|
522
|
+
sshEnabled: z.boolean().optional(),
|
|
523
|
+
sshHost: z.string().optional(),
|
|
524
|
+
sshPort: z.string().optional(),
|
|
525
|
+
sshUser: z.string().optional(),
|
|
526
|
+
sslEnabled: z.boolean().optional(),
|
|
527
|
+
caCertificateName: z.string().optional(),
|
|
528
|
+
caCertificateText: z.string().optional()
|
|
529
|
+
});
|
|
530
|
+
const redshiftMetadataSchema = z.discriminatedUnion("authMethod", [
|
|
531
|
+
commonRedshiftMetadataSchema.extend({
|
|
532
|
+
authMethod: z.literal(DatabaseAuthMethods.UsernameAndPassword),
|
|
533
|
+
user: z.string(),
|
|
534
|
+
password: z.string()
|
|
535
|
+
}),
|
|
536
|
+
commonRedshiftMetadataSchema.extend({
|
|
537
|
+
authMethod: z.literal(AwsAuthMethods.IamRole),
|
|
538
|
+
roleArn: z.string(),
|
|
539
|
+
roleExternalId: z.string(),
|
|
540
|
+
roleNonce: z.string()
|
|
541
|
+
}),
|
|
542
|
+
commonRedshiftMetadataSchema.extend({ authMethod: z.literal(DatabaseAuthMethods.IndividualCredentials) })
|
|
543
|
+
]);
|
|
544
|
+
const commonSnowflakeMetadataSchema = z.object({
|
|
545
|
+
accountName: z.string(),
|
|
546
|
+
warehouse: z.string().optional(),
|
|
547
|
+
database: z.string().optional(),
|
|
548
|
+
role: z.string().optional()
|
|
549
|
+
});
|
|
550
|
+
const snowflakeMetadataSchema = z.discriminatedUnion("authMethod", [
|
|
551
|
+
commonSnowflakeMetadataSchema.extend({
|
|
552
|
+
authMethod: z.literal(SnowflakeAuthMethods.Password),
|
|
553
|
+
username: z.string(),
|
|
554
|
+
password: z.string()
|
|
555
|
+
}),
|
|
556
|
+
commonSnowflakeMetadataSchema.extend({
|
|
557
|
+
authMethod: z.literal(SnowflakeAuthMethods.Okta),
|
|
558
|
+
clientId: z.string(),
|
|
559
|
+
clientSecret: z.string(),
|
|
560
|
+
oktaSubdomain: z.string(),
|
|
561
|
+
identityProvider: z.string(),
|
|
562
|
+
authorizationServer: z.string()
|
|
563
|
+
}),
|
|
564
|
+
commonSnowflakeMetadataSchema.extend({
|
|
565
|
+
authMethod: z.literal(SnowflakeAuthMethods.NativeSnowflake),
|
|
566
|
+
clientId: z.string(),
|
|
567
|
+
clientSecret: z.string()
|
|
568
|
+
}),
|
|
569
|
+
commonSnowflakeMetadataSchema.extend({
|
|
570
|
+
authMethod: z.literal(SnowflakeAuthMethods.AzureAd),
|
|
571
|
+
clientId: z.string(),
|
|
572
|
+
clientSecret: z.string(),
|
|
573
|
+
resource: z.string(),
|
|
574
|
+
tenant: z.string()
|
|
575
|
+
}),
|
|
576
|
+
commonSnowflakeMetadataSchema.extend({ authMethod: z.literal(SnowflakeAuthMethods.KeyPair) }),
|
|
577
|
+
commonSnowflakeMetadataSchema.extend({
|
|
578
|
+
authMethod: z.literal(SnowflakeAuthMethods.ServiceAccountKeyPair),
|
|
579
|
+
username: z.string(),
|
|
580
|
+
privateKey: z.string(),
|
|
581
|
+
privateKeyPassphrase: z.string().optional()
|
|
582
|
+
})
|
|
583
|
+
]);
|
|
584
|
+
const spannerMetadataSchema = z.object({
|
|
585
|
+
service_account: z.string(),
|
|
586
|
+
dataBoostEnabled: z.boolean(),
|
|
587
|
+
instance: z.string(),
|
|
588
|
+
database: z.string()
|
|
589
|
+
});
|
|
590
|
+
const trinoMetadataSchema = z.object({
|
|
591
|
+
host: z.string(),
|
|
592
|
+
user: z.string(),
|
|
593
|
+
password: z.string(),
|
|
594
|
+
database: z.string(),
|
|
595
|
+
port: z.string(),
|
|
596
|
+
sshEnabled: z.boolean().optional(),
|
|
597
|
+
sshHost: z.string().optional(),
|
|
598
|
+
sshPort: z.string().optional(),
|
|
599
|
+
sshUser: z.string().optional(),
|
|
600
|
+
sslEnabled: z.boolean().optional(),
|
|
601
|
+
caCertificateName: z.string().optional(),
|
|
602
|
+
caCertificateText: z.string().optional()
|
|
603
|
+
});
|
|
604
|
+
const commonDatabaseSchema = z.object({
|
|
605
|
+
host: z.string(),
|
|
606
|
+
user: z.string(),
|
|
607
|
+
password: z.string(),
|
|
608
|
+
database: z.string(),
|
|
609
|
+
sshEnabled: z.boolean().optional(),
|
|
610
|
+
sshHost: z.string().optional(),
|
|
611
|
+
sshPort: z.string().optional(),
|
|
612
|
+
sshUser: z.string().optional()
|
|
613
|
+
});
|
|
614
|
+
const alloydbMetadataSchema = commonDatabaseSchema.extend({
|
|
615
|
+
port: z.string().optional(),
|
|
616
|
+
sslEnabled: z.boolean().optional(),
|
|
617
|
+
caCertificateName: z.string().optional(),
|
|
618
|
+
caCertificateText: z.string().optional()
|
|
619
|
+
});
|
|
620
|
+
const mariadbMetadataSchema = commonDatabaseSchema.extend({
|
|
621
|
+
port: z.string().optional(),
|
|
622
|
+
caCertificateName: z.string().optional()
|
|
623
|
+
});
|
|
624
|
+
const mindsdbMetadataSchema = commonDatabaseSchema.extend({
|
|
625
|
+
port: z.string().optional(),
|
|
626
|
+
caCertificateName: z.string().optional()
|
|
627
|
+
});
|
|
628
|
+
const mysqlMetadataSchema = commonDatabaseSchema.extend({
|
|
629
|
+
port: z.string().optional(),
|
|
630
|
+
caCertificateName: z.string().optional()
|
|
631
|
+
});
|
|
632
|
+
const pgsqlMetadataSchema = commonDatabaseSchema.extend({
|
|
633
|
+
port: z.string().optional(),
|
|
634
|
+
sslEnabled: z.boolean().optional(),
|
|
635
|
+
caCertificateName: z.string().optional(),
|
|
636
|
+
caCertificateText: z.string().optional()
|
|
637
|
+
});
|
|
638
|
+
const materializeMetadataSchema = pgsqlMetadataSchema.extend({ cluster: z.string() });
|
|
639
|
+
const sqlServerMetadataSchema = commonDatabaseSchema.extend({ port: z.string() });
|
|
640
|
+
const databaseMetadataSchemasByType = {
|
|
641
|
+
"alloydb": alloydbMetadataSchema,
|
|
642
|
+
"athena": athenaMetadataSchema,
|
|
643
|
+
"big-query": bigqueryMetadataSchema,
|
|
644
|
+
"clickhouse": clickhouseMetadataSchema,
|
|
645
|
+
"databricks": databricksMetadataSchema,
|
|
646
|
+
"dremio": dremioMetadataSchema,
|
|
647
|
+
"mariadb": mariadbMetadataSchema,
|
|
648
|
+
"materialize": materializeMetadataSchema,
|
|
649
|
+
"mindsdb": mindsdbMetadataSchema,
|
|
650
|
+
"mongodb": mongodbMetadataSchema,
|
|
651
|
+
"mysql": mysqlMetadataSchema,
|
|
652
|
+
"pandas-dataframe": pandasDataframeMetadataSchema,
|
|
653
|
+
"pgsql": pgsqlMetadataSchema,
|
|
654
|
+
"redshift": redshiftMetadataSchema,
|
|
655
|
+
"snowflake": snowflakeMetadataSchema,
|
|
656
|
+
"spanner": spannerMetadataSchema,
|
|
657
|
+
"sql-server": sqlServerMetadataSchema,
|
|
658
|
+
"trino": trinoMetadataSchema
|
|
659
|
+
};
|
|
660
|
+
|
|
661
|
+
//#endregion
|
|
662
|
+
//#region src/database-integration-types.ts
|
|
663
|
+
const sqlIntegrationTypes = [
|
|
664
|
+
"alloydb",
|
|
665
|
+
"athena",
|
|
666
|
+
"big-query",
|
|
667
|
+
"clickhouse",
|
|
668
|
+
"databricks",
|
|
669
|
+
"dremio",
|
|
670
|
+
"mariadb",
|
|
671
|
+
"materialize",
|
|
672
|
+
"mindsdb",
|
|
673
|
+
"mysql",
|
|
674
|
+
"pandas-dataframe",
|
|
675
|
+
"pgsql",
|
|
676
|
+
"redshift",
|
|
677
|
+
"snowflake",
|
|
678
|
+
"spanner",
|
|
679
|
+
"sql-server",
|
|
680
|
+
"trino"
|
|
681
|
+
];
|
|
682
|
+
const databaseIntegrationTypes = [...sqlIntegrationTypes, "mongodb"];
|
|
683
|
+
const databaseIntegrationTypesWithSslSupport = [
|
|
684
|
+
"clickhouse",
|
|
685
|
+
"dremio",
|
|
686
|
+
"mariadb",
|
|
687
|
+
"mindsdb",
|
|
688
|
+
"mongodb",
|
|
689
|
+
"mysql",
|
|
690
|
+
"pgsql",
|
|
691
|
+
"redshift",
|
|
692
|
+
"trino"
|
|
693
|
+
];
|
|
694
|
+
|
|
695
|
+
//#endregion
|
|
696
|
+
export { AwsAuthMethods, BigQueryAuthMethods, BigQueryServiceAccountParseError, DatabaseAuthMethods, SnowflakeAuthMethods, SpannerServiceAccountParseError, databaseIntegrationTypes, databaseIntegrationTypesWithSslSupport, databaseMetadataSchemasByType, getIntegrationListEnv, getSqlAlchemyInput, isFederatedAuthMetadata, isFederatedAuthMethod, sqlIntegrationTypes };
|