@deepnote/database-integrations 1.4.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +588 -3
- package/dist/index.d.cts +3894 -779
- package/dist/index.d.ts +3894 -779
- package/dist/index.js +766 -219
- package/package.json +4 -2
package/dist/index.cjs
CHANGED
|
@@ -23,6 +23,10 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
23
23
|
//#endregion
|
|
24
24
|
let zod = require("zod");
|
|
25
25
|
zod = __toESM(zod);
|
|
26
|
+
let yaml = require("yaml");
|
|
27
|
+
yaml = __toESM(yaml);
|
|
28
|
+
let __deepnote_blocks = require("@deepnote/blocks");
|
|
29
|
+
__deepnote_blocks = __toESM(__deepnote_blocks);
|
|
26
30
|
|
|
27
31
|
//#region src/sql-integration-auth-methods.ts
|
|
28
32
|
const AwsAuthMethods = {
|
|
@@ -97,6 +101,7 @@ const clickhouseMetadataSchema = zod.z.object({
|
|
|
97
101
|
caCertificateName: zod.z.string().optional(),
|
|
98
102
|
caCertificateText: zod.z.string().optional()
|
|
99
103
|
});
|
|
104
|
+
const cloudSqlMetadataSchema = zod.z.object({ service_account: zod.z.string() });
|
|
100
105
|
const databricksMetadataSchema = zod.z.object({
|
|
101
106
|
host: zod.z.string(),
|
|
102
107
|
httpPath: zod.z.string(),
|
|
@@ -298,6 +303,7 @@ const databaseMetadataSchemasByType = {
|
|
|
298
303
|
"athena": athenaMetadataSchema,
|
|
299
304
|
"big-query": bigqueryMetadataSchema,
|
|
300
305
|
"clickhouse": clickhouseMetadataSchema,
|
|
306
|
+
"cloud-sql": cloudSqlMetadataSchema,
|
|
301
307
|
"databricks": databricksMetadataSchema,
|
|
302
308
|
"dremio": dremioMetadataSchema,
|
|
303
309
|
"mariadb": mariadbMetadataSchema,
|
|
@@ -341,6 +347,11 @@ const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
|
|
|
341
347
|
metadata: databaseMetadataSchemasByType.clickhouse,
|
|
342
348
|
federated_auth_method: zod.default.null().optional()
|
|
343
349
|
}),
|
|
350
|
+
commonIntegrationConfig.extend({
|
|
351
|
+
type: zod.default.literal("cloud-sql"),
|
|
352
|
+
metadata: databaseMetadataSchemasByType["cloud-sql"],
|
|
353
|
+
federated_auth_method: zod.default.null().optional()
|
|
354
|
+
}),
|
|
344
355
|
commonIntegrationConfig.extend({
|
|
345
356
|
type: zod.default.literal("databricks"),
|
|
346
357
|
metadata: databaseMetadataSchemasByType.databricks,
|
|
@@ -567,6 +578,7 @@ function getSqlAlchemyInput(integration, params) {
|
|
|
567
578
|
case "athena": return getAthenaSqlAlchemyInput(integration.metadata);
|
|
568
579
|
case "big-query": return getBigQuerySqlAlchemyInput(integration.metadata);
|
|
569
580
|
case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
|
|
581
|
+
case "cloud-sql": return null;
|
|
570
582
|
case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata);
|
|
571
583
|
case "dremio": return getDremioSqlAlchemyInput(integration.metadata);
|
|
572
584
|
case "mariadb":
|
|
@@ -861,6 +873,7 @@ const sqlIntegrationTypes = [
|
|
|
861
873
|
"athena",
|
|
862
874
|
"big-query",
|
|
863
875
|
"clickhouse",
|
|
876
|
+
"cloud-sql",
|
|
864
877
|
"databricks",
|
|
865
878
|
"dremio",
|
|
866
879
|
"mariadb",
|
|
@@ -898,6 +911,205 @@ function isDatabaseIntegrationTypeWithSslSupport(type) {
|
|
|
898
911
|
return databaseIntegrationTypesWithSslSupport.includes(type);
|
|
899
912
|
}
|
|
900
913
|
|
|
914
|
+
//#endregion
|
|
915
|
+
//#region src/loading/api-error.ts
|
|
916
|
+
/**
|
|
917
|
+
* Error thrown when a Deepnote API request fails.
|
|
918
|
+
*/
|
|
919
|
+
var ApiError = class extends Error {
|
|
920
|
+
statusCode;
|
|
921
|
+
constructor(statusCode, message) {
|
|
922
|
+
super(message);
|
|
923
|
+
this.name = "ApiError";
|
|
924
|
+
this.statusCode = statusCode;
|
|
925
|
+
}
|
|
926
|
+
};
|
|
927
|
+
|
|
928
|
+
//#endregion
|
|
929
|
+
//#region src/loading/constants.ts
|
|
930
|
+
/** Built-in integrations that don't require external configuration. */
|
|
931
|
+
const BUILTIN_INTEGRATIONS = new Set(["deepnote-dataframe-sql", "pandas-dataframe"]);
|
|
932
|
+
/** Default `.env` file name for storing integration secrets. */
|
|
933
|
+
const DEFAULT_ENV_FILE = ".env";
|
|
934
|
+
/** Default integrations file name. */
|
|
935
|
+
const DEFAULT_INTEGRATIONS_FILE = ".deepnote.env.yaml";
|
|
936
|
+
/** Default Deepnote API base URL. */
|
|
937
|
+
const DEFAULT_API_URL = "https://api.deepnote.com";
|
|
938
|
+
|
|
939
|
+
//#endregion
|
|
940
|
+
//#region src/loading/env-var-refs.ts
|
|
941
|
+
const ENV_VAR_REF_PREFIX = "env:";
|
|
942
|
+
function generateEnvVarName(integrationId, fieldPath) {
|
|
943
|
+
return `${integrationId.toUpperCase().replace(/-/g, "_")}__${fieldPath.toUpperCase()}`;
|
|
944
|
+
}
|
|
945
|
+
function parseEnvVarRef(value) {
|
|
946
|
+
if (typeof value !== "string") return null;
|
|
947
|
+
if (!value.startsWith(ENV_VAR_REF_PREFIX)) return null;
|
|
948
|
+
const varName = value.slice(4);
|
|
949
|
+
if (!varName) return null;
|
|
950
|
+
return {
|
|
951
|
+
prefix: "env",
|
|
952
|
+
varName
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
function createEnvVarRef(varName) {
|
|
956
|
+
return `${ENV_VAR_REF_PREFIX}${varName}`;
|
|
957
|
+
}
|
|
958
|
+
function isEnvVarRef(value) {
|
|
959
|
+
return typeof value === "string" && parseEnvVarRef(value) !== null;
|
|
960
|
+
}
|
|
961
|
+
function extractEnvVarName(value) {
|
|
962
|
+
if (typeof value !== "string") return null;
|
|
963
|
+
return parseEnvVarRef(value)?.varName ?? null;
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* Error thrown when an environment variable reference cannot be resolved.
|
|
967
|
+
*/
|
|
968
|
+
var EnvVarResolutionError = class extends Error {
|
|
969
|
+
varName;
|
|
970
|
+
path;
|
|
971
|
+
constructor(varName, path) {
|
|
972
|
+
const pathInfo = path ? ` at "${path}"` : "";
|
|
973
|
+
super(`Environment variable "${varName}" is not defined${pathInfo}`);
|
|
974
|
+
this.name = "EnvVarResolutionError";
|
|
975
|
+
this.varName = varName;
|
|
976
|
+
this.path = path;
|
|
977
|
+
}
|
|
978
|
+
};
|
|
979
|
+
/**
|
|
980
|
+
* Recursively resolves environment variable references in an object using an explicit variable map.
|
|
981
|
+
* Replaces any string value matching `env:VAR_NAME` with the value from the provided vars map.
|
|
982
|
+
*
|
|
983
|
+
* @param obj - The object to process (can be any value: primitive, array, or object)
|
|
984
|
+
* @param vars - Map of variable names to values (e.g. from a parsed .env file, process.env, or a secret store)
|
|
985
|
+
* @param currentPath - Internal parameter for tracking the current path in the object (for error messages)
|
|
986
|
+
* @returns A new object with all env var references resolved
|
|
987
|
+
* @throws {EnvVarResolutionError} If an environment variable reference cannot be resolved
|
|
988
|
+
*/
|
|
989
|
+
function resolveEnvVarRefsFromMap(obj, vars, currentPath) {
|
|
990
|
+
if (obj === null || obj === void 0) return obj;
|
|
991
|
+
if (typeof obj === "string") {
|
|
992
|
+
const parsed = parseEnvVarRef(obj);
|
|
993
|
+
if (parsed) {
|
|
994
|
+
const envValue = vars[parsed.varName];
|
|
995
|
+
if (envValue === void 0) throw new EnvVarResolutionError(parsed.varName, currentPath);
|
|
996
|
+
return envValue;
|
|
997
|
+
}
|
|
998
|
+
return obj;
|
|
999
|
+
}
|
|
1000
|
+
if (Array.isArray(obj)) return obj.map((item, index) => {
|
|
1001
|
+
return resolveEnvVarRefsFromMap(item, vars, currentPath ? `${currentPath}[${index}]` : `[${index}]`);
|
|
1002
|
+
});
|
|
1003
|
+
if (typeof obj === "object") {
|
|
1004
|
+
const result = {};
|
|
1005
|
+
for (const [key, value] of Object.entries(obj)) result[key] = resolveEnvVarRefsFromMap(value, vars, currentPath ? `${currentPath}.${key}` : key);
|
|
1006
|
+
return result;
|
|
1007
|
+
}
|
|
1008
|
+
return obj;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
//#endregion
|
|
1012
|
+
//#region src/loading/fetch-integrations.ts
|
|
1013
|
+
/**
|
|
1014
|
+
* Schema for a single integration from the API.
|
|
1015
|
+
*/
|
|
1016
|
+
const apiIntegrationSchema = zod.z.object({
|
|
1017
|
+
id: zod.z.string(),
|
|
1018
|
+
name: zod.z.string(),
|
|
1019
|
+
type: zod.z.string(),
|
|
1020
|
+
metadata: zod.z.unknown(),
|
|
1021
|
+
is_public: zod.z.boolean().nullish(),
|
|
1022
|
+
created_at: zod.z.string().nullish(),
|
|
1023
|
+
updated_at: zod.z.string().nullish(),
|
|
1024
|
+
federated_auth_method: zod.z.string().nullish()
|
|
1025
|
+
}).passthrough();
|
|
1026
|
+
/**
|
|
1027
|
+
* Schema for the full API response.
|
|
1028
|
+
*/
|
|
1029
|
+
const apiResponseSchema = zod.z.object({ integrations: zod.z.array(apiIntegrationSchema) }).passthrough();
|
|
1030
|
+
const REQUEST_TIMEOUT_MS = 3e4;
|
|
1031
|
+
function trimTrailingSlash(url) {
|
|
1032
|
+
return url.replace(/\/+$/, "");
|
|
1033
|
+
}
|
|
1034
|
+
/**
|
|
1035
|
+
* Fetch integrations from the Deepnote API.
|
|
1036
|
+
*
|
|
1037
|
+
* Uses the global `fetch`, so it works in Node (18+) and browser-like environments alike.
|
|
1038
|
+
*
|
|
1039
|
+
* @param baseUrl - The base URL of the Deepnote API
|
|
1040
|
+
* @param token - The authentication token
|
|
1041
|
+
* @param integrationIds - Optional list of integration IDs to fetch. When provided, only these integrations are returned.
|
|
1042
|
+
* @returns Array of integrations from the API
|
|
1043
|
+
* @throws ApiError if the request fails
|
|
1044
|
+
*/
|
|
1045
|
+
async function fetchIntegrations(baseUrl, token, integrationIds) {
|
|
1046
|
+
const endpoint = new URL(`${trimTrailingSlash(baseUrl)}/v2/integrations`);
|
|
1047
|
+
endpoint.searchParams.set("includeMetadata", "true");
|
|
1048
|
+
if (integrationIds && integrationIds.length > 0) endpoint.searchParams.set("integrationIds", integrationIds.join(","));
|
|
1049
|
+
const url = endpoint.toString();
|
|
1050
|
+
const response = await fetch(url, {
|
|
1051
|
+
method: "GET",
|
|
1052
|
+
headers: {
|
|
1053
|
+
Authorization: `Bearer ${token}`,
|
|
1054
|
+
"Content-Type": "application/json"
|
|
1055
|
+
},
|
|
1056
|
+
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
|
1057
|
+
});
|
|
1058
|
+
if (!response.ok) {
|
|
1059
|
+
if (response.status === 401) throw new ApiError(401, "Authentication failed. Please check your API token.");
|
|
1060
|
+
if (response.status === 403) throw new ApiError(403, "Access denied. You may not have permission to access integrations.");
|
|
1061
|
+
throw new ApiError(response.status, `API request failed with status ${response.status}: ${response.statusText}`);
|
|
1062
|
+
}
|
|
1063
|
+
const json = await response.json();
|
|
1064
|
+
const parseResult = apiResponseSchema.safeParse(json);
|
|
1065
|
+
if (!parseResult.success) throw new Error(`Invalid API response: ${parseResult.error.message}`);
|
|
1066
|
+
return parseResult.data.integrations;
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
//#endregion
|
|
1070
|
+
//#region src/loading/integrations-document.ts
|
|
1071
|
+
/**
|
|
1072
|
+
* Parse integrations YAML content into a mutable `yaml` Document that preserves
|
|
1073
|
+
* comments and formatting. Returns `null` for empty content.
|
|
1074
|
+
*
|
|
1075
|
+
* This is the content-accepting counterpart of the Node-only `readIntegrationsDocument`.
|
|
1076
|
+
*/
|
|
1077
|
+
function parseIntegrationsDocument(content) {
|
|
1078
|
+
if (!content.trim()) return null;
|
|
1079
|
+
return (0, yaml.parseDocument)(content, {
|
|
1080
|
+
strict: true,
|
|
1081
|
+
version: "1.2"
|
|
1082
|
+
});
|
|
1083
|
+
}
|
|
1084
|
+
/**
|
|
1085
|
+
* Serialize an integrations Document back to YAML text, preserving comments and
|
|
1086
|
+
* formatting and avoiding line wrapping.
|
|
1087
|
+
*/
|
|
1088
|
+
function serializeIntegrationsDocument(doc) {
|
|
1089
|
+
return doc.toString({ lineWidth: 0 });
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
//#endregion
|
|
1093
|
+
//#region src/loading/integrations-file-schema.ts
|
|
1094
|
+
/**
|
|
1095
|
+
* Schema for the integrations YAML file structure.
|
|
1096
|
+
* Uses loose validation to accept any array of objects,
|
|
1097
|
+
* allowing per-entry validation to report specific issues.
|
|
1098
|
+
*
|
|
1099
|
+
* Example file format:
|
|
1100
|
+
* ```yaml
|
|
1101
|
+
* integrations:
|
|
1102
|
+
* - id: my-postgres
|
|
1103
|
+
* name: My PostgreSQL
|
|
1104
|
+
* type: pgsql
|
|
1105
|
+
* metadata:
|
|
1106
|
+
* host: localhost
|
|
1107
|
+
* ...
|
|
1108
|
+
* ```
|
|
1109
|
+
*/
|
|
1110
|
+
const baseIntegrationsFileSchema = zod.z.object({ integrations: zod.z.array(zod.z.record(zod.z.unknown())).optional().default([]) });
|
|
1111
|
+
const integrationsFileSchema = zod.z.object({ integrations: zod.z.array(databaseIntegrationConfigSchema) });
|
|
1112
|
+
|
|
901
1113
|
//#endregion
|
|
902
1114
|
//#region src/secret-field-paths.ts
|
|
903
1115
|
/**
|
|
@@ -920,6 +1132,7 @@ const SECRET_PATHS = {
|
|
|
920
1132
|
dremio: ["token"],
|
|
921
1133
|
athena: ["secret_access_key"],
|
|
922
1134
|
spanner: ["service_account"],
|
|
1135
|
+
"cloud-sql": ["service_account"],
|
|
923
1136
|
mongodb: [
|
|
924
1137
|
"password",
|
|
925
1138
|
"connection_string",
|
|
@@ -932,7 +1145,8 @@ const SECRET_PATHS = {
|
|
|
932
1145
|
"password",
|
|
933
1146
|
"clientSecret",
|
|
934
1147
|
"privateKey",
|
|
935
|
-
"privateKeyPassphrase"
|
|
1148
|
+
"privateKeyPassphrase",
|
|
1149
|
+
"dbtServiceToken"
|
|
936
1150
|
],
|
|
937
1151
|
redshift: ["password", "caCertificateText"],
|
|
938
1152
|
trino: [
|
|
@@ -949,32 +1163,403 @@ const SECRET_PATHS = {
|
|
|
949
1163
|
*
|
|
950
1164
|
* @example
|
|
951
1165
|
* getSecretFieldPaths('pgsql') // => ['password', 'caCertificateText']
|
|
952
|
-
* getSecretFieldPaths('snowflake') // => ['password', 'clientSecret', 'privateKey', 'privateKeyPassphrase']
|
|
1166
|
+
* getSecretFieldPaths('snowflake') // => ['password', 'clientSecret', 'privateKey', 'privateKeyPassphrase', 'dbtServiceToken']
|
|
953
1167
|
*/
|
|
954
1168
|
function getSecretFieldPaths(type) {
|
|
955
1169
|
return SECRET_PATHS[type] ?? [];
|
|
956
1170
|
}
|
|
957
1171
|
|
|
958
1172
|
//#endregion
|
|
1173
|
+
//#region src/loading/merge-integrations.ts
|
|
1174
|
+
/**
|
|
1175
|
+
* JSON schema URL for the integrations file.
|
|
1176
|
+
* TODO - change to main branch when the PR is merged into main
|
|
1177
|
+
*/
|
|
1178
|
+
const JSON_SCHEMA_URL = "https://raw.githubusercontent.com/deepnote/deepnote/refs/heads/tk/integrations-config-file-schema/json-schemas/integrations-file-schema.json";
|
|
1179
|
+
/**
|
|
1180
|
+
* Schema comment for the YAML file.
|
|
1181
|
+
*/
|
|
1182
|
+
const SCHEMA_COMMENT = `yaml-language-server: $schema=${JSON_SCHEMA_URL}`;
|
|
1183
|
+
/**
|
|
1184
|
+
* Error thrown when the integrations property has an invalid type.
|
|
1185
|
+
*/
|
|
1186
|
+
var InvalidIntegrationsTypeError = class extends Error {
|
|
1187
|
+
constructor() {
|
|
1188
|
+
super(`Invalid 'integrations' property: expected a list`);
|
|
1189
|
+
this.name = "InvalidIntegrationsTypeError";
|
|
1190
|
+
}
|
|
1191
|
+
};
|
|
1192
|
+
/**
|
|
1193
|
+
* Get the integrations sequence from a YAML Document, creating an empty one if it doesn't exist.
|
|
1194
|
+
* If the integrations property is missing or null, creates an empty array and returns a reference to it.
|
|
1195
|
+
* Throws InvalidIntegrationsTypeError if integrations exists but is not an array.
|
|
1196
|
+
*/
|
|
1197
|
+
function getOrCreateIntegrationsFromDocument(doc) {
|
|
1198
|
+
const integrations = doc.get("integrations");
|
|
1199
|
+
if (integrations == null) {
|
|
1200
|
+
const emptySeq = doc.createNode([]);
|
|
1201
|
+
emptySeq.flow = false;
|
|
1202
|
+
doc.set("integrations", emptySeq);
|
|
1203
|
+
return emptySeq;
|
|
1204
|
+
}
|
|
1205
|
+
if (!(0, yaml.isSeq)(integrations)) throw new InvalidIntegrationsTypeError();
|
|
1206
|
+
return integrations;
|
|
1207
|
+
}
|
|
1208
|
+
function getOrCreateIntegrationMetadata(doc, integrationMap) {
|
|
1209
|
+
const existingMetadata = integrationMap.get("metadata", true);
|
|
1210
|
+
if ((0, yaml.isMap)(existingMetadata)) return existingMetadata;
|
|
1211
|
+
const metadata = doc.createNode({});
|
|
1212
|
+
integrationMap.set("metadata", metadata);
|
|
1213
|
+
return metadata;
|
|
1214
|
+
}
|
|
1215
|
+
function updateIntegrationMetadataMap({ metadataMap, integrationId, integrationMetadata, secretPaths }) {
|
|
1216
|
+
const secrets = {};
|
|
1217
|
+
for (const [key, value] of Object.entries(integrationMetadata)) {
|
|
1218
|
+
if (value === void 0) continue;
|
|
1219
|
+
if (value === null) {
|
|
1220
|
+
metadataMap.set(key, null);
|
|
1221
|
+
continue;
|
|
1222
|
+
}
|
|
1223
|
+
if (!secretPaths.includes(key)) {
|
|
1224
|
+
metadataMap.set(key, value);
|
|
1225
|
+
continue;
|
|
1226
|
+
}
|
|
1227
|
+
const envVarName = extractEnvVarName(metadataMap.get(key)) ?? generateEnvVarName(integrationId, key);
|
|
1228
|
+
secrets[envVarName] = String(value);
|
|
1229
|
+
metadataMap.set(key, createEnvVarRef(envVarName));
|
|
1230
|
+
}
|
|
1231
|
+
return secrets;
|
|
1232
|
+
}
|
|
1233
|
+
/**
|
|
1234
|
+
* Update an existing integration entry in the document.
|
|
1235
|
+
* Preserves any comments or extra fields while updating known fields.
|
|
1236
|
+
* Extracts secrets and replaces them with env var references, preserving custom env var names.
|
|
1237
|
+
*
|
|
1238
|
+
* @returns Record of extracted secrets (envVarName -> secretValue)
|
|
1239
|
+
*/
|
|
1240
|
+
function updateIntegrationInDocument(doc, integrationMap, integration) {
|
|
1241
|
+
const { id, name, type, metadata, federated_auth_method,...rest } = integration;
|
|
1242
|
+
integrationMap.set("id", id);
|
|
1243
|
+
integrationMap.set("name", name);
|
|
1244
|
+
integrationMap.set("type", type);
|
|
1245
|
+
const secretPaths = getSecretFieldPaths(type);
|
|
1246
|
+
const secrets = updateIntegrationMetadataMap({
|
|
1247
|
+
metadataMap: getOrCreateIntegrationMetadata(doc, integrationMap),
|
|
1248
|
+
integrationId: id,
|
|
1249
|
+
integrationMetadata: metadata,
|
|
1250
|
+
secretPaths
|
|
1251
|
+
});
|
|
1252
|
+
if (federated_auth_method !== void 0) integrationMap.set("federated_auth_method", federated_auth_method);
|
|
1253
|
+
else integrationMap.delete("federated_auth_method");
|
|
1254
|
+
return secrets;
|
|
1255
|
+
}
|
|
1256
|
+
/**
|
|
1257
|
+
* Add a new integration to the sequence.
|
|
1258
|
+
* Extracts secrets and replaces them with env var references.
|
|
1259
|
+
*
|
|
1260
|
+
* @returns Record of extracted secrets (envVarName -> secretValue)
|
|
1261
|
+
*/
|
|
1262
|
+
function addIntegrationToSeq(doc, integrationsSeq, integration) {
|
|
1263
|
+
const secretPaths = getSecretFieldPaths(integration.type);
|
|
1264
|
+
const { metadata,...restIntegration } = integration;
|
|
1265
|
+
const integrationMap = doc.createNode(restIntegration);
|
|
1266
|
+
const metadataMap = doc.createNode({});
|
|
1267
|
+
const secrets = updateIntegrationMetadataMap({
|
|
1268
|
+
metadataMap,
|
|
1269
|
+
integrationId: integration.id,
|
|
1270
|
+
integrationMetadata: metadata,
|
|
1271
|
+
secretPaths
|
|
1272
|
+
});
|
|
1273
|
+
integrationMap.setIn(["metadata"], metadataMap);
|
|
1274
|
+
integrationsSeq.add(integrationMap);
|
|
1275
|
+
return secrets;
|
|
1276
|
+
}
|
|
1277
|
+
/**
|
|
1278
|
+
* Create a new Document with the schema comment.
|
|
1279
|
+
*/
|
|
1280
|
+
function createNewDocument() {
|
|
1281
|
+
const doc = (0, yaml.parseDocument)("integrations: []\n", { version: "1.2" });
|
|
1282
|
+
doc.commentBefore = SCHEMA_COMMENT;
|
|
1283
|
+
const integrations = doc.get("integrations", true);
|
|
1284
|
+
if ((0, yaml.isSeq)(integrations)) integrations.flow = false;
|
|
1285
|
+
return doc;
|
|
1286
|
+
}
|
|
1287
|
+
/**
|
|
1288
|
+
* Merge processed integrations into the document.
|
|
1289
|
+
* Updates existing entries in-place (preserving comments) and adds new ones.
|
|
1290
|
+
* Extracts secrets and replaces them with env var references during the merge.
|
|
1291
|
+
* Preserves custom environment variable names from existing entries.
|
|
1292
|
+
*
|
|
1293
|
+
* @returns Object containing existing count and extracted secrets
|
|
1294
|
+
*/
|
|
1295
|
+
function mergeProcessedIntegrations(doc, integrationsSeq, processedIntegrations) {
|
|
1296
|
+
const existingCount = integrationsSeq.items.length;
|
|
1297
|
+
let updatedCount = 0;
|
|
1298
|
+
let newCount = 0;
|
|
1299
|
+
const secrets = {};
|
|
1300
|
+
for (const databaseIntegration of processedIntegrations) {
|
|
1301
|
+
const existingIntegration = integrationsSeq.items.find((item) => {
|
|
1302
|
+
if (!(0, yaml.isMap)(item)) return false;
|
|
1303
|
+
return item.get("id") === databaseIntegration.id;
|
|
1304
|
+
});
|
|
1305
|
+
if (existingIntegration != null) {
|
|
1306
|
+
const extractedSecrets = updateIntegrationInDocument(doc, existingIntegration, databaseIntegration);
|
|
1307
|
+
Object.assign(secrets, extractedSecrets);
|
|
1308
|
+
updatedCount++;
|
|
1309
|
+
} else {
|
|
1310
|
+
const extractedSecrets = addIntegrationToSeq(doc, integrationsSeq, databaseIntegration);
|
|
1311
|
+
Object.assign(secrets, extractedSecrets);
|
|
1312
|
+
newCount++;
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
return {
|
|
1316
|
+
secrets,
|
|
1317
|
+
stats: {
|
|
1318
|
+
existingCount,
|
|
1319
|
+
newCount,
|
|
1320
|
+
updatedCount
|
|
1321
|
+
}
|
|
1322
|
+
};
|
|
1323
|
+
}
|
|
1324
|
+
/**
|
|
1325
|
+
* Error thrown when an API integration fails schema validation during conversion.
|
|
1326
|
+
*/
|
|
1327
|
+
var InvalidIntegrationError = class extends Error {
|
|
1328
|
+
integrationId;
|
|
1329
|
+
constructor(integrationId, message) {
|
|
1330
|
+
super(message);
|
|
1331
|
+
this.name = "InvalidIntegrationError";
|
|
1332
|
+
this.integrationId = integrationId;
|
|
1333
|
+
}
|
|
1334
|
+
};
|
|
1335
|
+
/**
|
|
1336
|
+
* Convert API integrations to DatabaseIntegrationConfig.
|
|
1337
|
+
* Validates each integration against the schema and collects errors for invalid ones.
|
|
1338
|
+
*
|
|
1339
|
+
* Shared by both `integrations pull` (via mergeApiIntegrationsIntoDocument) and `run` (for on-the-fly fetching).
|
|
1340
|
+
*
|
|
1341
|
+
* @param apiIntegrations - Integrations fetched from the API
|
|
1342
|
+
* @returns Object containing validated integrations and any validation errors
|
|
1343
|
+
*/
|
|
1344
|
+
function convertApiIntegrations(apiIntegrations) {
|
|
1345
|
+
const integrations = [];
|
|
1346
|
+
const errors = [];
|
|
1347
|
+
for (const apiIntegration of apiIntegrations) {
|
|
1348
|
+
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
|
|
1349
|
+
if (!config.success) {
|
|
1350
|
+
errors.push(new InvalidIntegrationError(apiIntegration.id, "Invalid integration returned by API."));
|
|
1351
|
+
continue;
|
|
1352
|
+
}
|
|
1353
|
+
integrations.push(config.data);
|
|
1354
|
+
}
|
|
1355
|
+
return {
|
|
1356
|
+
integrations,
|
|
1357
|
+
errors
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
1360
|
+
/**
|
|
1361
|
+
* Merge API integrations into an existing document (or create a new one).
|
|
1362
|
+
* Extracts secrets and replaces them with env var references during the merge.
|
|
1363
|
+
* Preserves custom environment variable names from existing entries.
|
|
1364
|
+
* Invalid or unsupported integrations are silently skipped.
|
|
1365
|
+
*
|
|
1366
|
+
* @param doc - The YAML document to merge into
|
|
1367
|
+
* @param apiIntegrations - Integrations fetched from the API
|
|
1368
|
+
* @returns The extracted secrets and merge statistics
|
|
1369
|
+
*/
|
|
1370
|
+
function mergeApiIntegrationsIntoDocument(doc, apiIntegrations) {
|
|
1371
|
+
const integrationsSeq = getOrCreateIntegrationsFromDocument(doc);
|
|
1372
|
+
const skipped = [];
|
|
1373
|
+
return {
|
|
1374
|
+
...mergeProcessedIntegrations(doc, integrationsSeq, apiIntegrations.reduce((acc, apiIntegration) => {
|
|
1375
|
+
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
|
|
1376
|
+
if (!config.success) {
|
|
1377
|
+
skipped.push({
|
|
1378
|
+
id: apiIntegration.id,
|
|
1379
|
+
name: apiIntegration.name,
|
|
1380
|
+
type: apiIntegration.type,
|
|
1381
|
+
issues: config.error.issues
|
|
1382
|
+
});
|
|
1383
|
+
return acc;
|
|
1384
|
+
}
|
|
1385
|
+
acc.push(config.data);
|
|
1386
|
+
return acc;
|
|
1387
|
+
}, [])),
|
|
1388
|
+
skipped
|
|
1389
|
+
};
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
//#endregion
|
|
1393
|
+
//#region src/loading/merge-into-yaml.ts
|
|
1394
|
+
/**
|
|
1395
|
+
* Merge API integrations into existing integrations YAML content (or create a new
|
|
1396
|
+
* document), returning the serialized YAML, the extracted secrets, and merge stats.
|
|
1397
|
+
*
|
|
1398
|
+
* This is a string-in / string-out convenience that mirrors what `deepnote
|
|
1399
|
+
* integrations pull` does to the YAML file — usable without touching the `yaml`
|
|
1400
|
+
* Document API. Callers persist `content` and `secrets` however they like (e.g.
|
|
1401
|
+
* the CLI writes files; the VS Code extension can write via `workspace.fs`).
|
|
1402
|
+
*
|
|
1403
|
+
* @param existingContent - Current YAML content, or `null` if the file doesn't exist yet
|
|
1404
|
+
* @param apiIntegrations - Integrations fetched from the API
|
|
1405
|
+
*/
|
|
1406
|
+
function mergeApiIntegrationsIntoYaml(existingContent, apiIntegrations) {
|
|
1407
|
+
const doc = (existingContent != null ? parseIntegrationsDocument(existingContent) : null) ?? createNewDocument();
|
|
1408
|
+
const { secrets, stats, skipped } = mergeApiIntegrationsIntoDocument(doc, apiIntegrations);
|
|
1409
|
+
if (doc.commentBefore == null || !doc.commentBefore.includes("yaml-language-server")) doc.commentBefore = SCHEMA_COMMENT;
|
|
1410
|
+
return {
|
|
1411
|
+
content: serializeIntegrationsDocument(doc),
|
|
1412
|
+
secrets,
|
|
1413
|
+
stats,
|
|
1414
|
+
skipped
|
|
1415
|
+
};
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
//#endregion
|
|
1419
|
+
//#region src/loading/parse-integrations.ts
|
|
1420
|
+
/**
|
|
1421
|
+
* Format Zod validation errors into ValidationIssue format.
|
|
1422
|
+
*/
|
|
1423
|
+
function formatZodIssues(issues, pathPrefix) {
|
|
1424
|
+
return issues.map((issue) => ({
|
|
1425
|
+
path: [pathPrefix, ...issue.path].filter(Boolean).join("."),
|
|
1426
|
+
message: issue.message,
|
|
1427
|
+
code: issue.code
|
|
1428
|
+
}));
|
|
1429
|
+
}
|
|
1430
|
+
/**
|
|
1431
|
+
* Parse integrations YAML content gracefully.
|
|
1432
|
+
* Returns all valid integrations and collects validation issues for invalid ones.
|
|
1433
|
+
*
|
|
1434
|
+
* This is the content-accepting, environment-agnostic core (no filesystem or
|
|
1435
|
+
* `process.env` access) — callers supply the YAML text and the variable map used
|
|
1436
|
+
* to resolve `env:` references. The Node-only `parseIntegrationsFile` wraps this
|
|
1437
|
+
* to read from disk and default the map to `process.env`.
|
|
1438
|
+
*
|
|
1439
|
+
* @returns Object containing valid integrations and any validation issues
|
|
1440
|
+
*/
|
|
1441
|
+
function parseIntegrations({ yaml: yaml$1, env = {} }) {
|
|
1442
|
+
const emptyIntegrations = [];
|
|
1443
|
+
const issues = [];
|
|
1444
|
+
let parsed;
|
|
1445
|
+
try {
|
|
1446
|
+
parsed = (0, __deepnote_blocks.parseYaml)(yaml$1);
|
|
1447
|
+
} catch (error) {
|
|
1448
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1449
|
+
issues.push({
|
|
1450
|
+
path: "",
|
|
1451
|
+
message: `Invalid YAML in integrations file: ${message}`,
|
|
1452
|
+
code: "yaml_parse_error"
|
|
1453
|
+
});
|
|
1454
|
+
return {
|
|
1455
|
+
integrations: emptyIntegrations,
|
|
1456
|
+
issues
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
if (parsed === null || parsed === void 0) return {
|
|
1460
|
+
integrations: emptyIntegrations,
|
|
1461
|
+
issues
|
|
1462
|
+
};
|
|
1463
|
+
const fileResult = baseIntegrationsFileSchema.safeParse(parsed);
|
|
1464
|
+
if (!fileResult.success) {
|
|
1465
|
+
issues.push(...formatZodIssues(fileResult.error.issues, ""));
|
|
1466
|
+
return {
|
|
1467
|
+
integrations: emptyIntegrations,
|
|
1468
|
+
issues
|
|
1469
|
+
};
|
|
1470
|
+
}
|
|
1471
|
+
const entries = fileResult.data.integrations;
|
|
1472
|
+
const integrations = [];
|
|
1473
|
+
for (let i = 0; i < entries.length; i++) {
|
|
1474
|
+
const entry = entries[i];
|
|
1475
|
+
const pathPrefix = `integrations[${i}]`;
|
|
1476
|
+
const entryId = zod.z.string().safeParse(entry.id).data;
|
|
1477
|
+
const integrationLabel = zod.z.string().safeParse(entry.name).data || entryId;
|
|
1478
|
+
let resolvedEntry;
|
|
1479
|
+
try {
|
|
1480
|
+
resolvedEntry = resolveEnvVarRefsFromMap(entry, env);
|
|
1481
|
+
} catch (error) {
|
|
1482
|
+
if (error instanceof EnvVarResolutionError) {
|
|
1483
|
+
const context = integrationLabel ? `Integration "${integrationLabel}": ` : "";
|
|
1484
|
+
issues.push({
|
|
1485
|
+
path: error.path ? `${pathPrefix}.${error.path}` : pathPrefix,
|
|
1486
|
+
message: `${context}${error.message}`,
|
|
1487
|
+
code: "env_var_not_defined"
|
|
1488
|
+
});
|
|
1489
|
+
continue;
|
|
1490
|
+
}
|
|
1491
|
+
throw error;
|
|
1492
|
+
}
|
|
1493
|
+
const result = databaseIntegrationConfigSchema.safeParse(resolvedEntry);
|
|
1494
|
+
if (result.success) integrations.push(result.data);
|
|
1495
|
+
else {
|
|
1496
|
+
const formattedIssues = formatZodIssues(result.error.issues, pathPrefix);
|
|
1497
|
+
if (formattedIssues.length > 0 && integrationLabel) formattedIssues[0].message = `Integration "${integrationLabel}": ${formattedIssues[0].message}`;
|
|
1498
|
+
issues.push(...formattedIssues);
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
return {
|
|
1502
|
+
integrations,
|
|
1503
|
+
issues
|
|
1504
|
+
};
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
//#endregion
|
|
1508
|
+
exports.ApiError = ApiError;
|
|
959
1509
|
exports.AwsAuthMethods = AwsAuthMethods;
|
|
1510
|
+
exports.BUILTIN_INTEGRATIONS = BUILTIN_INTEGRATIONS;
|
|
960
1511
|
exports.BigQueryAuthMethods = BigQueryAuthMethods;
|
|
961
1512
|
exports.BigQueryServiceAccountParseError = BigQueryServiceAccountParseError;
|
|
1513
|
+
exports.DEFAULT_API_URL = DEFAULT_API_URL;
|
|
1514
|
+
exports.DEFAULT_ENV_FILE = DEFAULT_ENV_FILE;
|
|
1515
|
+
exports.DEFAULT_INTEGRATIONS_FILE = DEFAULT_INTEGRATIONS_FILE;
|
|
962
1516
|
exports.DatabaseAuthMethods = DatabaseAuthMethods;
|
|
1517
|
+
exports.ENV_VAR_REF_PREFIX = ENV_VAR_REF_PREFIX;
|
|
1518
|
+
exports.EnvVarResolutionError = EnvVarResolutionError;
|
|
1519
|
+
exports.InvalidIntegrationError = InvalidIntegrationError;
|
|
1520
|
+
exports.InvalidIntegrationsTypeError = InvalidIntegrationsTypeError;
|
|
1521
|
+
exports.JSON_SCHEMA_URL = JSON_SCHEMA_URL;
|
|
1522
|
+
exports.SCHEMA_COMMENT = SCHEMA_COMMENT;
|
|
963
1523
|
exports.SnowflakeAuthMethods = SnowflakeAuthMethods;
|
|
964
1524
|
exports.SpannerServiceAccountParseError = SpannerServiceAccountParseError;
|
|
965
1525
|
exports.TrinoAuthMethods = TrinoAuthMethods;
|
|
1526
|
+
exports.addIntegrationToSeq = addIntegrationToSeq;
|
|
1527
|
+
exports.apiIntegrationSchema = apiIntegrationSchema;
|
|
1528
|
+
exports.apiResponseSchema = apiResponseSchema;
|
|
1529
|
+
exports.baseIntegrationsFileSchema = baseIntegrationsFileSchema;
|
|
1530
|
+
exports.convertApiIntegrations = convertApiIntegrations;
|
|
1531
|
+
exports.createEnvVarRef = createEnvVarRef;
|
|
1532
|
+
exports.createNewDocument = createNewDocument;
|
|
966
1533
|
exports.databaseIntegrationConfigSchema = databaseIntegrationConfigSchema;
|
|
967
1534
|
exports.databaseIntegrationTypes = databaseIntegrationTypes;
|
|
968
1535
|
exports.databaseIntegrationTypesWithSslSupport = databaseIntegrationTypesWithSslSupport;
|
|
969
1536
|
exports.databaseMetadataSchemasByType = databaseMetadataSchemasByType;
|
|
1537
|
+
exports.extractEnvVarName = extractEnvVarName;
|
|
970
1538
|
exports.federatedAuthMethods = federatedAuthMethods;
|
|
1539
|
+
exports.fetchIntegrations = fetchIntegrations;
|
|
1540
|
+
exports.generateEnvVarName = generateEnvVarName;
|
|
971
1541
|
exports.getEnvironmentVariablesForIntegrations = getEnvironmentVariablesForIntegrations;
|
|
1542
|
+
exports.getOrCreateIntegrationMetadata = getOrCreateIntegrationMetadata;
|
|
1543
|
+
exports.getOrCreateIntegrationsFromDocument = getOrCreateIntegrationsFromDocument;
|
|
972
1544
|
exports.getSecretFieldPaths = getSecretFieldPaths;
|
|
973
1545
|
exports.getSnowflakeFederatedAuthSqlAlchemyInput = getSnowflakeFederatedAuthSqlAlchemyInput;
|
|
974
1546
|
exports.getSqlAlchemyInput = getSqlAlchemyInput;
|
|
1547
|
+
exports.getSqlEnvVarName = getSqlEnvVarName;
|
|
1548
|
+
exports.integrationsFileSchema = integrationsFileSchema;
|
|
975
1549
|
exports.isDatabaseIntegrationType = isDatabaseIntegrationType;
|
|
976
1550
|
exports.isDatabaseIntegrationTypeWithSslSupport = isDatabaseIntegrationTypeWithSslSupport;
|
|
1551
|
+
exports.isEnvVarRef = isEnvVarRef;
|
|
977
1552
|
exports.isFederatedAuthMetadata = isFederatedAuthMetadata;
|
|
978
1553
|
exports.isFederatedAuthMethod = isFederatedAuthMethod;
|
|
979
1554
|
exports.isSqlIntegrationType = isSqlIntegrationType;
|
|
980
|
-
exports.
|
|
1555
|
+
exports.mergeApiIntegrationsIntoDocument = mergeApiIntegrationsIntoDocument;
|
|
1556
|
+
exports.mergeApiIntegrationsIntoYaml = mergeApiIntegrationsIntoYaml;
|
|
1557
|
+
exports.mergeProcessedIntegrations = mergeProcessedIntegrations;
|
|
1558
|
+
exports.parseEnvVarRef = parseEnvVarRef;
|
|
1559
|
+
exports.parseIntegrations = parseIntegrations;
|
|
1560
|
+
exports.parseIntegrationsDocument = parseIntegrationsDocument;
|
|
1561
|
+
exports.resolveEnvVarRefsFromMap = resolveEnvVarRefsFromMap;
|
|
1562
|
+
exports.serializeIntegrationsDocument = serializeIntegrationsDocument;
|
|
1563
|
+
exports.sqlIntegrationTypes = sqlIntegrationTypes;
|
|
1564
|
+
exports.updateIntegrationInDocument = updateIntegrationInDocument;
|
|
1565
|
+
exports.updateIntegrationMetadataMap = updateIntegrationMetadataMap;
|