@deepnote/database-integrations 1.4.3 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +580 -1
- package/dist/index.d.cts +3893 -778
- package/dist/index.d.ts +3893 -778
- package/dist/index.js +758 -217
- package/package.json +4 -2
package/dist/index.cjs
CHANGED
|
@@ -23,6 +23,10 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
23
23
|
//#endregion
|
|
24
24
|
let zod = require("zod");
|
|
25
25
|
zod = __toESM(zod);
|
|
26
|
+
let yaml = require("yaml");
|
|
27
|
+
yaml = __toESM(yaml);
|
|
28
|
+
let __deepnote_blocks = require("@deepnote/blocks");
|
|
29
|
+
__deepnote_blocks = __toESM(__deepnote_blocks);
|
|
26
30
|
|
|
27
31
|
//#region src/sql-integration-auth-methods.ts
|
|
28
32
|
const AwsAuthMethods = {
|
|
@@ -97,6 +101,7 @@ const clickhouseMetadataSchema = zod.z.object({
|
|
|
97
101
|
caCertificateName: zod.z.string().optional(),
|
|
98
102
|
caCertificateText: zod.z.string().optional()
|
|
99
103
|
});
|
|
104
|
+
const cloudSqlMetadataSchema = zod.z.object({ service_account: zod.z.string() });
|
|
100
105
|
const databricksMetadataSchema = zod.z.object({
|
|
101
106
|
host: zod.z.string(),
|
|
102
107
|
httpPath: zod.z.string(),
|
|
@@ -298,6 +303,7 @@ const databaseMetadataSchemasByType = {
|
|
|
298
303
|
"athena": athenaMetadataSchema,
|
|
299
304
|
"big-query": bigqueryMetadataSchema,
|
|
300
305
|
"clickhouse": clickhouseMetadataSchema,
|
|
306
|
+
"cloud-sql": cloudSqlMetadataSchema,
|
|
301
307
|
"databricks": databricksMetadataSchema,
|
|
302
308
|
"dremio": dremioMetadataSchema,
|
|
303
309
|
"mariadb": mariadbMetadataSchema,
|
|
@@ -341,6 +347,11 @@ const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
|
|
|
341
347
|
metadata: databaseMetadataSchemasByType.clickhouse,
|
|
342
348
|
federated_auth_method: zod.default.null().optional()
|
|
343
349
|
}),
|
|
350
|
+
commonIntegrationConfig.extend({
|
|
351
|
+
type: zod.default.literal("cloud-sql"),
|
|
352
|
+
metadata: databaseMetadataSchemasByType["cloud-sql"],
|
|
353
|
+
federated_auth_method: zod.default.null().optional()
|
|
354
|
+
}),
|
|
344
355
|
commonIntegrationConfig.extend({
|
|
345
356
|
type: zod.default.literal("databricks"),
|
|
346
357
|
metadata: databaseMetadataSchemasByType.databricks,
|
|
@@ -567,6 +578,7 @@ function getSqlAlchemyInput(integration, params) {
|
|
|
567
578
|
case "athena": return getAthenaSqlAlchemyInput(integration.metadata);
|
|
568
579
|
case "big-query": return getBigQuerySqlAlchemyInput(integration.metadata);
|
|
569
580
|
case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
|
|
581
|
+
case "cloud-sql": return null;
|
|
570
582
|
case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata);
|
|
571
583
|
case "dremio": return getDremioSqlAlchemyInput(integration.metadata);
|
|
572
584
|
case "mariadb":
|
|
@@ -861,6 +873,7 @@ const sqlIntegrationTypes = [
|
|
|
861
873
|
"athena",
|
|
862
874
|
"big-query",
|
|
863
875
|
"clickhouse",
|
|
876
|
+
"cloud-sql",
|
|
864
877
|
"databricks",
|
|
865
878
|
"dremio",
|
|
866
879
|
"mariadb",
|
|
@@ -898,6 +911,200 @@ function isDatabaseIntegrationTypeWithSslSupport(type) {
|
|
|
898
911
|
return databaseIntegrationTypesWithSslSupport.includes(type);
|
|
899
912
|
}
|
|
900
913
|
|
|
914
|
+
//#endregion
|
|
915
|
+
//#region src/loading/api-error.ts
|
|
916
|
+
/**
|
|
917
|
+
* Error thrown when a Deepnote API request fails.
|
|
918
|
+
*/
|
|
919
|
+
var ApiError = class extends Error {
|
|
920
|
+
statusCode;
|
|
921
|
+
constructor(statusCode, message) {
|
|
922
|
+
super(message);
|
|
923
|
+
this.name = "ApiError";
|
|
924
|
+
this.statusCode = statusCode;
|
|
925
|
+
}
|
|
926
|
+
};
|
|
927
|
+
|
|
928
|
+
//#endregion
|
|
929
|
+
//#region src/loading/constants.ts
|
|
930
|
+
/** Built-in integrations that don't require external configuration. */
|
|
931
|
+
const BUILTIN_INTEGRATIONS = new Set(["deepnote-dataframe-sql", "pandas-dataframe"]);
|
|
932
|
+
/** Default `.env` file name for storing integration secrets. */
|
|
933
|
+
const DEFAULT_ENV_FILE = ".env";
|
|
934
|
+
/** Default integrations file name. */
|
|
935
|
+
const DEFAULT_INTEGRATIONS_FILE = ".deepnote.env.yaml";
|
|
936
|
+
/** Default Deepnote API base URL. */
|
|
937
|
+
const DEFAULT_API_URL = "https://api.deepnote.com";
|
|
938
|
+
|
|
939
|
+
//#endregion
|
|
940
|
+
//#region src/loading/env-var-refs.ts
|
|
941
|
+
const ENV_VAR_REF_PREFIX = "env:";
|
|
942
|
+
function generateEnvVarName(integrationId, fieldPath) {
|
|
943
|
+
return `${integrationId.toUpperCase().replace(/-/g, "_")}__${fieldPath.toUpperCase()}`;
|
|
944
|
+
}
|
|
945
|
+
function parseEnvVarRef(value) {
|
|
946
|
+
if (typeof value !== "string") return null;
|
|
947
|
+
if (!value.startsWith(ENV_VAR_REF_PREFIX)) return null;
|
|
948
|
+
const varName = value.slice(4);
|
|
949
|
+
if (!varName) return null;
|
|
950
|
+
return {
|
|
951
|
+
prefix: "env",
|
|
952
|
+
varName
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
function createEnvVarRef(varName) {
|
|
956
|
+
return `${ENV_VAR_REF_PREFIX}${varName}`;
|
|
957
|
+
}
|
|
958
|
+
function isEnvVarRef(value) {
|
|
959
|
+
return typeof value === "string" && parseEnvVarRef(value) !== null;
|
|
960
|
+
}
|
|
961
|
+
function extractEnvVarName(value) {
|
|
962
|
+
if (typeof value !== "string") return null;
|
|
963
|
+
return parseEnvVarRef(value)?.varName ?? null;
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* Error thrown when an environment variable reference cannot be resolved.
|
|
967
|
+
*/
|
|
968
|
+
var EnvVarResolutionError = class extends Error {
|
|
969
|
+
varName;
|
|
970
|
+
path;
|
|
971
|
+
constructor(varName, path) {
|
|
972
|
+
const pathInfo = path ? ` at "${path}"` : "";
|
|
973
|
+
super(`Environment variable "${varName}" is not defined${pathInfo}`);
|
|
974
|
+
this.name = "EnvVarResolutionError";
|
|
975
|
+
this.varName = varName;
|
|
976
|
+
this.path = path;
|
|
977
|
+
}
|
|
978
|
+
};
|
|
979
|
+
/**
|
|
980
|
+
* Recursively resolves environment variable references in an object using an explicit variable map.
|
|
981
|
+
* Replaces any string value matching `env:VAR_NAME` with the value from the provided vars map.
|
|
982
|
+
*
|
|
983
|
+
* @param obj - The object to process (can be any value: primitive, array, or object)
|
|
984
|
+
* @param vars - Map of variable names to values (e.g. from a parsed .env file, process.env, or a secret store)
|
|
985
|
+
* @param currentPath - Internal parameter for tracking the current path in the object (for error messages)
|
|
986
|
+
* @returns A new object with all env var references resolved
|
|
987
|
+
* @throws {EnvVarResolutionError} If an environment variable reference cannot be resolved
|
|
988
|
+
*/
|
|
989
|
+
function resolveEnvVarRefsFromMap(obj, vars, currentPath) {
|
|
990
|
+
if (obj === null || obj === void 0) return obj;
|
|
991
|
+
if (typeof obj === "string") {
|
|
992
|
+
const parsed = parseEnvVarRef(obj);
|
|
993
|
+
if (parsed) {
|
|
994
|
+
const envValue = vars[parsed.varName];
|
|
995
|
+
if (envValue === void 0) throw new EnvVarResolutionError(parsed.varName, currentPath);
|
|
996
|
+
return envValue;
|
|
997
|
+
}
|
|
998
|
+
return obj;
|
|
999
|
+
}
|
|
1000
|
+
if (Array.isArray(obj)) return obj.map((item, index) => {
|
|
1001
|
+
return resolveEnvVarRefsFromMap(item, vars, currentPath ? `${currentPath}[${index}]` : `[${index}]`);
|
|
1002
|
+
});
|
|
1003
|
+
if (typeof obj === "object") {
|
|
1004
|
+
const result = {};
|
|
1005
|
+
for (const [key, value] of Object.entries(obj)) result[key] = resolveEnvVarRefsFromMap(value, vars, currentPath ? `${currentPath}.${key}` : key);
|
|
1006
|
+
return result;
|
|
1007
|
+
}
|
|
1008
|
+
return obj;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
//#endregion
|
|
1012
|
+
//#region src/loading/fetch-integrations.ts
|
|
1013
|
+
/**
|
|
1014
|
+
* Schema for a single integration from the API.
|
|
1015
|
+
*/
|
|
1016
|
+
const apiIntegrationSchema = zod.z.object({
|
|
1017
|
+
id: zod.z.string(),
|
|
1018
|
+
name: zod.z.string(),
|
|
1019
|
+
type: zod.z.string(),
|
|
1020
|
+
metadata: zod.z.unknown(),
|
|
1021
|
+
is_public: zod.z.boolean(),
|
|
1022
|
+
created_at: zod.z.string(),
|
|
1023
|
+
updated_at: zod.z.string(),
|
|
1024
|
+
federated_auth_method: zod.z.string().nullable()
|
|
1025
|
+
}).passthrough();
|
|
1026
|
+
/**
|
|
1027
|
+
* Schema for the full API response.
|
|
1028
|
+
*/
|
|
1029
|
+
const apiResponseSchema = zod.z.object({ integrations: zod.z.array(apiIntegrationSchema) }).passthrough();
|
|
1030
|
+
/**
|
|
1031
|
+
* Fetch integrations from the Deepnote API.
|
|
1032
|
+
*
|
|
1033
|
+
* Uses the global `fetch`, so it works in Node (18+) and browser-like environments alike.
|
|
1034
|
+
*
|
|
1035
|
+
* @param baseUrl - The base URL of the Deepnote API
|
|
1036
|
+
* @param token - The authentication token
|
|
1037
|
+
* @param integrationIds - Optional list of integration IDs to fetch. When provided, only these integrations are returned.
|
|
1038
|
+
* @returns Array of integrations from the API
|
|
1039
|
+
* @throws ApiError if the request fails
|
|
1040
|
+
*/
|
|
1041
|
+
async function fetchIntegrations(baseUrl, token, integrationIds) {
|
|
1042
|
+
const endpoint = new URL(`${baseUrl}/v2/integrations`);
|
|
1043
|
+
endpoint.searchParams.set("includeMetadata", "true");
|
|
1044
|
+
if (integrationIds && integrationIds.length > 0) endpoint.searchParams.set("integrationIds", integrationIds.join(","));
|
|
1045
|
+
const url = endpoint.toString();
|
|
1046
|
+
const response = await fetch(url, {
|
|
1047
|
+
method: "GET",
|
|
1048
|
+
headers: {
|
|
1049
|
+
Authorization: `Bearer ${token}`,
|
|
1050
|
+
"Content-Type": "application/json"
|
|
1051
|
+
}
|
|
1052
|
+
});
|
|
1053
|
+
if (!response.ok) {
|
|
1054
|
+
if (response.status === 401) throw new ApiError(401, "Authentication failed. Please check your API token.");
|
|
1055
|
+
if (response.status === 403) throw new ApiError(403, "Access denied. You may not have permission to access integrations.");
|
|
1056
|
+
throw new ApiError(response.status, `API request failed with status ${response.status}: ${response.statusText}`);
|
|
1057
|
+
}
|
|
1058
|
+
const json = await response.json();
|
|
1059
|
+
const parseResult = apiResponseSchema.safeParse(json);
|
|
1060
|
+
if (!parseResult.success) throw new Error(`Invalid API response: ${parseResult.error.message}`);
|
|
1061
|
+
return parseResult.data.integrations;
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
//#endregion
|
|
1065
|
+
//#region src/loading/integrations-document.ts
|
|
1066
|
+
/**
|
|
1067
|
+
* Parse integrations YAML content into a mutable `yaml` Document that preserves
|
|
1068
|
+
* comments and formatting. Returns `null` for empty content.
|
|
1069
|
+
*
|
|
1070
|
+
* This is the content-accepting counterpart of the Node-only `readIntegrationsDocument`.
|
|
1071
|
+
*/
|
|
1072
|
+
function parseIntegrationsDocument(content) {
|
|
1073
|
+
if (!content.trim()) return null;
|
|
1074
|
+
return (0, yaml.parseDocument)(content, {
|
|
1075
|
+
strict: true,
|
|
1076
|
+
version: "1.2"
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
/**
|
|
1080
|
+
* Serialize an integrations Document back to YAML text, preserving comments and
|
|
1081
|
+
* formatting and avoiding line wrapping.
|
|
1082
|
+
*/
|
|
1083
|
+
function serializeIntegrationsDocument(doc) {
|
|
1084
|
+
return doc.toString({ lineWidth: 0 });
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
//#endregion
|
|
1088
|
+
//#region src/loading/integrations-file-schema.ts
|
|
1089
|
+
/**
|
|
1090
|
+
* Schema for the integrations YAML file structure.
|
|
1091
|
+
* Uses loose validation to accept any array of objects,
|
|
1092
|
+
* allowing per-entry validation to report specific issues.
|
|
1093
|
+
*
|
|
1094
|
+
* Example file format:
|
|
1095
|
+
* ```yaml
|
|
1096
|
+
* integrations:
|
|
1097
|
+
* - id: my-postgres
|
|
1098
|
+
* name: My PostgreSQL
|
|
1099
|
+
* type: pgsql
|
|
1100
|
+
* metadata:
|
|
1101
|
+
* host: localhost
|
|
1102
|
+
* ...
|
|
1103
|
+
* ```
|
|
1104
|
+
*/
|
|
1105
|
+
const baseIntegrationsFileSchema = zod.z.object({ integrations: zod.z.array(zod.z.record(zod.z.unknown())).optional().default([]) });
|
|
1106
|
+
const integrationsFileSchema = zod.z.object({ integrations: zod.z.array(databaseIntegrationConfigSchema) });
|
|
1107
|
+
|
|
901
1108
|
//#endregion
|
|
902
1109
|
//#region src/secret-field-paths.ts
|
|
903
1110
|
/**
|
|
@@ -920,6 +1127,7 @@ const SECRET_PATHS = {
|
|
|
920
1127
|
dremio: ["token"],
|
|
921
1128
|
athena: ["secret_access_key"],
|
|
922
1129
|
spanner: ["service_account"],
|
|
1130
|
+
"cloud-sql": ["service_account"],
|
|
923
1131
|
mongodb: [
|
|
924
1132
|
"password",
|
|
925
1133
|
"connection_string",
|
|
@@ -956,25 +1164,396 @@ function getSecretFieldPaths(type) {
|
|
|
956
1164
|
}
|
|
957
1165
|
|
|
958
1166
|
//#endregion
|
|
1167
|
+
//#region src/loading/merge-integrations.ts
|
|
1168
|
+
/**
|
|
1169
|
+
* JSON schema URL for the integrations file.
|
|
1170
|
+
* TODO - change to main branch when the PR is merged into main
|
|
1171
|
+
*/
|
|
1172
|
+
const JSON_SCHEMA_URL = "https://raw.githubusercontent.com/deepnote/deepnote/refs/heads/tk/integrations-config-file-schema/json-schemas/integrations-file-schema.json";
|
|
1173
|
+
/**
|
|
1174
|
+
* Schema comment for the YAML file.
|
|
1175
|
+
*/
|
|
1176
|
+
const SCHEMA_COMMENT = `yaml-language-server: $schema=${JSON_SCHEMA_URL}`;
|
|
1177
|
+
/**
|
|
1178
|
+
* Error thrown when the integrations property has an invalid type.
|
|
1179
|
+
*/
|
|
1180
|
+
var InvalidIntegrationsTypeError = class extends Error {
|
|
1181
|
+
constructor() {
|
|
1182
|
+
super(`Invalid 'integrations' property: expected a list`);
|
|
1183
|
+
this.name = "InvalidIntegrationsTypeError";
|
|
1184
|
+
}
|
|
1185
|
+
};
|
|
1186
|
+
/**
|
|
1187
|
+
* Get the integrations sequence from a YAML Document, creating an empty one if it doesn't exist.
|
|
1188
|
+
* If the integrations property is missing or null, creates an empty array and returns a reference to it.
|
|
1189
|
+
* Throws InvalidIntegrationsTypeError if integrations exists but is not an array.
|
|
1190
|
+
*/
|
|
1191
|
+
function getOrCreateIntegrationsFromDocument(doc) {
|
|
1192
|
+
const integrations = doc.get("integrations");
|
|
1193
|
+
if (integrations == null) {
|
|
1194
|
+
const emptySeq = doc.createNode([]);
|
|
1195
|
+
emptySeq.flow = false;
|
|
1196
|
+
doc.set("integrations", emptySeq);
|
|
1197
|
+
return emptySeq;
|
|
1198
|
+
}
|
|
1199
|
+
if (!(0, yaml.isSeq)(integrations)) throw new InvalidIntegrationsTypeError();
|
|
1200
|
+
return integrations;
|
|
1201
|
+
}
|
|
1202
|
+
function getOrCreateIntegrationMetadata(doc, integrationMap) {
|
|
1203
|
+
const existingMetadata = integrationMap.get("metadata", true);
|
|
1204
|
+
if ((0, yaml.isMap)(existingMetadata)) return existingMetadata;
|
|
1205
|
+
const metadata = doc.createNode({});
|
|
1206
|
+
integrationMap.set("metadata", metadata);
|
|
1207
|
+
return metadata;
|
|
1208
|
+
}
|
|
1209
|
+
function updateIntegrationMetadataMap({ metadataMap, integrationId, integrationMetadata, secretPaths }) {
|
|
1210
|
+
const secrets = {};
|
|
1211
|
+
for (const [key, value] of Object.entries(integrationMetadata)) {
|
|
1212
|
+
if (value === void 0) continue;
|
|
1213
|
+
if (value === null) {
|
|
1214
|
+
metadataMap.set(key, null);
|
|
1215
|
+
continue;
|
|
1216
|
+
}
|
|
1217
|
+
if (!secretPaths.includes(key)) {
|
|
1218
|
+
metadataMap.set(key, value);
|
|
1219
|
+
continue;
|
|
1220
|
+
}
|
|
1221
|
+
const envVarName = extractEnvVarName(metadataMap.get(key)) ?? generateEnvVarName(integrationId, key);
|
|
1222
|
+
secrets[envVarName] = String(value);
|
|
1223
|
+
metadataMap.set(key, createEnvVarRef(envVarName));
|
|
1224
|
+
}
|
|
1225
|
+
return secrets;
|
|
1226
|
+
}
|
|
1227
|
+
/**
|
|
1228
|
+
* Update an existing integration entry in the document.
|
|
1229
|
+
* Preserves any comments or extra fields while updating known fields.
|
|
1230
|
+
* Extracts secrets and replaces them with env var references, preserving custom env var names.
|
|
1231
|
+
*
|
|
1232
|
+
* @returns Record of extracted secrets (envVarName -> secretValue)
|
|
1233
|
+
*/
|
|
1234
|
+
function updateIntegrationInDocument(doc, integrationMap, integration) {
|
|
1235
|
+
const { id, name, type, metadata, federated_auth_method,...rest } = integration;
|
|
1236
|
+
integrationMap.set("id", id);
|
|
1237
|
+
integrationMap.set("name", name);
|
|
1238
|
+
integrationMap.set("type", type);
|
|
1239
|
+
const secretPaths = getSecretFieldPaths(type);
|
|
1240
|
+
const secrets = updateIntegrationMetadataMap({
|
|
1241
|
+
metadataMap: getOrCreateIntegrationMetadata(doc, integrationMap),
|
|
1242
|
+
integrationId: id,
|
|
1243
|
+
integrationMetadata: metadata,
|
|
1244
|
+
secretPaths
|
|
1245
|
+
});
|
|
1246
|
+
if (federated_auth_method !== void 0) integrationMap.set("federated_auth_method", federated_auth_method);
|
|
1247
|
+
else integrationMap.delete("federated_auth_method");
|
|
1248
|
+
return secrets;
|
|
1249
|
+
}
|
|
1250
|
+
/**
|
|
1251
|
+
* Add a new integration to the sequence.
|
|
1252
|
+
* Extracts secrets and replaces them with env var references.
|
|
1253
|
+
*
|
|
1254
|
+
* @returns Record of extracted secrets (envVarName -> secretValue)
|
|
1255
|
+
*/
|
|
1256
|
+
function addIntegrationToSeq(doc, integrationsSeq, integration) {
|
|
1257
|
+
const secretPaths = getSecretFieldPaths(integration.type);
|
|
1258
|
+
const { metadata,...restIntegration } = integration;
|
|
1259
|
+
const integrationMap = doc.createNode(restIntegration);
|
|
1260
|
+
const metadataMap = doc.createNode({});
|
|
1261
|
+
const secrets = updateIntegrationMetadataMap({
|
|
1262
|
+
metadataMap,
|
|
1263
|
+
integrationId: integration.id,
|
|
1264
|
+
integrationMetadata: metadata,
|
|
1265
|
+
secretPaths
|
|
1266
|
+
});
|
|
1267
|
+
integrationMap.setIn(["metadata"], metadataMap);
|
|
1268
|
+
integrationsSeq.add(integrationMap);
|
|
1269
|
+
return secrets;
|
|
1270
|
+
}
|
|
1271
|
+
/**
|
|
1272
|
+
* Create a new Document with the schema comment.
|
|
1273
|
+
*/
|
|
1274
|
+
function createNewDocument() {
|
|
1275
|
+
const doc = (0, yaml.parseDocument)("integrations: []\n", { version: "1.2" });
|
|
1276
|
+
doc.commentBefore = SCHEMA_COMMENT;
|
|
1277
|
+
const integrations = doc.get("integrations", true);
|
|
1278
|
+
if ((0, yaml.isSeq)(integrations)) integrations.flow = false;
|
|
1279
|
+
return doc;
|
|
1280
|
+
}
|
|
1281
|
+
/**
|
|
1282
|
+
* Merge processed integrations into the document.
|
|
1283
|
+
* Updates existing entries in-place (preserving comments) and adds new ones.
|
|
1284
|
+
* Extracts secrets and replaces them with env var references during the merge.
|
|
1285
|
+
* Preserves custom environment variable names from existing entries.
|
|
1286
|
+
*
|
|
1287
|
+
* @returns Object containing existing count and extracted secrets
|
|
1288
|
+
*/
|
|
1289
|
+
function mergeProcessedIntegrations(doc, integrationsSeq, processedIntegrations) {
|
|
1290
|
+
const existingCount = integrationsSeq.items.length;
|
|
1291
|
+
let updatedCount = 0;
|
|
1292
|
+
let newCount = 0;
|
|
1293
|
+
const secrets = {};
|
|
1294
|
+
for (const databaseIntegration of processedIntegrations) {
|
|
1295
|
+
const existingIntegration = integrationsSeq.items.find((item) => {
|
|
1296
|
+
if (!(0, yaml.isMap)(item)) return false;
|
|
1297
|
+
return item.get("id") === databaseIntegration.id;
|
|
1298
|
+
});
|
|
1299
|
+
if (existingIntegration != null) {
|
|
1300
|
+
const extractedSecrets = updateIntegrationInDocument(doc, existingIntegration, databaseIntegration);
|
|
1301
|
+
Object.assign(secrets, extractedSecrets);
|
|
1302
|
+
updatedCount++;
|
|
1303
|
+
} else {
|
|
1304
|
+
const extractedSecrets = addIntegrationToSeq(doc, integrationsSeq, databaseIntegration);
|
|
1305
|
+
Object.assign(secrets, extractedSecrets);
|
|
1306
|
+
newCount++;
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
return {
|
|
1310
|
+
secrets,
|
|
1311
|
+
stats: {
|
|
1312
|
+
existingCount,
|
|
1313
|
+
newCount,
|
|
1314
|
+
updatedCount
|
|
1315
|
+
}
|
|
1316
|
+
};
|
|
1317
|
+
}
|
|
1318
|
+
/**
|
|
1319
|
+
* Error thrown when an API integration fails schema validation during conversion.
|
|
1320
|
+
*/
|
|
1321
|
+
var InvalidIntegrationError = class extends Error {
|
|
1322
|
+
integrationId;
|
|
1323
|
+
constructor(integrationId, message) {
|
|
1324
|
+
super(message);
|
|
1325
|
+
this.name = "InvalidIntegrationError";
|
|
1326
|
+
this.integrationId = integrationId;
|
|
1327
|
+
}
|
|
1328
|
+
};
|
|
1329
|
+
/**
|
|
1330
|
+
* Convert API integrations to DatabaseIntegrationConfig.
|
|
1331
|
+
* Validates each integration against the schema and collects errors for invalid ones.
|
|
1332
|
+
*
|
|
1333
|
+
* Shared by both `integrations pull` (via mergeApiIntegrationsIntoDocument) and `run` (for on-the-fly fetching).
|
|
1334
|
+
*
|
|
1335
|
+
* @param apiIntegrations - Integrations fetched from the API
|
|
1336
|
+
* @returns Object containing validated integrations and any validation errors
|
|
1337
|
+
*/
|
|
1338
|
+
function convertApiIntegrations(apiIntegrations) {
|
|
1339
|
+
const integrations = [];
|
|
1340
|
+
const errors = [];
|
|
1341
|
+
for (const apiIntegration of apiIntegrations) {
|
|
1342
|
+
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
|
|
1343
|
+
if (!config.success) {
|
|
1344
|
+
errors.push(new InvalidIntegrationError(apiIntegration.id, "Invalid integration returned by API."));
|
|
1345
|
+
continue;
|
|
1346
|
+
}
|
|
1347
|
+
integrations.push(config.data);
|
|
1348
|
+
}
|
|
1349
|
+
return {
|
|
1350
|
+
integrations,
|
|
1351
|
+
errors
|
|
1352
|
+
};
|
|
1353
|
+
}
|
|
1354
|
+
/**
|
|
1355
|
+
* Merge API integrations into an existing document (or create a new one).
|
|
1356
|
+
* Extracts secrets and replaces them with env var references during the merge.
|
|
1357
|
+
* Preserves custom environment variable names from existing entries.
|
|
1358
|
+
* Invalid or unsupported integrations are silently skipped.
|
|
1359
|
+
*
|
|
1360
|
+
* @param doc - The YAML document to merge into
|
|
1361
|
+
* @param apiIntegrations - Integrations fetched from the API
|
|
1362
|
+
* @returns The extracted secrets and merge statistics
|
|
1363
|
+
*/
|
|
1364
|
+
function mergeApiIntegrationsIntoDocument(doc, apiIntegrations) {
|
|
1365
|
+
const integrationsSeq = getOrCreateIntegrationsFromDocument(doc);
|
|
1366
|
+
const skipped = [];
|
|
1367
|
+
return {
|
|
1368
|
+
...mergeProcessedIntegrations(doc, integrationsSeq, apiIntegrations.reduce((acc, apiIntegration) => {
|
|
1369
|
+
const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
|
|
1370
|
+
if (!config.success) {
|
|
1371
|
+
skipped.push({
|
|
1372
|
+
id: apiIntegration.id,
|
|
1373
|
+
name: apiIntegration.name,
|
|
1374
|
+
type: apiIntegration.type,
|
|
1375
|
+
issues: config.error.issues
|
|
1376
|
+
});
|
|
1377
|
+
return acc;
|
|
1378
|
+
}
|
|
1379
|
+
acc.push(config.data);
|
|
1380
|
+
return acc;
|
|
1381
|
+
}, [])),
|
|
1382
|
+
skipped
|
|
1383
|
+
};
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
//#endregion
|
|
1387
|
+
//#region src/loading/merge-into-yaml.ts
|
|
1388
|
+
/**
|
|
1389
|
+
* Merge API integrations into existing integrations YAML content (or create a new
|
|
1390
|
+
* document), returning the serialized YAML, the extracted secrets, and merge stats.
|
|
1391
|
+
*
|
|
1392
|
+
* This is a string-in / string-out convenience that mirrors what `deepnote
|
|
1393
|
+
* integrations pull` does to the YAML file — usable without touching the `yaml`
|
|
1394
|
+
* Document API. Callers persist `content` and `secrets` however they like (e.g.
|
|
1395
|
+
* the CLI writes files; the VS Code extension can write via `workspace.fs`).
|
|
1396
|
+
*
|
|
1397
|
+
* @param existingContent - Current YAML content, or `null` if the file doesn't exist yet
|
|
1398
|
+
* @param apiIntegrations - Integrations fetched from the API
|
|
1399
|
+
*/
|
|
1400
|
+
function mergeApiIntegrationsIntoYaml(existingContent, apiIntegrations) {
|
|
1401
|
+
const doc = (existingContent != null ? parseIntegrationsDocument(existingContent) : null) ?? createNewDocument();
|
|
1402
|
+
const { secrets, stats, skipped } = mergeApiIntegrationsIntoDocument(doc, apiIntegrations);
|
|
1403
|
+
if (doc.commentBefore == null || !doc.commentBefore.includes("yaml-language-server")) doc.commentBefore = SCHEMA_COMMENT;
|
|
1404
|
+
return {
|
|
1405
|
+
content: serializeIntegrationsDocument(doc),
|
|
1406
|
+
secrets,
|
|
1407
|
+
stats,
|
|
1408
|
+
skipped
|
|
1409
|
+
};
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
//#endregion
|
|
1413
|
+
//#region src/loading/parse-integrations.ts
|
|
1414
|
+
/**
|
|
1415
|
+
* Format Zod validation errors into ValidationIssue format.
|
|
1416
|
+
*/
|
|
1417
|
+
function formatZodIssues(issues, pathPrefix) {
|
|
1418
|
+
return issues.map((issue) => ({
|
|
1419
|
+
path: [pathPrefix, ...issue.path].filter(Boolean).join("."),
|
|
1420
|
+
message: issue.message,
|
|
1421
|
+
code: issue.code
|
|
1422
|
+
}));
|
|
1423
|
+
}
|
|
1424
|
+
/**
|
|
1425
|
+
* Parse integrations YAML content gracefully.
|
|
1426
|
+
* Returns all valid integrations and collects validation issues for invalid ones.
|
|
1427
|
+
*
|
|
1428
|
+
* This is the content-accepting, environment-agnostic core (no filesystem or
|
|
1429
|
+
* `process.env` access) — callers supply the YAML text and the variable map used
|
|
1430
|
+
* to resolve `env:` references. The Node-only `parseIntegrationsFile` wraps this
|
|
1431
|
+
* to read from disk and default the map to `process.env`.
|
|
1432
|
+
*
|
|
1433
|
+
* @returns Object containing valid integrations and any validation issues
|
|
1434
|
+
*/
|
|
1435
|
+
function parseIntegrations({ yaml: yaml$1, env = {} }) {
|
|
1436
|
+
const emptyIntegrations = [];
|
|
1437
|
+
const issues = [];
|
|
1438
|
+
let parsed;
|
|
1439
|
+
try {
|
|
1440
|
+
parsed = (0, __deepnote_blocks.parseYaml)(yaml$1);
|
|
1441
|
+
} catch (error) {
|
|
1442
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1443
|
+
issues.push({
|
|
1444
|
+
path: "",
|
|
1445
|
+
message: `Invalid YAML in integrations file: ${message}`,
|
|
1446
|
+
code: "yaml_parse_error"
|
|
1447
|
+
});
|
|
1448
|
+
return {
|
|
1449
|
+
integrations: emptyIntegrations,
|
|
1450
|
+
issues
|
|
1451
|
+
};
|
|
1452
|
+
}
|
|
1453
|
+
if (parsed === null || parsed === void 0) return {
|
|
1454
|
+
integrations: emptyIntegrations,
|
|
1455
|
+
issues
|
|
1456
|
+
};
|
|
1457
|
+
const fileResult = baseIntegrationsFileSchema.safeParse(parsed);
|
|
1458
|
+
if (!fileResult.success) {
|
|
1459
|
+
issues.push(...formatZodIssues(fileResult.error.issues, ""));
|
|
1460
|
+
return {
|
|
1461
|
+
integrations: emptyIntegrations,
|
|
1462
|
+
issues
|
|
1463
|
+
};
|
|
1464
|
+
}
|
|
1465
|
+
const entries = fileResult.data.integrations;
|
|
1466
|
+
const integrations = [];
|
|
1467
|
+
for (let i = 0; i < entries.length; i++) {
|
|
1468
|
+
const entry = entries[i];
|
|
1469
|
+
const pathPrefix = `integrations[${i}]`;
|
|
1470
|
+
const entryId = zod.z.string().safeParse(entry.id).data;
|
|
1471
|
+
const integrationLabel = zod.z.string().safeParse(entry.name).data || entryId;
|
|
1472
|
+
let resolvedEntry;
|
|
1473
|
+
try {
|
|
1474
|
+
resolvedEntry = resolveEnvVarRefsFromMap(entry, env);
|
|
1475
|
+
} catch (error) {
|
|
1476
|
+
if (error instanceof EnvVarResolutionError) {
|
|
1477
|
+
const context = integrationLabel ? `Integration "${integrationLabel}": ` : "";
|
|
1478
|
+
issues.push({
|
|
1479
|
+
path: error.path ? `${pathPrefix}.${error.path}` : pathPrefix,
|
|
1480
|
+
message: `${context}${error.message}`,
|
|
1481
|
+
code: "env_var_not_defined"
|
|
1482
|
+
});
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1485
|
+
throw error;
|
|
1486
|
+
}
|
|
1487
|
+
const result = databaseIntegrationConfigSchema.safeParse(resolvedEntry);
|
|
1488
|
+
if (result.success) integrations.push(result.data);
|
|
1489
|
+
else {
|
|
1490
|
+
const formattedIssues = formatZodIssues(result.error.issues, pathPrefix);
|
|
1491
|
+
if (formattedIssues.length > 0 && integrationLabel) formattedIssues[0].message = `Integration "${integrationLabel}": ${formattedIssues[0].message}`;
|
|
1492
|
+
issues.push(...formattedIssues);
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
return {
|
|
1496
|
+
integrations,
|
|
1497
|
+
issues
|
|
1498
|
+
};
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
//#endregion
|
|
1502
|
+
exports.ApiError = ApiError;
|
|
959
1503
|
exports.AwsAuthMethods = AwsAuthMethods;
|
|
1504
|
+
exports.BUILTIN_INTEGRATIONS = BUILTIN_INTEGRATIONS;
|
|
960
1505
|
exports.BigQueryAuthMethods = BigQueryAuthMethods;
|
|
961
1506
|
exports.BigQueryServiceAccountParseError = BigQueryServiceAccountParseError;
|
|
1507
|
+
exports.DEFAULT_API_URL = DEFAULT_API_URL;
|
|
1508
|
+
exports.DEFAULT_ENV_FILE = DEFAULT_ENV_FILE;
|
|
1509
|
+
exports.DEFAULT_INTEGRATIONS_FILE = DEFAULT_INTEGRATIONS_FILE;
|
|
962
1510
|
exports.DatabaseAuthMethods = DatabaseAuthMethods;
|
|
1511
|
+
exports.ENV_VAR_REF_PREFIX = ENV_VAR_REF_PREFIX;
|
|
1512
|
+
exports.EnvVarResolutionError = EnvVarResolutionError;
|
|
1513
|
+
exports.InvalidIntegrationError = InvalidIntegrationError;
|
|
1514
|
+
exports.InvalidIntegrationsTypeError = InvalidIntegrationsTypeError;
|
|
1515
|
+
exports.JSON_SCHEMA_URL = JSON_SCHEMA_URL;
|
|
1516
|
+
exports.SCHEMA_COMMENT = SCHEMA_COMMENT;
|
|
963
1517
|
exports.SnowflakeAuthMethods = SnowflakeAuthMethods;
|
|
964
1518
|
exports.SpannerServiceAccountParseError = SpannerServiceAccountParseError;
|
|
965
1519
|
exports.TrinoAuthMethods = TrinoAuthMethods;
|
|
1520
|
+
exports.addIntegrationToSeq = addIntegrationToSeq;
|
|
1521
|
+
exports.apiIntegrationSchema = apiIntegrationSchema;
|
|
1522
|
+
exports.apiResponseSchema = apiResponseSchema;
|
|
1523
|
+
exports.baseIntegrationsFileSchema = baseIntegrationsFileSchema;
|
|
1524
|
+
exports.convertApiIntegrations = convertApiIntegrations;
|
|
1525
|
+
exports.createEnvVarRef = createEnvVarRef;
|
|
1526
|
+
exports.createNewDocument = createNewDocument;
|
|
966
1527
|
exports.databaseIntegrationConfigSchema = databaseIntegrationConfigSchema;
|
|
967
1528
|
exports.databaseIntegrationTypes = databaseIntegrationTypes;
|
|
968
1529
|
exports.databaseIntegrationTypesWithSslSupport = databaseIntegrationTypesWithSslSupport;
|
|
969
1530
|
exports.databaseMetadataSchemasByType = databaseMetadataSchemasByType;
|
|
1531
|
+
exports.extractEnvVarName = extractEnvVarName;
|
|
970
1532
|
exports.federatedAuthMethods = federatedAuthMethods;
|
|
1533
|
+
exports.fetchIntegrations = fetchIntegrations;
|
|
1534
|
+
exports.generateEnvVarName = generateEnvVarName;
|
|
971
1535
|
exports.getEnvironmentVariablesForIntegrations = getEnvironmentVariablesForIntegrations;
|
|
1536
|
+
exports.getOrCreateIntegrationMetadata = getOrCreateIntegrationMetadata;
|
|
1537
|
+
exports.getOrCreateIntegrationsFromDocument = getOrCreateIntegrationsFromDocument;
|
|
972
1538
|
exports.getSecretFieldPaths = getSecretFieldPaths;
|
|
973
1539
|
exports.getSnowflakeFederatedAuthSqlAlchemyInput = getSnowflakeFederatedAuthSqlAlchemyInput;
|
|
974
1540
|
exports.getSqlAlchemyInput = getSqlAlchemyInput;
|
|
1541
|
+
exports.getSqlEnvVarName = getSqlEnvVarName;
|
|
1542
|
+
exports.integrationsFileSchema = integrationsFileSchema;
|
|
975
1543
|
exports.isDatabaseIntegrationType = isDatabaseIntegrationType;
|
|
976
1544
|
exports.isDatabaseIntegrationTypeWithSslSupport = isDatabaseIntegrationTypeWithSslSupport;
|
|
1545
|
+
exports.isEnvVarRef = isEnvVarRef;
|
|
977
1546
|
exports.isFederatedAuthMetadata = isFederatedAuthMetadata;
|
|
978
1547
|
exports.isFederatedAuthMethod = isFederatedAuthMethod;
|
|
979
1548
|
exports.isSqlIntegrationType = isSqlIntegrationType;
|
|
980
|
-
exports.
|
|
1549
|
+
exports.mergeApiIntegrationsIntoDocument = mergeApiIntegrationsIntoDocument;
|
|
1550
|
+
exports.mergeApiIntegrationsIntoYaml = mergeApiIntegrationsIntoYaml;
|
|
1551
|
+
exports.mergeProcessedIntegrations = mergeProcessedIntegrations;
|
|
1552
|
+
exports.parseEnvVarRef = parseEnvVarRef;
|
|
1553
|
+
exports.parseIntegrations = parseIntegrations;
|
|
1554
|
+
exports.parseIntegrationsDocument = parseIntegrationsDocument;
|
|
1555
|
+
exports.resolveEnvVarRefsFromMap = resolveEnvVarRefsFromMap;
|
|
1556
|
+
exports.serializeIntegrationsDocument = serializeIntegrationsDocument;
|
|
1557
|
+
exports.sqlIntegrationTypes = sqlIntegrationTypes;
|
|
1558
|
+
exports.updateIntegrationInDocument = updateIntegrationInDocument;
|
|
1559
|
+
exports.updateIntegrationMetadataMap = updateIntegrationMetadataMap;
|