@deepnote/database-integrations 1.4.2 → 1.5.0

package/dist/index.cjs

@@ -23,6 +23,10 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 //#endregion
 let zod = require("zod");
 zod = __toESM(zod);
+let yaml = require("yaml");
+yaml = __toESM(yaml);
+let __deepnote_blocks = require("@deepnote/blocks");
+__deepnote_blocks = __toESM(__deepnote_blocks);
 
 //#region src/sql-integration-auth-methods.ts
 const AwsAuthMethods = {
@@ -97,6 +101,7 @@ const clickhouseMetadataSchema = zod.z.object({
 	caCertificateName: zod.z.string().optional(),
 	caCertificateText: zod.z.string().optional()
 });
+const cloudSqlMetadataSchema = zod.z.object({ service_account: zod.z.string() });
 const databricksMetadataSchema = zod.z.object({
 	host: zod.z.string(),
 	httpPath: zod.z.string(),
@@ -298,6 +303,7 @@ const databaseMetadataSchemasByType = {
 	"athena": athenaMetadataSchema,
 	"big-query": bigqueryMetadataSchema,
 	"clickhouse": clickhouseMetadataSchema,
+	"cloud-sql": cloudSqlMetadataSchema,
 	"databricks": databricksMetadataSchema,
 	"dremio": dremioMetadataSchema,
 	"mariadb": mariadbMetadataSchema,
@@ -323,12 +329,12 @@ const commonIntegrationConfig = zod.default.object({
 const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("alloydb"),
-		metadata: databaseMetadataSchemasByType["alloydb"],
+		metadata: databaseMetadataSchemasByType.alloydb,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("athena"),
-		metadata: databaseMetadataSchemasByType["athena"],
+		metadata: databaseMetadataSchemasByType.athena,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
@@ -338,42 +344,47 @@ const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("clickhouse"),
-		metadata: databaseMetadataSchemasByType["clickhouse"],
+		metadata: databaseMetadataSchemasByType.clickhouse,
+		federated_auth_method: zod.default.null().optional()
+	}),
+	commonIntegrationConfig.extend({
+		type: zod.default.literal("cloud-sql"),
+		metadata: databaseMetadataSchemasByType["cloud-sql"],
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("databricks"),
-		metadata: databaseMetadataSchemasByType["databricks"],
+		metadata: databaseMetadataSchemasByType.databricks,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("dremio"),
-		metadata: databaseMetadataSchemasByType["dremio"],
+		metadata: databaseMetadataSchemasByType.dremio,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("mariadb"),
-		metadata: databaseMetadataSchemasByType["mariadb"],
+		metadata: databaseMetadataSchemasByType.mariadb,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("materialize"),
-		metadata: databaseMetadataSchemasByType["materialize"],
+		metadata: databaseMetadataSchemasByType.materialize,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("mindsdb"),
-		metadata: databaseMetadataSchemasByType["mindsdb"],
+		metadata: databaseMetadataSchemasByType.mindsdb,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("mongodb"),
-		metadata: databaseMetadataSchemasByType["mongodb"],
+		metadata: databaseMetadataSchemasByType.mongodb,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("mysql"),
-		metadata: databaseMetadataSchemasByType["mysql"],
+		metadata: databaseMetadataSchemasByType.mysql,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
@@ -383,12 +394,12 @@ const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("pgsql"),
-		metadata: databaseMetadataSchemasByType["pgsql"],
+		metadata: databaseMetadataSchemasByType.pgsql,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("redshift"),
-		metadata: databaseMetadataSchemasByType["redshift"],
+		metadata: databaseMetadataSchemasByType.redshift,
 		federated_auth_method: zod.default.enum([
 			"username-and-password",
 			"iam-role",
@@ -397,7 +408,7 @@ const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("snowflake"),
-		metadata: databaseMetadataSchemasByType["snowflake"],
+		metadata: databaseMetadataSchemasByType.snowflake,
 		federated_auth_method: zod.default.enum([
 			"okta",
 			"snowflake",
@@ -409,7 +420,7 @@ const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("spanner"),
-		metadata: databaseMetadataSchemasByType["spanner"],
+		metadata: databaseMetadataSchemasByType.spanner,
 		federated_auth_method: zod.default.null().optional()
 	}),
 	commonIntegrationConfig.extend({
@@ -419,7 +430,7 @@ const databaseIntegrationConfigSchema = zod.default.discriminatedUnion("type", [
 	}),
 	commonIntegrationConfig.extend({
 		type: zod.default.literal("trino"),
-		metadata: databaseMetadataSchemasByType["trino"],
+		metadata: databaseMetadataSchemasByType.trino,
 		federated_auth_method: zod.default.enum(["trino-oauth", "password"]).optional().nullable()
 	})
 ]);
@@ -567,6 +578,7 @@ function getSqlAlchemyInput(integration, params) {
 		case "athena": return getAthenaSqlAlchemyInput(integration.metadata);
 		case "big-query": return getBigQuerySqlAlchemyInput(integration.metadata);
 		case "clickhouse": return getClickHouseSqlAlchemyInput(integration.id, params.projectRootDirectory, integration.metadata);
+		case "cloud-sql": return null;
 		case "databricks": return getDatabricksSqlAlchemyInput(integration.metadata);
 		case "dremio": return getDremioSqlAlchemyInput(integration.metadata);
 		case "mariadb":
@@ -861,6 +873,7 @@ const sqlIntegrationTypes = [
 	"athena",
 	"big-query",
 	"clickhouse",
+	"cloud-sql",
 	"databricks",
 	"dremio",
 	"mariadb",
@@ -899,24 +912,648 @@ function isDatabaseIntegrationTypeWithSslSupport(type) {
 }
 
 //#endregion
+//#region src/loading/api-error.ts
+/**
+* Error thrown when a Deepnote API request fails.
+*/
+var ApiError = class extends Error {
+	statusCode;
+	constructor(statusCode, message) {
+		super(message);
+		this.name = "ApiError";
+		this.statusCode = statusCode;
+	}
+};
+
+//#endregion
+//#region src/loading/constants.ts
+/** Built-in integrations that don't require external configuration. */
+const BUILTIN_INTEGRATIONS = new Set(["deepnote-dataframe-sql", "pandas-dataframe"]);
+/** Default `.env` file name for storing integration secrets. */
+const DEFAULT_ENV_FILE = ".env";
+/** Default integrations file name. */
+const DEFAULT_INTEGRATIONS_FILE = ".deepnote.env.yaml";
+/** Default Deepnote API base URL. */
+const DEFAULT_API_URL = "https://api.deepnote.com";
+
+//#endregion
+//#region src/loading/env-var-refs.ts
+const ENV_VAR_REF_PREFIX = "env:";
+function generateEnvVarName(integrationId, fieldPath) {
+	return `${integrationId.toUpperCase().replace(/-/g, "_")}__${fieldPath.toUpperCase()}`;
+}
+function parseEnvVarRef(value) {
+	if (typeof value !== "string") return null;
+	if (!value.startsWith(ENV_VAR_REF_PREFIX)) return null;
+	const varName = value.slice(4);
+	if (!varName) return null;
+	return {
+		prefix: "env",
+		varName
+	};
+}
+function createEnvVarRef(varName) {
+	return `${ENV_VAR_REF_PREFIX}${varName}`;
+}
+function isEnvVarRef(value) {
+	return typeof value === "string" && parseEnvVarRef(value) !== null;
+}
+function extractEnvVarName(value) {
+	if (typeof value !== "string") return null;
+	return parseEnvVarRef(value)?.varName ?? null;
+}
+/**
+* Error thrown when an environment variable reference cannot be resolved.
+*/
+var EnvVarResolutionError = class extends Error {
+	varName;
+	path;
+	constructor(varName, path) {
+		const pathInfo = path ? ` at "${path}"` : "";
+		super(`Environment variable "${varName}" is not defined${pathInfo}`);
+		this.name = "EnvVarResolutionError";
+		this.varName = varName;
+		this.path = path;
+	}
+};
+/**
+* Recursively resolves environment variable references in an object using an explicit variable map.
+* Replaces any string value matching `env:VAR_NAME` with the value from the provided vars map.
+*
+* @param obj - The object to process (can be any value: primitive, array, or object)
+* @param vars - Map of variable names to values (e.g. from a parsed .env file, process.env, or a secret store)
+* @param currentPath - Internal parameter for tracking the current path in the object (for error messages)
+* @returns A new object with all env var references resolved
+* @throws {EnvVarResolutionError} If an environment variable reference cannot be resolved
+*/
+function resolveEnvVarRefsFromMap(obj, vars, currentPath) {
+	if (obj === null || obj === void 0) return obj;
+	if (typeof obj === "string") {
+		const parsed = parseEnvVarRef(obj);
+		if (parsed) {
+			const envValue = vars[parsed.varName];
+			if (envValue === void 0) throw new EnvVarResolutionError(parsed.varName, currentPath);
+			return envValue;
+		}
+		return obj;
+	}
+	if (Array.isArray(obj)) return obj.map((item, index) => {
+		return resolveEnvVarRefsFromMap(item, vars, currentPath ? `${currentPath}[${index}]` : `[${index}]`);
+	});
+	if (typeof obj === "object") {
+		const result = {};
+		for (const [key, value] of Object.entries(obj)) result[key] = resolveEnvVarRefsFromMap(value, vars, currentPath ? `${currentPath}.${key}` : key);
+		return result;
+	}
+	return obj;
+}
+
+//#endregion
+//#region src/loading/fetch-integrations.ts
+/**
+* Schema for a single integration from the API.
+*/
+const apiIntegrationSchema = zod.z.object({
+	id: zod.z.string(),
+	name: zod.z.string(),
+	type: zod.z.string(),
+	metadata: zod.z.unknown(),
+	is_public: zod.z.boolean(),
+	created_at: zod.z.string(),
+	updated_at: zod.z.string(),
+	federated_auth_method: zod.z.string().nullable()
+}).passthrough();
+/**
+* Schema for the full API response.
+*/
+const apiResponseSchema = zod.z.object({ integrations: zod.z.array(apiIntegrationSchema) }).passthrough();
+/**
+* Fetch integrations from the Deepnote API.
+*
+* Uses the global `fetch`, so it works in Node (18+) and browser-like environments alike.
+*
+* @param baseUrl - The base URL of the Deepnote API
+* @param token - The authentication token
+* @param integrationIds - Optional list of integration IDs to fetch. When provided, only these integrations are returned.
+* @returns Array of integrations from the API
+* @throws ApiError if the request fails
+*/
+async function fetchIntegrations(baseUrl, token, integrationIds) {
+	const endpoint = new URL(`${baseUrl}/v2/integrations`);
+	endpoint.searchParams.set("includeMetadata", "true");
+	if (integrationIds && integrationIds.length > 0) endpoint.searchParams.set("integrationIds", integrationIds.join(","));
+	const url = endpoint.toString();
+	const response = await fetch(url, {
+		method: "GET",
+		headers: {
+			Authorization: `Bearer ${token}`,
+			"Content-Type": "application/json"
+		}
+	});
+	if (!response.ok) {
+		if (response.status === 401) throw new ApiError(401, "Authentication failed. Please check your API token.");
+		if (response.status === 403) throw new ApiError(403, "Access denied. You may not have permission to access integrations.");
+		throw new ApiError(response.status, `API request failed with status ${response.status}: ${response.statusText}`);
+	}
+	const json = await response.json();
+	const parseResult = apiResponseSchema.safeParse(json);
+	if (!parseResult.success) throw new Error(`Invalid API response: ${parseResult.error.message}`);
+	return parseResult.data.integrations;
+}
+
+//#endregion
+//#region src/loading/integrations-document.ts
+/**
+* Parse integrations YAML content into a mutable `yaml` Document that preserves
+* comments and formatting. Returns `null` for empty content.
+*
+* This is the content-accepting counterpart of the Node-only `readIntegrationsDocument`.
+*/
+function parseIntegrationsDocument(content) {
+	if (!content.trim()) return null;
+	return (0, yaml.parseDocument)(content, {
+		strict: true,
+		version: "1.2"
+	});
+}
+/**
+* Serialize an integrations Document back to YAML text, preserving comments and
+* formatting and avoiding line wrapping.
+*/
+function serializeIntegrationsDocument(doc) {
+	return doc.toString({ lineWidth: 0 });
+}
+
+//#endregion
+//#region src/loading/integrations-file-schema.ts
+/**
+* Schema for the integrations YAML file structure.
+* Uses loose validation to accept any array of objects,
+* allowing per-entry validation to report specific issues.
+*
+* Example file format:
+* ```yaml
+* integrations:
+*   - id: my-postgres
+*     name: My PostgreSQL
+*     type: pgsql
+*     metadata:
+*       host: localhost
+*       ...
+* ```
+*/
+const baseIntegrationsFileSchema = zod.z.object({ integrations: zod.z.array(zod.z.record(zod.z.unknown())).optional().default([]) });
+const integrationsFileSchema = zod.z.object({ integrations: zod.z.array(databaseIntegrationConfigSchema) });
+
+//#endregion
+//#region src/secret-field-paths.ts
+/**
+* All secret fields for each integration type.
+* This includes all possible secret fields regardless of auth method.
+*
+* Note: For integrations with multiple auth methods (e.g., snowflake, bigquery),
+* this includes all secret fields from all auth variants.
+*/
+const SECRET_PATHS = {
+	alloydb: ["password", "caCertificateText"],
+	mariadb: ["password", "caCertificateText"],
+	materialize: ["password", "caCertificateText"],
+	mindsdb: ["password", "caCertificateText"],
+	mysql: ["password", "caCertificateText"],
+	pgsql: ["password", "caCertificateText"],
+	"sql-server": ["password"],
+	clickhouse: ["password", "caCertificateText"],
+	databricks: ["token"],
+	dremio: ["token"],
+	athena: ["secret_access_key"],
+	spanner: ["service_account"],
+	"cloud-sql": ["service_account"],
+	mongodb: [
+		"password",
+		"connection_string",
+		"rawConnectionString",
+		"caCertificateText"
+	],
+	"pandas-dataframe": [],
+	"big-query": ["service_account", "clientSecret"],
+	snowflake: [
+		"password",
+		"clientSecret",
+		"privateKey",
+		"privateKeyPassphrase"
+	],
+	redshift: ["password", "caCertificateText"],
+	trino: [
+		"password",
+		"clientSecret",
+		"caCertificateText"
+	]
+};
+/**
+* Get the list of secret field paths for a given integration type.
+*
+* @param type - The integration type (e.g., 'pgsql', 'snowflake')
+* @returns Array of field names that contain secrets
+*
+* @example
+* getSecretFieldPaths('pgsql') // => ['password', 'caCertificateText']
+* getSecretFieldPaths('snowflake') // => ['password', 'clientSecret', 'privateKey', 'privateKeyPassphrase']
+*/
+function getSecretFieldPaths(type) {
+	return SECRET_PATHS[type] ?? [];
+}
+
+//#endregion
+//#region src/loading/merge-integrations.ts
+/**
+* JSON schema URL for the integrations file.
+* TODO - change to main branch when the PR is merged into main
+*/
+const JSON_SCHEMA_URL = "https://raw.githubusercontent.com/deepnote/deepnote/refs/heads/tk/integrations-config-file-schema/json-schemas/integrations-file-schema.json";
+/**
+* Schema comment for the YAML file.
+*/
+const SCHEMA_COMMENT = `yaml-language-server: $schema=${JSON_SCHEMA_URL}`;
+/**
+* Error thrown when the integrations property has an invalid type.
+*/
+var InvalidIntegrationsTypeError = class extends Error {
+	constructor() {
+		super(`Invalid 'integrations' property: expected a list`);
+		this.name = "InvalidIntegrationsTypeError";
+	}
+};
+/**
+* Get the integrations sequence from a YAML Document, creating an empty one if it doesn't exist.
+* If the integrations property is missing or null, creates an empty array and returns a reference to it.
+* Throws InvalidIntegrationsTypeError if integrations exists but is not an array.
+*/
+function getOrCreateIntegrationsFromDocument(doc) {
+	const integrations = doc.get("integrations");
+	if (integrations == null) {
+		const emptySeq = doc.createNode([]);
+		emptySeq.flow = false;
+		doc.set("integrations", emptySeq);
+		return emptySeq;
+	}
+	if (!(0, yaml.isSeq)(integrations)) throw new InvalidIntegrationsTypeError();
+	return integrations;
+}
+function getOrCreateIntegrationMetadata(doc, integrationMap) {
+	const existingMetadata = integrationMap.get("metadata", true);
+	if ((0, yaml.isMap)(existingMetadata)) return existingMetadata;
+	const metadata = doc.createNode({});
+	integrationMap.set("metadata", metadata);
+	return metadata;
+}
+function updateIntegrationMetadataMap({ metadataMap, integrationId, integrationMetadata, secretPaths }) {
+	const secrets = {};
+	for (const [key, value] of Object.entries(integrationMetadata)) {
+		if (value === void 0) continue;
+		if (value === null) {
+			metadataMap.set(key, null);
+			continue;
+		}
+		if (!secretPaths.includes(key)) {
+			metadataMap.set(key, value);
+			continue;
+		}
+		const envVarName = extractEnvVarName(metadataMap.get(key)) ?? generateEnvVarName(integrationId, key);
+		secrets[envVarName] = String(value);
+		metadataMap.set(key, createEnvVarRef(envVarName));
+	}
+	return secrets;
+}
+/**
+* Update an existing integration entry in the document.
+* Preserves any comments or extra fields while updating known fields.
+* Extracts secrets and replaces them with env var references, preserving custom env var names.
+*
+* @returns Record of extracted secrets (envVarName -> secretValue)
+*/
+function updateIntegrationInDocument(doc, integrationMap, integration) {
+	const { id, name, type, metadata, federated_auth_method,...rest } = integration;
+	integrationMap.set("id", id);
+	integrationMap.set("name", name);
+	integrationMap.set("type", type);
+	const secretPaths = getSecretFieldPaths(type);
+	const secrets = updateIntegrationMetadataMap({
+		metadataMap: getOrCreateIntegrationMetadata(doc, integrationMap),
+		integrationId: id,
+		integrationMetadata: metadata,
+		secretPaths
+	});
+	if (federated_auth_method !== void 0) integrationMap.set("federated_auth_method", federated_auth_method);
+	else integrationMap.delete("federated_auth_method");
+	return secrets;
+}
+/**
+* Add a new integration to the sequence.
+* Extracts secrets and replaces them with env var references.
+*
+* @returns Record of extracted secrets (envVarName -> secretValue)
+*/
+function addIntegrationToSeq(doc, integrationsSeq, integration) {
+	const secretPaths = getSecretFieldPaths(integration.type);
+	const { metadata,...restIntegration } = integration;
+	const integrationMap = doc.createNode(restIntegration);
+	const metadataMap = doc.createNode({});
+	const secrets = updateIntegrationMetadataMap({
+		metadataMap,
+		integrationId: integration.id,
+		integrationMetadata: metadata,
+		secretPaths
+	});
+	integrationMap.setIn(["metadata"], metadataMap);
+	integrationsSeq.add(integrationMap);
+	return secrets;
+}
+/**
+* Create a new Document with the schema comment.
+*/
+function createNewDocument() {
+	const doc = (0, yaml.parseDocument)("integrations: []\n", { version: "1.2" });
+	doc.commentBefore = SCHEMA_COMMENT;
+	const integrations = doc.get("integrations", true);
+	if ((0, yaml.isSeq)(integrations)) integrations.flow = false;
+	return doc;
+}
+/**
+* Merge processed integrations into the document.
+* Updates existing entries in-place (preserving comments) and adds new ones.
+* Extracts secrets and replaces them with env var references during the merge.
+* Preserves custom environment variable names from existing entries.
+*
+* @returns Object containing existing count and extracted secrets
+*/
+function mergeProcessedIntegrations(doc, integrationsSeq, processedIntegrations) {
+	const existingCount = integrationsSeq.items.length;
+	let updatedCount = 0;
+	let newCount = 0;
+	const secrets = {};
+	for (const databaseIntegration of processedIntegrations) {
+		const existingIntegration = integrationsSeq.items.find((item) => {
+			if (!(0, yaml.isMap)(item)) return false;
+			return item.get("id") === databaseIntegration.id;
+		});
+		if (existingIntegration != null) {
+			const extractedSecrets = updateIntegrationInDocument(doc, existingIntegration, databaseIntegration);
+			Object.assign(secrets, extractedSecrets);
+			updatedCount++;
+		} else {
+			const extractedSecrets = addIntegrationToSeq(doc, integrationsSeq, databaseIntegration);
+			Object.assign(secrets, extractedSecrets);
+			newCount++;
+		}
+	}
+	return {
+		secrets,
+		stats: {
+			existingCount,
+			newCount,
+			updatedCount
+		}
+	};
+}
+/**
+* Error thrown when an API integration fails schema validation during conversion.
+*/
+var InvalidIntegrationError = class extends Error {
+	integrationId;
+	constructor(integrationId, message) {
+		super(message);
+		this.name = "InvalidIntegrationError";
+		this.integrationId = integrationId;
+	}
+};
+/**
+* Convert API integrations to DatabaseIntegrationConfig.
+* Validates each integration against the schema and collects errors for invalid ones.
+*
+* Shared by both `integrations pull` (via mergeApiIntegrationsIntoDocument) and `run` (for on-the-fly fetching).
+*
+* @param apiIntegrations - Integrations fetched from the API
+* @returns Object containing validated integrations and any validation errors
+*/
+function convertApiIntegrations(apiIntegrations) {
+	const integrations = [];
+	const errors = [];
+	for (const apiIntegration of apiIntegrations) {
+		const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
+		if (!config.success) {
+			errors.push(new InvalidIntegrationError(apiIntegration.id, "Invalid integration returned by API."));
+			continue;
+		}
+		integrations.push(config.data);
+	}
+	return {
+		integrations,
+		errors
+	};
+}
+/**
+* Merge API integrations into an existing document (or create a new one).
+* Extracts secrets and replaces them with env var references during the merge.
+* Preserves custom environment variable names from existing entries.
+* Invalid or unsupported integrations are silently skipped.
+*
+* @param doc - The YAML document to merge into
+* @param apiIntegrations - Integrations fetched from the API
+* @returns The extracted secrets and merge statistics
+*/
+function mergeApiIntegrationsIntoDocument(doc, apiIntegrations) {
+	const integrationsSeq = getOrCreateIntegrationsFromDocument(doc);
+	const skipped = [];
+	return {
+		...mergeProcessedIntegrations(doc, integrationsSeq, apiIntegrations.reduce((acc, apiIntegration) => {
+			const config = databaseIntegrationConfigSchema.safeParse(apiIntegration);
+			if (!config.success) {
+				skipped.push({
+					id: apiIntegration.id,
+					name: apiIntegration.name,
+					type: apiIntegration.type,
+					issues: config.error.issues
+				});
+				return acc;
+			}
+			acc.push(config.data);
+			return acc;
+		}, [])),
+		skipped
+	};
+}
+
+//#endregion
+//#region src/loading/merge-into-yaml.ts
+/**
+* Merge API integrations into existing integrations YAML content (or create a new
+* document), returning the serialized YAML, the extracted secrets, and merge stats.
+*
+* This is a string-in / string-out convenience that mirrors what `deepnote
+* integrations pull` does to the YAML file — usable without touching the `yaml`
+* Document API. Callers persist `content` and `secrets` however they like (e.g.
+* the CLI writes files; the VS Code extension can write via `workspace.fs`).
+*
+* @param existingContent - Current YAML content, or `null` if the file doesn't exist yet
+* @param apiIntegrations - Integrations fetched from the API
+*/
+function mergeApiIntegrationsIntoYaml(existingContent, apiIntegrations) {
+	const doc = (existingContent != null ? parseIntegrationsDocument(existingContent) : null) ?? createNewDocument();
+	const { secrets, stats, skipped } = mergeApiIntegrationsIntoDocument(doc, apiIntegrations);
+	if (doc.commentBefore == null || !doc.commentBefore.includes("yaml-language-server")) doc.commentBefore = SCHEMA_COMMENT;
+	return {
+		content: serializeIntegrationsDocument(doc),
+		secrets,
+		stats,
+		skipped
+	};
+}
+
+//#endregion
+//#region src/loading/parse-integrations.ts
+/**
+* Format Zod validation errors into ValidationIssue format.
+*/
+function formatZodIssues(issues, pathPrefix) {
+	return issues.map((issue) => ({
+		path: [pathPrefix, ...issue.path].filter(Boolean).join("."),
+		message: issue.message,
+		code: issue.code
+	}));
+}
+/**
+* Parse integrations YAML content gracefully.
+* Returns all valid integrations and collects validation issues for invalid ones.
+*
+* This is the content-accepting, environment-agnostic core (no filesystem or
+* `process.env` access) — callers supply the YAML text and the variable map used
+* to resolve `env:` references. The Node-only `parseIntegrationsFile` wraps this
+* to read from disk and default the map to `process.env`.
+*
+* @returns Object containing valid integrations and any validation issues
+*/
+function parseIntegrations({ yaml: yaml$1, env = {} }) {
+	const emptyIntegrations = [];
+	const issues = [];
+	let parsed;
+	try {
+		parsed = (0, __deepnote_blocks.parseYaml)(yaml$1);
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		issues.push({
+			path: "",
+			message: `Invalid YAML in integrations file: ${message}`,
+			code: "yaml_parse_error"
+		});
+		return {
+			integrations: emptyIntegrations,
+			issues
+		};
+	}
+	if (parsed === null || parsed === void 0) return {
+		integrations: emptyIntegrations,
+		issues
+	};
+	const fileResult = baseIntegrationsFileSchema.safeParse(parsed);
+	if (!fileResult.success) {
+		issues.push(...formatZodIssues(fileResult.error.issues, ""));
+		return {
+			integrations: emptyIntegrations,
+			issues
+		};
+	}
+	const entries = fileResult.data.integrations;
+	const integrations = [];
+	for (let i = 0; i < entries.length; i++) {
+		const entry = entries[i];
+		const pathPrefix = `integrations[${i}]`;
+		const entryId = zod.z.string().safeParse(entry.id).data;
+		const integrationLabel = zod.z.string().safeParse(entry.name).data || entryId;
+		let resolvedEntry;
+		try {
+			resolvedEntry = resolveEnvVarRefsFromMap(entry, env);
+		} catch (error) {
+			if (error instanceof EnvVarResolutionError) {
+				const context = integrationLabel ? `Integration "${integrationLabel}": ` : "";
+				issues.push({
+					path: error.path ? `${pathPrefix}.${error.path}` : pathPrefix,
+					message: `${context}${error.message}`,
+					code: "env_var_not_defined"
+				});
+				continue;
+			}
+			throw error;
+		}
+		const result = databaseIntegrationConfigSchema.safeParse(resolvedEntry);
+		if (result.success) integrations.push(result.data);
+		else {
+			const formattedIssues = formatZodIssues(result.error.issues, pathPrefix);
+			if (formattedIssues.length > 0 && integrationLabel) formattedIssues[0].message = `Integration "${integrationLabel}": ${formattedIssues[0].message}`;
+			issues.push(...formattedIssues);
+		}
+	}
+	return {
+		integrations,
+		issues
+	};
+}
+
+//#endregion
+exports.ApiError = ApiError;
 exports.AwsAuthMethods = AwsAuthMethods;
+exports.BUILTIN_INTEGRATIONS = BUILTIN_INTEGRATIONS;
 exports.BigQueryAuthMethods = BigQueryAuthMethods;
 exports.BigQueryServiceAccountParseError = BigQueryServiceAccountParseError;
+exports.DEFAULT_API_URL = DEFAULT_API_URL;
+exports.DEFAULT_ENV_FILE = DEFAULT_ENV_FILE;
+exports.DEFAULT_INTEGRATIONS_FILE = DEFAULT_INTEGRATIONS_FILE;
 exports.DatabaseAuthMethods = DatabaseAuthMethods;
+exports.ENV_VAR_REF_PREFIX = ENV_VAR_REF_PREFIX;
+exports.EnvVarResolutionError = EnvVarResolutionError;
+exports.InvalidIntegrationError = InvalidIntegrationError;
+exports.InvalidIntegrationsTypeError = InvalidIntegrationsTypeError;
+exports.JSON_SCHEMA_URL = JSON_SCHEMA_URL;
+exports.SCHEMA_COMMENT = SCHEMA_COMMENT;
 exports.SnowflakeAuthMethods = SnowflakeAuthMethods;
 exports.SpannerServiceAccountParseError = SpannerServiceAccountParseError;
 exports.TrinoAuthMethods = TrinoAuthMethods;
+exports.addIntegrationToSeq = addIntegrationToSeq;
+exports.apiIntegrationSchema = apiIntegrationSchema;
+exports.apiResponseSchema = apiResponseSchema;
+exports.baseIntegrationsFileSchema = baseIntegrationsFileSchema;
+exports.convertApiIntegrations = convertApiIntegrations;
+exports.createEnvVarRef = createEnvVarRef;
+exports.createNewDocument = createNewDocument;
 exports.databaseIntegrationConfigSchema = databaseIntegrationConfigSchema;
 exports.databaseIntegrationTypes = databaseIntegrationTypes;
 exports.databaseIntegrationTypesWithSslSupport = databaseIntegrationTypesWithSslSupport;
 exports.databaseMetadataSchemasByType = databaseMetadataSchemasByType;
+exports.extractEnvVarName = extractEnvVarName;
 exports.federatedAuthMethods = federatedAuthMethods;
+exports.fetchIntegrations = fetchIntegrations;
+exports.generateEnvVarName = generateEnvVarName;
 exports.getEnvironmentVariablesForIntegrations = getEnvironmentVariablesForIntegrations;
+exports.getOrCreateIntegrationMetadata = getOrCreateIntegrationMetadata;
+exports.getOrCreateIntegrationsFromDocument = getOrCreateIntegrationsFromDocument;
+exports.getSecretFieldPaths = getSecretFieldPaths;
 exports.getSnowflakeFederatedAuthSqlAlchemyInput = getSnowflakeFederatedAuthSqlAlchemyInput;
 exports.getSqlAlchemyInput = getSqlAlchemyInput;
+exports.getSqlEnvVarName = getSqlEnvVarName;
+exports.integrationsFileSchema = integrationsFileSchema;
 exports.isDatabaseIntegrationType = isDatabaseIntegrationType;
 exports.isDatabaseIntegrationTypeWithSslSupport = isDatabaseIntegrationTypeWithSslSupport;
+exports.isEnvVarRef = isEnvVarRef;
 exports.isFederatedAuthMetadata = isFederatedAuthMetadata;
 exports.isFederatedAuthMethod = isFederatedAuthMethod;
 exports.isSqlIntegrationType = isSqlIntegrationType;
-exports.sqlIntegrationTypes = sqlIntegrationTypes;
+exports.mergeApiIntegrationsIntoDocument = mergeApiIntegrationsIntoDocument;
+exports.mergeApiIntegrationsIntoYaml = mergeApiIntegrationsIntoYaml;
+exports.mergeProcessedIntegrations = mergeProcessedIntegrations;
+exports.parseEnvVarRef = parseEnvVarRef;
+exports.parseIntegrations = parseIntegrations;
+exports.parseIntegrationsDocument = parseIntegrationsDocument;
+exports.resolveEnvVarRefsFromMap = resolveEnvVarRefsFromMap;
+exports.serializeIntegrationsDocument = serializeIntegrationsDocument;
+exports.sqlIntegrationTypes = sqlIntegrationTypes;
+exports.updateIntegrationInDocument = updateIntegrationInDocument;
+exports.updateIntegrationMetadataMap = updateIntegrationMetadataMap;