@ingestron/pack-databricks 0.0.0-stage → 1.0.1

package/dist/bundle.js

@@ -0,0 +1,136 @@
+import { stringify } from "yaml";
+/** `databricks.yml`: one target, named after the environment. */
+export function bundleFile(model) {
+    const { platform } = model;
+    const target = { mode: platform.mode, default: true };
+    if (platform.workspace.host || platform.workspace.root_path)
+        target.workspace = { ...platform.workspace };
+    if (platform.run_as)
+        target.run_as = platform.run_as;
+    if (platform.permissions.length > 0)
+        target.permissions = platform.permissions;
+    return {
+        bundle: { name: model.project.id },
+        include: ["resources/*.yml"],
+        targets: { [model.project.environment]: target },
+    };
+}
+/** The pipeline that builds the tables with SQL files under `src/`. */
+export function pipelineFile(model, tables, schemaKeys) {
+    const { platform, names } = model;
+    const first = tables[0].fullName;
+    const pipeline = {
+        name: names.pipeline,
+        catalog: first.catalog,
+        schema: schemaRef(schemaKeys, first.catalog, first.schema),
+        serverless: platform.compute.serverless,
+        ...(platform.compute.budget_policy_id && {
+            budget_policy_id: platform.compute.budget_policy_id,
+        }),
+        ...(platform.compute.channel && { channel: platform.compute.channel }),
+        root_path: `../src/${names.pipeline}`,
+        libraries: [{ glob: { include: `../src/${names.pipeline}/**` } }],
+    };
+    return {
+        resources: { pipelines: { [names.pipeline]: { ...pipeline, ...pipelineExtras(model) } } },
+    };
+}
+/** The serverless environment Python tasks run in. */
+const environment = { environment_key: "ingestron", spec: { environment_version: "6" } };
+/**
+ * A job that refreshes the pipelines in order, on the platform's schedule.
+ * With a Python task it is built without a schedule too, to run on demand.
+ */
+export function jobFile(model, tasks) {
+    const { platform, names } = model;
+    const python = tasks.some((task) => task.python);
+    if ((!platform.schedule && !python) || tasks.length === 0)
+        return undefined;
+    const job = {
+        name: names.job,
+        ...(platform.schedule && {
+            schedule: {
+                quartz_cron_expression: platform.schedule.cron,
+                timezone_id: platform.schedule.timezone,
+                pause_status: platform.schedule.paused ? "PAUSED" : "UNPAUSED",
+            },
+        }),
+        tasks: tasks.map(jobTask),
+        ...(python && { environments: [environment] }),
+    };
+    if (Object.keys(platform.tags).length > 0)
+        job.tags = platform.tags;
+    if (platform.notifications.on_failure.length > 0)
+        job.email_notifications = { on_failure: platform.notifications.on_failure };
+    return { resources: { jobs: { [names.job]: job } } };
+}
+function jobTask(task) {
+    return {
+        task_key: task.key,
+        ...(task.after.length > 0 && { depends_on: task.after.map((key) => ({ task_key: key })) }),
+        ...(task.pipeline && {
+            pipeline_task: { pipeline_id: `\${resources.pipelines.${task.pipeline}.id}` },
+        }),
+        ...(task.python && {
+            environment_key: environment.environment_key,
+            spark_python_task: { python_file: task.python },
+        }),
+    };
+}
+/** Bundle keys of the schemas this bundle manages, by `catalog.schema`. Empty unless `manage.schemas`. */
+export function schemaKeys(model) {
+    if (!model.platform.manage.schemas)
+        return new Map();
+    const keys = model.tables.map(({ fullName }) => [`${fullName.catalog}.${fullName.schema}`, `${fullName.catalog}_${fullName.schema}`]);
+    return new Map(keys);
+}
+/** A schema's name, or a reference to it when the bundle manages it, so the bundle creates it first. */
+export function schemaRef(keys, catalog, schema) {
+    const key = keys.get(`${catalog}.${schema}`);
+    return key ? `\${resources.schemas.${key}.name}` : schema;
+}
+/** Managed schemas, with the platform's grants for their layer. */
+export function schemasFile(model, keys) {
+    if (keys.size === 0)
+        return undefined;
+    const schemas = {};
+    for (const table of uniqueSchemas(model.tables)) {
+        const key = keys.get(`${table.fullName.catalog}.${table.fullName.schema}`);
+        const grants = model.platform.grants[table.layer] ?? [];
+        schemas[key] = {
+            catalog_name: table.fullName.catalog,
+            name: table.fullName.schema,
+            comment: `Layer ${table.layer} of ${model.project.id}.`,
+            ...(grants.length > 0 && { grants }),
+        };
+    }
+    return { resources: { schemas } };
+}
+function uniqueSchemas(tables) {
+    const seen = new Set();
+    return tables.filter(({ fullName }) => {
+        const key = `${fullName.catalog}.${fullName.schema}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+/** Tags and failure notifications, the same on every pipeline. */
+export function pipelineExtras(model) {
+    const { platform } = model;
+    const alerts = ["on-update-failure", "on-update-fatal-failure", "on-flow-failure"];
+    return {
+        ...(Object.keys(platform.tags).length > 0 && { tags: platform.tags }),
+        ...(platform.notifications.on_failure.length > 0 && {
+            notifications: [{ email_recipients: platform.notifications.on_failure, alerts }],
+        }),
+    };
+}
+/** YAML text with header comments, one per line given. */
+export function yamlText(headers, data) {
+    const comments = (Array.isArray(headers) ? headers : [headers])
+        .map((line) => `# ${line}\n`)
+        .join("");
+    return `${comments}${stringify(data, { lineWidth: 0 })}`;
+}

package/dist/capabilities.d.ts

@@ -0,0 +1,7 @@
+import type { Capability, ResolvedModel } from "@ingestron/definitions";
+/** The Databricks pack's capabilities. */
+export declare const capabilities: Capability[];
+/** Tag keys Unity Catalog refuses: `. , - = / :` or longer than 255 characters. */
+export declare function tagProblems(model: ResolvedModel): string[];
+/** SQL an administrator runs once, so the run_as identity may do what is turned on. */
+export declare function prerequisites(model: ResolvedModel, header: string): string | undefined;

package/dist/capabilities.js

@@ -0,0 +1,77 @@
+import { tagParts } from "./apply-contract.js";
+import { quoteName } from "./query.js";
+const owner = "The run_as identity owns the tables the pipelines create, so it may change them; any other identity needs MODIFY on each table";
+/** The Databricks pack's capabilities. */
+export const capabilities = [
+    {
+        id: "schemas",
+        title: "Managed schemas",
+        does: "Creates each layer's schema in the bundle and applies the platform's grants to it.",
+        needs: [
+            "USE CATALOG and CREATE SCHEMA on each catalog for the identity that deploys",
+            "MANAGE on the schemas, or ownership, to apply grants",
+        ],
+    },
+    {
+        id: "tags",
+        title: "Tags",
+        does: "Sets the contract's tags and classifications on tables and columns after each refresh.",
+        needs: [
+            "Databricks Runtime 16.3 or later (serverless jobs have it)",
+            `${owner}, or APPLY TAG with USE SCHEMA and USE CATALOG`,
+            "ASSIGN on each governed tag the contract uses",
+        ],
+    },
+    {
+        id: "comments",
+        title: "Column comments",
+        does: "Sets each column's description as its comment after each refresh.",
+        needs: ["Databricks Runtime 16.3 or later (serverless jobs have it)", owner],
+    },
+    {
+        id: "checks",
+        title: "Table checks",
+        does: "Counts the contract's rules over all rows after each refresh, then warns or fails the run.",
+        needs: ["SELECT on each table for the run_as identity (an owner has it)", "Serverless jobs"],
+    },
+];
+/** Tag keys Unity Catalog refuses: `. , - = / :` or longer than 255 characters. */
+export function tagProblems(model) {
+    if (!model.platform.manage.tags)
+        return [];
+    const tags = model.tables.flatMap((table) => [
+        ...table.contract.tags,
+        ...table.contract.columns.flatMap((column) => column.tags ?? []),
+    ]);
+    return [...new Set(tags)]
+        .filter((tag) => /[.,\-=/:]/.test(tagParts(tag)[0]) || tagParts(tag)[0].length > 255)
+        .map((tag) => `the tag "${tag}" has a key Unity Catalog does not allow: keys have no . , - = / : and at most 255 characters; write key:value for a value`);
+}
+/** SQL an administrator runs once, so the run_as identity may do what is turned on. */
+export function prerequisites(model, header) {
+    const { manage, run_as: runAs } = model.platform;
+    const on = capabilities.filter((capability) => manage[capability.id]);
+    if (on.length === 0)
+        return undefined;
+    const principal = runAs
+        ? quoteName(Object.values(runAs)[0] ?? "")
+        : "`<the identity that deploys>`";
+    const catalogs = [...new Set(model.tables.map((table) => table.fullName.catalog))];
+    const lines = [
+        `-- ${header}`,
+        "-- Run once as an administrator; review before running.",
+        runAs
+            ? "-- The grants go to run_as; give the same to the identity that deploys when it differs."
+            : "-- Set run_as in the platform, or replace the principal below with the identity that deploys.",
+        "",
+    ];
+    for (const capability of on) {
+        lines.push(`-- ${capability.title} needs:`, ...capability.needs.map((need) => `--   - ${need}.`));
+        if (capability.id === "schemas")
+            lines.push(...catalogs.map((catalog) => `GRANT USE CATALOG, CREATE SCHEMA ON CATALOG ${quoteName(catalog)} TO ${principal};`));
+        if (capability.id === "tags")
+            lines.push("-- Governed tags: an account administrator grants ASSIGN on each governed tag in the tag policy.");
+        lines.push("");
+    }
+    return `${lines.join("\n").trimEnd()}\n`;
+}

package/dist/connect-method.d.ts

@@ -0,0 +1,7 @@
+import type { Method } from "@ingestron/definitions";
+/** Connectors that read through an ingestion gateway (database change capture). */
+export declare const gatewayConnectors: Set<string>;
+/** Every connector this method supports, verified against the Databricks docs. */
+export declare const managedConnectors: string[];
+/** Lakeflow Connect managed ingestion. */
+export declare const lakeflowConnect: Method;

package/dist/connect-method.js

@@ -0,0 +1,88 @@
+/** Connectors that read through an ingestion gateway (database change capture). */
+export const gatewayConnectors = new Set(["sqlserver"]);
+/** Every connector this method supports, verified against the Databricks docs. */
+export const managedConnectors = ["sqlserver", "salesforce", "kafka"];
+/** Lakeflow Connect managed ingestion. */
+export const lakeflowConnect = {
+    action: "ingest",
+    description: "Databricks Lakeflow Connect: managed ingestion from databases (with a gateway), SaaS applications and Kafka.",
+    connectors: managedConnectors,
+    connectivity: ["public", "workspace"],
+    produces: "table",
+    defaults: { mode: "upsert", columns: "contract" },
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+            mode: {
+                enum: ["append", "upsert", "history"],
+                enumDescriptions: [
+                    "Add every change as a row.",
+                    "Keep the latest row per key (SCD type 1).",
+                    "Keep every version of a row with when it was valid (SCD type 2).",
+                ],
+                description: "How rows are kept.",
+            },
+            columns: {
+                enum: ["contract", "all"],
+                enumDescriptions: ["Only the contract's columns.", "Every source column."],
+                description: "Which source columns are ingested.",
+            },
+            gateway_policy_id: {
+                type: "string",
+                description: "Cluster policy for the gateway, which runs on classic compute.",
+                examples: ["0012AB34CD56EF78"],
+            },
+            staging_schema: {
+                type: "string",
+                description: "Schema for the gateway's staging volume. Default: the table's schema.",
+                examples: ["staging"],
+            },
+            value_format: {
+                enum: ["json", "avro", "protobuf", "string"],
+                enumDescriptions: ["JSON text.", "Avro.", "Protocol Buffers.", "Plain text, not parsed."],
+                description: "Kafka: how message values are parsed.",
+            },
+            continuous: {
+                type: "boolean",
+                description: "Kafka: run the ingestion pipeline continuously. Default: true in production; elsewhere the refresh job runs it, which costs less.",
+            },
+            starting_offset: {
+                enum: ["earliest", "latest"],
+                enumDescriptions: [
+                    "From the oldest message the topic keeps.",
+                    "From the messages that arrive after the pipeline starts.",
+                ],
+                description: "Kafka: where a new pipeline starts reading.",
+            },
+        },
+    },
+    check: checkConnect,
+};
+/** Lakeflow Connect writes the table with the source's names, so it cannot rename columns. */
+function renamesOf(table) {
+    return table.contract.columns
+        .filter((column) => column.physicalName && column.physicalName !== column.name)
+        .map((column) => `Lakeflow Connect keeps the source's column names, so it cannot read "${column.physicalName}" as "${column.name}"; remove physicalName, or rename the column in a transform`);
+}
+function checkConnect(table) {
+    const ingest = table.ingest;
+    if (!ingest)
+        return [];
+    const problems = renamesOf(table);
+    if (typeof ingest.connection.settings.uc_connection !== "string") {
+        problems.push('the connection needs "settings.uc_connection", the name of a Unity Catalog connection');
+    }
+    if (gatewayConnectors.has(ingest.connection.connector) && ingest.object.split(".").length !== 3) {
+        problems.push(`a ${ingest.connection.connector} object is written database.schema.table, not "${ingest.object}"`);
+    }
+    if (ingest.connection.connector === "kafka" && ingest.settings.mode !== "append") {
+        problems.push("kafka topics can only be appended (mode: append)");
+    }
+    const own = table.expectations
+        .filter((item) => item.from === "definition")
+        .map((item) => item.name);
+    if (own.length > 0)
+        problems.push(`managed ingestion cannot run expectations (${own.join(", ")}); check the data in a transform`);
+    return problems;
+}

package/dist/connect.d.ts

@@ -0,0 +1,27 @@
+import type { ResolvedModel, ResolvedTable } from "@ingestron/definitions";
+type Data = Record<string, unknown>;
+/** The tables one connection ingests, and the pipelines it needs. */
+export interface ConnectGroup {
+    connection: NonNullable<ResolvedTable["ingest"]>["connection"];
+    tables: ResolvedTable[];
+    ingestion: string;
+    gateway?: string;
+    /** Kafka runs continuously (by default in production only); the others are refreshed by the job. */
+    continuous: boolean;
+}
+/** Bundle keys of managed schemas, by `catalog.schema` (see bundle.ts). */
+type Keys = Map<string, string>;
+/** Tables ingested by lakeflow-connect, grouped by connection in path order. */
+export declare function connectGroups(model: ResolvedModel): ConnectGroup[];
+/**
+ * Problems with settings that one connection's tables share: its gateway, or
+ * whether its ingestion runs continuously.
+ */
+export declare function sharedProblems(groups: ConnectGroup[]): string[];
+/** The gateway pipeline: captures changes continuously on classic compute. */
+export declare function gatewayPipeline(group: ConnectGroup, keys: Keys): Data;
+/** The serverless ingestion pipeline that writes the tables. */
+export declare function ingestionPipeline(group: ConnectGroup, keys: Keys): Data;
+/** A note for generated files that use features Databricks has not made generally available. */
+export declare function previewNote(group: ConnectGroup): string;
+export {};

package/dist/connect.js

@@ -0,0 +1,152 @@
+import { schemaRef } from "./bundle.js";
+import { gatewayConnectors } from "./connect-method.js";
+import { quoteName } from "./query.js";
+/** Tables ingested by lakeflow-connect, grouped by connection in path order. */
+export function connectGroups(model) {
+    const groups = new Map();
+    for (const table of model.tables.filter((candidate) => candidate.ingest?.method === "lakeflow-connect")) {
+        const connection = table.ingest.connection;
+        const base = `${model.names.pipeline}_${connection.id}`;
+        const continuous = table.ingest.settings.continuous ?? model.platform.mode === "production";
+        const group = groups.get(connection.id) ?? {
+            connection,
+            tables: [],
+            ingestion: `${base}_ingestion`,
+            ...(gatewayConnectors.has(connection.connector) && { gateway: `${base}_gateway` }),
+            continuous: connection.connector === "kafka" && continuous === true,
+        };
+        group.tables.push(table);
+        groups.set(connection.id, group);
+    }
+    return [...groups.values()];
+}
+/**
+ * Problems with settings that one connection's tables share: its gateway, or
+ * whether its ingestion runs continuously.
+ */
+export function sharedProblems(groups) {
+    return groups.flatMap((group) => {
+        const keys = group.gateway ? ["gateway_policy_id", "staging_schema"] : ["continuous"];
+        const shared = (table) => JSON.stringify([
+            group.gateway ? gatewayStorage(table) : undefined,
+            ...keys.map((key) => table.ingest?.settings[key]),
+        ]);
+        const [first] = group.tables;
+        const different = group.tables.find((table) => shared(table) !== shared(first));
+        if (!first || !different)
+            return [];
+        const what = group.gateway ? "one gateway" : "one ingestion pipeline";
+        return [
+            `the tables of connection ${group.connection.id} share ${what}, so ${keys.join(" and ")} must be the same; ${different.name} differs from ${first.name}`,
+        ];
+    });
+}
+/** Where the gateway stages changes: the table's catalog, and its schema unless set. */
+function gatewayStorage(table) {
+    const staging = table.ingest?.settings.staging_schema;
+    return {
+        catalog: table.fullName.catalog,
+        schema: typeof staging === "string" ? staging : table.fullName.schema,
+    };
+}
+/** The gateway pipeline: captures changes continuously on classic compute. */
+export function gatewayPipeline(group, keys) {
+    const first = group.tables[0];
+    const { catalog, schema } = gatewayStorage(first);
+    const staging = schemaRef(keys, catalog, schema);
+    const policy = first.ingest?.settings.gateway_policy_id;
+    return {
+        name: group.gateway,
+        catalog,
+        schema: staging,
+        continuous: true,
+        gateway_definition: {
+            connection_name: group.connection.settings.uc_connection,
+            gateway_storage_catalog: catalog,
+            gateway_storage_schema: staging,
+        },
+        ...(typeof policy === "string" && {
+            clusters: [{ label: "default", policy_id: policy, apply_policy_default_values: true }],
+        }),
+    };
+}
+/** The serverless ingestion pipeline that writes the tables. */
+export function ingestionPipeline(group, keys) {
+    const first = group.tables[0];
+    const kafka = group.connection.connector === "kafka";
+    const source = group.gateway
+        ? { ingestion_gateway_id: `\${resources.pipelines.${group.gateway}.id}` }
+        : { connection_name: group.connection.settings.uc_connection };
+    return {
+        name: group.ingestion,
+        catalog: first.fullName.catalog,
+        schema: schemaRef(keys, first.fullName.catalog, first.fullName.schema),
+        serverless: true,
+        ...(group.continuous && { continuous: true }),
+        ...(kafka && { channel: "PREVIEW" }),
+        ingestion_definition: {
+            ...source,
+            objects: group.tables.map((table) => ({ table: tableObject(group, table, keys) })),
+        },
+    };
+}
+function tableObject(group, table, keys) {
+    const ingest = table.ingest;
+    const destination = {
+        destination_catalog: table.fullName.catalog,
+        destination_schema: schemaRef(keys, table.fullName.catalog, table.fullName.schema),
+        destination_table: table.fullName.table,
+        table_configuration: tableConfiguration(table),
+    };
+    if (group.connection.connector === "kafka")
+        return {
+            source_table: "N/A",
+            ...destination,
+            connector_options: { kafka_options: kafkaOptions(table) },
+        };
+    if (group.gateway) {
+        const [catalog, schema, name] = ingest.object.split(".");
+        return { source_catalog: catalog, source_schema: schema, source_table: name, ...destination };
+    }
+    return { source_schema: "objects", source_table: ingest.object, ...destination };
+}
+const scdTypes = {
+    append: "APPEND_ONLY",
+    upsert: "SCD_TYPE_1",
+    history: "SCD_TYPE_2",
+};
+function tableConfiguration(table) {
+    const settings = table.ingest.settings;
+    const keyed = settings.mode !== "append" && table.contract.primaryKey.length > 0;
+    // Kafka values are parsed with the contract's schema hints instead.
+    const kafka = table.ingest.connection.connector === "kafka";
+    return {
+        scd_type: scdTypes[String(settings.mode)],
+        ...(settings.columns === "contract" &&
+            !kafka && { include_columns: table.contract.columns.map((column) => column.name) }),
+        ...(keyed && { primary_keys: table.contract.primaryKey }),
+    };
+}
+function kafkaOptions(table) {
+    const settings = table.ingest.settings;
+    const format = String(settings.value_format ?? "json");
+    const hints = table.contract.columns
+        .filter((column) => column.type)
+        .map((column) => `${quoteName(column.name)} ${column.type}`);
+    const json = format === "json" && hints.length > 0
+        ? { json_options: { schema_hints: hints.join(", ") } }
+        : {};
+    return {
+        topics: [table.ingest.object],
+        ...(typeof settings.starting_offset === "string" && {
+            starting_offset: settings.starting_offset,
+        }),
+        value_transformer: { format: format.toUpperCase(), ...json },
+    };
+}
+/** A note for generated files that use features Databricks has not made generally available. */
+export function previewNote(group) {
+    if (group.connection.connector === "kafka")
+        return "Lakeflow Connect for Kafka is in Beta on Databricks (checked 2026-10-25).";
+    return "Lakeflow Connect fields used here are marked Preview in the Databricks CLI v1.22.1 bundle schema (checked 2026-10-25).";
+}

package/dist/connectors.d.ts

@@ -0,0 +1,3 @@
+import type { Connector } from "@ingestron/definitions";
+/** One entry per connector an ingest method reads. */
+export declare const connectors: Connector[];

package/dist/connectors.js

@@ -0,0 +1,55 @@
+const lakeflow = "https://docs.databricks.com/aws/en/ingestion/lakeflow-connect";
+/** One entry per connector an ingest method reads. */
+export const connectors = [
+    {
+        id: "files",
+        category: "Files and storage",
+        title: "Files",
+        method: "files",
+        object: "a folder under the connection's path",
+        settings: [
+            { name: "path", description: "Where the files land, such as /Volumes/main/landing/shop." },
+        ],
+        generates: "a streaming table that reads new files with Auto Loader",
+        docs: "https://docs.databricks.com/sql/language-manual/functions/read_files.html",
+    },
+    {
+        id: "sqlserver",
+        category: "Databases and analytics",
+        title: "SQL Server",
+        method: "lakeflow-connect",
+        object: "database.schema.table",
+        settings: [
+            { name: "uc_connection", description: "The Unity Catalog connection to the source." },
+        ],
+        generates: "an ingestion gateway and an ingestion pipeline (Lakeflow Connect)",
+        maturity: { status: "Public Preview", checked: "2026-10-25" },
+        docs: `${lakeflow}/sql-server-pipeline`,
+    },
+    {
+        id: "salesforce",
+        category: "Business applications",
+        title: "Salesforce",
+        method: "lakeflow-connect",
+        object: "the object name, such as Account",
+        settings: [
+            { name: "uc_connection", description: "The Unity Catalog connection to the source." },
+        ],
+        generates: "an ingestion pipeline (Lakeflow Connect)",
+        maturity: { status: "Public Preview", checked: "2026-10-25" },
+        docs: `${lakeflow}/salesforce-pipeline`,
+    },
+    {
+        id: "kafka",
+        category: "Streaming",
+        title: "Kafka",
+        method: "lakeflow-connect",
+        object: "the topic",
+        settings: [
+            { name: "uc_connection", description: "The Unity Catalog connection to the source." },
+        ],
+        generates: "an ingestion pipeline on the PREVIEW channel (Lakeflow Connect)",
+        maturity: { status: "Beta", checked: "2026-10-25" },
+        docs: `${lakeflow}/kafka-pipeline`,
+    },
+];

package/dist/contract.d.ts

@@ -0,0 +1,3 @@
+import { type ContractUse } from "@ingestron/definitions";
+/** The Databricks pack's use of every contract field, by path. */
+export declare const contract: Record<string, ContractUse>;

package/dist/contract.js

@@ -0,0 +1,53 @@
+// What the Databricks pack does with each ODCS contract field: used, not yet
+// (with why), or handled elsewhere. The documentation and the editor show
+// these, and a contract field no pack uses yet is a warning.
+import { contractCommon } from "@ingestron/definitions";
+const used = (does) => ({ use: "used", does });
+/** Used when the platform turns the capability on, after each refresh. */
+const needs = (id, does) => ({ use: "used", does, needs: id });
+/** Used, with the fields under it read as part of it. */
+const whole = (does) => ({ use: "used", does, whole: true });
+const notYet = (reason) => ({ use: "not-yet", reason });
+const elsewhere = (reason) => ({ use: "elsewhere", reason });
+const object = "schema.*";
+const column = "schema.*.properties.*";
+const tags = needs("tags", "Set as Unity Catalog tags after each refresh; key:value gives a value.");
+const relationships = notYet("Relationships between tables are not declared on the tables yet.");
+/** The Databricks pack's use of every contract field, by path. */
+export const contract = {
+    ...contractCommon,
+    schema: used("One object per table; each table uses the object named like it."),
+    [`${object}.name`]: used("Matched to the table by its name."),
+    [`${object}.logicalType`]: { use: "documentation" },
+    [`${object}.physicalName`]: elsewhere("The table's name comes from the standard's naming."),
+    [`${object}.physicalType`]: elsewhere("The store method decides: a streaming table or a materialised view."),
+    [`${object}.description`]: used("The table's comment."),
+    [`${object}.properties`]: used("The table's columns."),
+    [`${object}.relationships`]: relationships,
+    [`${object}.quality`]: {
+        use: "used",
+        whole: true,
+        needs: "checks",
+        does: "Rules with a comparison are counted over all rows after each refresh, then warn or fail the run as quality.default says.",
+    },
+    [`${object}.tags`]: tags,
+    [`${column}.name`]: used("The column's name in the table."),
+    [`${column}.logicalType`]: used("The column's type, through the standard's types; a schema hint when reading files."),
+    [`${column}.physicalType`]: used("The column's type, as written."),
+    [`${column}.required`]: used("A not-null expectation with the standard's quality.required action."),
+    [`${column}.primaryKey`]: used("The keys that upsert and history keep the current row by."),
+    [`${column}.primaryKeyPosition`]: used("The order of the keys."),
+    [`${column}.physicalName`]: used("The column's name at the source: files are read with it renamed to the contract's name. Lakeflow Connect keeps source names, so it refuses physicalName."),
+    [`${column}.logicalTypeOptions`]: whole("Limits become expectations with quality.default's action: length, pattern, minimum and maximum, multipleOf and array items. format is documentation."),
+    [`${column}.unique`]: needs("checks", "A table check after each refresh that no value repeats."),
+    [`${column}.partitioned`]: elsewhere("Tables use liquid clustering; set the store method's cluster_by."),
+    [`${column}.partitionKeyPosition`]: elsewhere("Tables use liquid clustering; set the store method's cluster_by."),
+    [`${column}.classification`]: needs("tags", "Set as the column's tag classification after each refresh."),
+    [`${column}.encryptedName`]: elsewhere("Encryption is managed outside Ingestron."),
+    [`${column}.relationships`]: relationships,
+    [`${column}.quality`]: whole("Rules that allow no failing row (nullValues, missingValues, invalidValues with mustBe: 0) become expectations; the others are a warning."),
+    [`${column}.description`]: needs("comments", "Set as the column's comment after each refresh: a pipeline takes column comments only with every column's type."),
+    [`${column}.tags`]: tags,
+    [`${column}.properties`]: notYet("The fields of an object column are not read yet; give the column's physicalType, such as STRUCT<street: STRING>."),
+    [`${column}.items`]: notYet("The element of an array column is not read yet; give the column's physicalType, such as ARRAY<STRING>."),
+};

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { databricks } from "./pack.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+// Public API of @ingestron/pack-databricks.
+export { databricks } from "./pack.js";

package/dist/methods.d.ts

@@ -0,0 +1,9 @@
+import type { Method } from "@ingestron/definitions";
+/** Reads files where they already are, with Auto Loader (`read_files`). */
+export declare const files: Method;
+/** A streaming table: each file is read once; optionally kept current with AUTO CDC. */
+export declare const streamingTable: Method;
+/** A materialised view, recomputed (incrementally where possible) on each update. */
+export declare const materialisedView: Method;
+/** The transform's own SQL, with `{{ tables.<input> }}` replaced by full names. */
+export declare const sql: Method;