@ingestron/pack-databricks 0.0.0-stage → 1.0.0

package/dist/contract.js

@@ -0,0 +1,54 @@
+// What the Databricks pack does with each ODCS contract field: used, not yet
+// (with why), or handled elsewhere. The documentation and the editor show
+// these, and a contract field no pack uses yet is a warning.
+import { contractCommon } from "@ingestron/definitions";
+const used = (does) => ({ use: "used", does });
+/** Used when the platform turns the capability on, after each refresh. */
+const needs = (id, does) => ({ use: "used", does, needs: id });
+/** Used, with the fields under it read as part of it. */
+const whole = (does) => ({ use: "used", does, whole: true });
+const notYet = (reason) => ({ use: "not-yet", reason });
+const elsewhere = (reason) => ({ use: "elsewhere", reason });
+const object = "schema.*";
+const column = "schema.*.properties.*";
+const tags = needs("tags", "Set as Unity Catalog tags after each refresh; key:value gives a value.");
+const relationships = notYet("Relationships between tables are not declared on the tables yet.");
+/** The Databricks pack's use of every contract field, by path. */
+export const contract = {
+    ...contractCommon,
+    schema: used("One object per table; each table uses the object named like it."),
+    [`${object}.name`]: used("Matched to the table by its name."),
+    [`${object}.logicalType`]: { use: "documentation" },
+    [`${object}.physicalName`]: elsewhere("The table's name comes from the standard's naming."),
+    [`${object}.physicalType`]: elsewhere("The store method decides: a streaming table or a materialised view."),
+    [`${object}.description`]: used("The table's comment."),
+    [`${object}.properties`]: used("The table's columns."),
+    [`${object}.relationships`]: relationships,
+    [`${object}.quality`]: {
+        use: "used",
+        whole: true,
+        needs: "checks",
+        does: "Rules with a comparison are counted over all rows after each refresh, then warn or fail the run as quality.default says.",
+    },
+    [`${object}.tags`]: tags,
+    [`${column}.name`]: used("The column's name in the table."),
+    [`${column}.logicalType`]: used("The column's type, through the standard's types; a schema hint when reading files."),
+    [`${column}.physicalType`]: used("The column's type, as written."),
+    [`${column}.required`]: used("A not-null expectation with the standard's quality.required action."),
+    [`${column}.primaryKey`]: used("The keys that upsert and history keep the current row by."),
+    [`${column}.primaryKeyPosition`]: used("The order of the keys."),
+    [`${column}.physicalName`]: used("The column's name at the source: files are read with it renamed to the contract's name. Lakeflow Connect keeps source names, so it refuses physicalName."),
+    [`${column}.logicalTypeOptions`]: whole("Limits become expectations with quality.default's action: length, pattern, minimum and maximum, multipleOf and array items. format is documentation."),
+    [`${column}.unique`]: needs("checks", "A table check after each refresh that no value repeats."),
+    [`${column}.partitioned`]: elsewhere("Tables use liquid clustering; set the store method's cluster_by."),
+    [`${column}.partitionKeyPosition`]: elsewhere("Tables use liquid clustering; set the store method's cluster_by."),
+    [`${column}.classification`]: needs("tags", "Set as the column's tag classification after each refresh."),
+    [`${column}.encryptedName`]: elsewhere("Encryption is managed outside Ingestron."),
+    [`${column}.relationships`]: relationships,
+    [`${column}.quality`]: whole("Rules that allow no failing row (nullValues, missingValues, invalidValues with mustBe: 0) become expectations; the others are a warning."),
+    [`${column}.description`]: needs("comments", "Set as the column's comment after each refresh: a pipeline takes column comments only with every column's type."),
+    [`${column}.tags`]: tags,
+    [`${column}.properties`]: notYet("The fields of an object column are not read yet; give the column's physicalType, such as STRUCT<street: STRING>."),
+    [`${column}.items`]: notYet("The element of an array column is not read yet; give the column's physicalType, such as ARRAY<STRING>."),
+};
+//# sourceMappingURL=contract.js.map

package/dist/contract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contract.js","sourceRoot":"","sources":["../src/contract.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,0EAA0E;AAC1E,6DAA6D;AAC7D,OAAO,EAAE,cAAc,EAAoB,MAAM,wBAAwB,CAAC;AAE1E,MAAM,IAAI,GAAG,CAAC,IAAY,EAAe,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AACpE,0EAA0E;AAC1E,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,IAAY,EAAe,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;AAC5F,yDAAyD;AACzD,MAAM,KAAK,GAAG,CAAC,IAAY,EAAe,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAClF,MAAM,MAAM,GAAG,CAAC,MAAc,EAAe,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;AAC7E,MAAM,SAAS,GAAG,CAAC,MAAc,EAAe,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;AAClF,MAAM,MAAM,GAAG,UAAU,CAAC;AAC1B,MAAM,MAAM,GAAG,uBAAuB,CAAC;AACvC,MAAM,IAAI,GAAG,KAAK,CAChB,MAAM,EACN,wEAAwE,CACzE,CAAC;AACF,MAAM,aAAa,GAAG,MAAM,CAAC,kEAAkE,CAAC,CAAC;AAEjG,kEAAkE;AAClE,MAAM,CAAC,MAAM,QAAQ,GAAgC;IACnD,GAAG,cAAc;IACjB,MAAM,EAAE,IAAI,CAAC,iEAAiE,CAAC;IAC/E,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE,IAAI,CAAC,mCAAmC,CAAC;IAC7D,CAAC,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,eAAe,EAAE;IACnD,CAAC,GAAG,MAAM,eAAe,CAAC,EAAE,SAAS,CAAC,oDAAoD,CAAC;IAC3F,CAAC,GAAG,MAAM,eAAe,CAAC,EAAE,SAAS,CACnC,qEAAqE,CACtE;IACD,CAAC,GAAG,MAAM,cAAc,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC;IACvD,CAAC,GAAG,MAAM,aAAa,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC;IACtD,CAAC,GAAG,MAAM,gBAAgB,CAAC,EAAE,aAAa;IAC1C,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE;QACrB,GAAG,EAAE,MAAM;QACX,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,QAAQ;QACf,IAAI,EAAE,0HAA0H;KACjI;IACD,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE,IAAI;IACxB,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE,IAAI,CAAC,iCAAiC,CAAC;IAC3D,CAAC,GAAG,MAAM,cAAc,CAAC,EAAE,IAAI,CAC7B,oFAAoF,CACrF;IACD,CAAC,GAAG,MAAM,eAAe,CAAC,EAAE,IAAI,CAAC,gCAAgC,CAAC;IAClE,CAAC,GAAG,MAAM,WAAW,CAAC,EAAE,IAAI,CAC1B,qEAAqE,CACtE;IACD,CAAC,GAAG,MAAM,aAAa,CAAC,EAAE,IAAI,CAAC,2DAA2D,CAAC;IAC3F,CAAC,GAAG,MAAM,qBAAqB,CAAC,EAAE,IAAI,CAAC,wBAAwB,CAAC;IAChE,CAAC,GAAG,MAAM,eAAe,CAAC,EAAE,IAAI,CAC9B,0JAA0J,CAC3J;IACD,CAAC,GAAG,MAAM,qBAAqB,CAAC,EAAE,KAAK,CACrC,sJAAsJ,CACvJ;IACD,CAAC,GAAG,MAAM,SAAS,CAAC,EAAE,KAAK,CAAC,QAAQ,EAAE,yDAAyD,CAAC;IAChG,CAAC,GAAG,MAAM,cAAc,CAAC,EAAE,SAAS,CAClC,kEAAkE,CACnE;IACD,CAAC,GAAG,MAAM,uBAAuB,CAAC,EAAE,SAAS,CAC3C,kEAAkE,CACnE;IACD,CAAC,GAAG,MAAM,iBAAiB,CAAC,EAAE,KAAK,CACjC,MAAM,EACN,4DAA4D,CAC7D;IACD,CAAC,GAAG,MAAM,gBAAgB,CAAC,EAAE,SAAS,CAAC,0CAA0C,CAAC;IAClF,CAAC,GAAG,MAAM,gBAAgB,CAAC,EAAE,aAAa;IAC1C,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,KAAK,CAC1B,0IAA0I,CAC3I;IACD,CAAC,GAAG,MAAM,cAAc,CAAC,EAAE,KAAK,CAC9B,UAAU,EACV,iHAAiH,CAClH;IACD,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE,IAAI;IACxB,CAAC,GAAG,MAAM,aAAa,CAAC,EAAE,MAAM,CAC9B,kHAAkH,CACnH;IACD,CAAC,GAAG,MAAM,QAAQ,CAAC,EAAE,MAAM,CACzB,wGAAwG,CACzG;CACF,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { databricks } from "./pack.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}

package/dist/index.js

@@ -0,0 +1,3 @@
+// Public API of @ingestron/pack-databricks.
+export { databricks } from "./pack.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}

package/dist/methods.d.ts

@@ -0,0 +1,10 @@
+import type { Method } from "@ingestron/definitions";
+/** Reads files where they already are, with Auto Loader (`read_files`). */
+export declare const files: Method;
+/** A streaming table: each file is read once; optionally kept current with AUTO CDC. */
+export declare const streamingTable: Method;
+/** A materialised view, recomputed (incrementally where possible) on each update. */
+export declare const materialisedView: Method;
+/** The transform's own SQL, with `{{ tables.<input> }}` replaced by full names. */
+export declare const sql: Method;
+//# sourceMappingURL=methods.d.ts.map

package/dist/methods.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"methods.d.ts","sourceRoot":"","sources":["../src/methods.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAiB,MAAM,wBAAwB,CAAC;AAoBpE,2EAA2E;AAC3E,eAAO,MAAM,KAAK,EAAE,MA6DnB,CAAC;AAKF,wFAAwF;AACxF,eAAO,MAAM,cAAc,EAAE,MA+C5B,CAAC;AAEF,qFAAqF;AACrF,eAAO,MAAM,gBAAgB,EAAE,MAY9B,CAAC;AAEF,mFAAmF;AACnF,eAAO,MAAM,GAAG,EAAE,MAUjB,CAAC"}

package/dist/methods.js

@@ -0,0 +1,167 @@
+import { renderQuery } from "./query.js";
+const clusterBy = {
+    description: "How the table is clustered: auto, none, or a list of columns.",
+    anyOf: [
+        {
+            enum: ["auto", "none"],
+            enumDescriptions: ["Databricks chooses the clustering columns.", "No clustering."],
+        },
+        { type: "array", minItems: 1, items: { type: "string" } },
+    ],
+    examples: [["customer_id", "ordered_at"]],
+};
+const properties = {
+    type: "object",
+    description: "Table properties (TBLPROPERTIES).",
+    additionalProperties: { type: "string" },
+};
+/** Reads files where they already are, with Auto Loader (`read_files`). */
+export const files = {
+    action: "ingest",
+    description: "Reads files in cloud storage or a volume with Auto Loader.",
+    connectors: ["files"],
+    connectivity: ["public", "workspace"],
+    produces: "files",
+    defaults: {
+        format: "parquet",
+        schema_evolution: "addNewColumns",
+        schema_hints: true,
+        options: {},
+    },
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+            format: {
+                enum: ["parquet", "csv", "json", "avro", "orc", "xml", "text"],
+                enumDescriptions: [
+                    "Parquet files.",
+                    "CSV text; set options such as header: true.",
+                    "JSON, one object per line unless options say otherwise.",
+                    "Avro files.",
+                    "ORC files.",
+                    "XML; set options such as rowTag.",
+                    "Each line as one row.",
+                ],
+                description: "The files' format.",
+            },
+            schema_evolution: {
+                enum: [
+                    "addNewColumns",
+                    "addNewColumnsWithTypeWidening",
+                    "rescue",
+                    "failOnNewColumns",
+                    "none",
+                ],
+                enumDescriptions: [
+                    "Stop when files gain columns, then go on with the columns added.",
+                    "As addNewColumns, and also widen column types, such as int to bigint.",
+                    "Keep the columns as they are; values that do not fit go to _rescued_data.",
+                    "Stop when files gain columns, until the table is changed.",
+                    "Ignore new columns.",
+                ],
+                description: "What happens when files gain columns.",
+            },
+            schema_hints: {
+                type: "boolean",
+                description: "Pass the contract's column types as schemaHints.",
+            },
+            options: {
+                type: "object",
+                description: "Other read_files options, such as header: true.",
+                additionalProperties: { type: ["string", "number", "boolean"] },
+            },
+        },
+    },
+    check: (table) => typeof table.ingest?.connection.settings.path === "string"
+        ? []
+        : ['the connection needs "settings.path"'],
+};
+/** Table properties are the organisation's, such as change data feed: written in the standard. */
+const storePlacement = { standard: ["properties"] };
+/** A streaming table: each file is read once; optionally kept current with AUTO CDC. */
+export const streamingTable = {
+    action: "store",
+    description: "A streaming table. mode append adds rows; upsert (SCD type 1) and history (SCD type 2) apply changes by key.",
+    accepts: ["files"],
+    produces: "table",
+    defaults: { mode: "append", cluster_by: "auto", properties: {}, changes_suffix: "_changes" },
+    placement: storePlacement,
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+            mode: {
+                enum: ["append", "upsert", "history"],
+                enumDescriptions: [
+                    "Add every row as it arrives.",
+                    "Keep the latest row per key (SCD type 1), ordered by sequence_by.",
+                    "Keep every version of a row with when it was valid (SCD type 2), ordered by sequence_by.",
+                ],
+                description: "How rows are kept.",
+            },
+            sequence_by: {
+                type: "string",
+                description: "Column that orders changes; needed for upsert and history.",
+                examples: ["updated_at"],
+            },
+            delete_when: {
+                type: "string",
+                description: "SQL condition for rows that are deletions, such as op = 'D'.",
+                examples: ["op = 'D'"],
+            },
+            except_columns: {
+                type: "array",
+                items: { type: "string" },
+                description: "Change columns not to keep, such as op.",
+            },
+            changes_suffix: {
+                type: "string",
+                pattern: "^_[a-z0-9_]+$",
+                description: "Suffix of the table that keeps changes as received.",
+                examples: ["_changes"],
+            },
+            cluster_by: clusterBy,
+            properties,
+        },
+    },
+    check: checkChangeData,
+};
+/** A materialised view, recomputed (incrementally where possible) on each update. */
+export const materialisedView = {
+    action: "store",
+    description: "A materialised view. Expectations make every refresh a full one.",
+    accepts: ["files", "table", "query"],
+    produces: "table",
+    defaults: { cluster_by: "auto", properties: {} },
+    placement: storePlacement,
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: { cluster_by: clusterBy, properties },
+    },
+};
+/** The transform's own SQL, with `{{ tables.<input> }}` replaced by full names. */
+export const sql = {
+    action: "transform",
+    description: "Runs the transform's SQL query.",
+    accepts: ["table"],
+    produces: "query",
+    settings: { type: "object", additionalProperties: false },
+    check: (table) => {
+        const result = renderQuery(table.transform?.code ?? "", table.transform?.inputs ?? {});
+        return "problem" in result ? [result.problem] : [];
+    },
+};
+function checkChangeData(table) {
+    const settings = table.store?.settings ?? {};
+    if (settings.mode === "append")
+        return [];
+    const problems = [];
+    if (table.contract.primaryKey.length === 0)
+        problems.push(`mode ${String(settings.mode)} needs primary key columns in the contract`);
+    if (typeof settings.sequence_by !== "string")
+        problems.push(`mode ${String(settings.mode)} needs "sequence_by"`);
+    return problems;
+}
+//# sourceMappingURL=methods.js.map

package/dist/methods.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"methods.js","sourceRoot":"","sources":["../src/methods.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,SAAS,GAAG;IAChB,WAAW,EAAE,+DAA+D;IAC5E,KAAK,EAAE;QACL;YACE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACtB,gBAAgB,EAAE,CAAC,4CAA4C,EAAE,gBAAgB,CAAC;SACnF;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;KAC1D;IACD,QAAQ,EAAE,CAAC,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;CAC1C,CAAC;AACF,MAAM,UAAU,GAAG;IACjB,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,mCAAmC;IAChD,oBAAoB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;CACzC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,KAAK,GAAW;IAC3B,MAAM,EAAE,QAAQ;IAChB,WAAW,EAAE,4DAA4D;IACzE,UAAU,EAAE,CAAC,OAAO,CAAC;IACrB,YAAY,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC;IACrC,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE;QACR,MAAM,EAAE,SAAS;QACjB,gBAAgB,EAAE,eAAe;QACjC,YAAY,EAAE,IAAI;QAClB,OAAO,EAAE,EAAE;KACZ;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,MAAM,EAAE;gBACN,IAAI,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;gBAC9D,gBAAgB,EAAE;oBAChB,gBAAgB;oBAChB,6CAA6C;oBAC7C,yDAAyD;oBACzD,aAAa;oBACb,YAAY;oBACZ,kCAAkC;oBAClC,uBAAuB;iBACxB;gBACD,WAAW,EAAE,oBAAoB;aAClC;YACD,gBAAgB,EAAE;gBAChB,IAAI,EAAE;oBACJ,eAAe;oBACf,+BAA+B;oBAC/B,QAAQ;oBACR,kBAAkB;oBAClB,MAAM;iBACP;gBACD,gBAAgB,EAAE;oBAChB,kEAAkE;oBAClE,uEAAuE;oBACvE,2EAA2E;oBAC3E,2DAA2D;oBAC3D,qBAAqB;iBACtB;gBACD,WAAW,EAAE,uCAAuC;aACrD;YACD,YAAY,EAAE;gBACZ,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,kDAAkD;aAChE;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,iDAAiD;gBAC9D,oBAAoB,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE;aAChE;SACF;KACF;IACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CACf,OAAO,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,KAAK,QAAQ;QACxD,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,CAAC,sCAAsC,CAAC;CAC/C,CAAC;AAEF,kGAAkG;AAClG,MAAM,cAAc,GAAG,EAAE,QAAQ,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;AAEpD,wFAAwF;AACxF,MAAM,CAAC,MAAM,cAAc,GAAW;IACpC,MAAM,EAAE,OAAO;IACf,WAAW,EACT,8GAA8G;IAChH,OAAO,EAAE,CAAC,OAAO,CAAC;IAClB,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,cAAc,EAAE,UAAU,EAAE;IAC5F,SAAS,EAAE,cAAc;IACzB,QAAQ,EAAE;QACR,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,IAAI,EAAE;gBACJ,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC;gBACrC,gBAAgB,EAAE;oBAChB,8BAA8B;oBAC9B,mEAAmE;oBACnE,0FAA0F;iBAC3F;gBACD,WAAW,EAAE,oBAAoB;aAClC;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,4DAA4D;gBACzE,QAAQ,EAAE,CAAC,YAAY,CAAC;aACzB;YACD,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,8DAA8D;gBAC3E,QAAQ,EAAE,CAAC,UAAU,CAAC;aACvB;YACD,cAAc,EAAE;gBACd,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,WAAW,EAAE,yCAAyC;aACvD;YACD,cAAc,EAAE;gBACd,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,eAAe;gBACxB,WAAW,EAAE,qDAAqD;gBAClE,QAAQ,EAAE,CAAC,UAAU,CAAC;aACvB;YACD,UAAU,EAAE,SAAS;YACrB,UAAU;SACX;KACF;IACD,KAAK,EAAE,eAAe;CACvB,CAAC;AAEF,qFAAqF;AACrF,MAAM,CAAC,MAAM,gBAAgB,GAAW;IACtC,MAAM,EAAE,OAAO;IACf,WAAW,EAAE,kEAAkE;IAC/E,OAAO,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;IACpC,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE;IAChD,SAAS,EAAE,cAAc;IACzB,QAAQ,EAAE;QACR,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE;KAClD;CACF,CAAC;AAEF,mFAAmF;AACnF,MAAM,CAAC,MAAM,GAAG,GAAW;IACzB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,iCAAiC;IAC9C,OAAO,EAAE,CAAC,OAAO,CAAC;IAClB,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,KAAK,EAAE;IACzD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;QACvF,OAAO,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrD,CAAC;CACF,CAAC;AAEF,SAAS,eAAe,CAAC,KAAoB;IAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC7C,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;QACxC,QAAQ,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAC3F,IAAI,OAAO,QAAQ,CAAC,WAAW,KAAK,QAAQ;QAC1C,QAAQ,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACrE,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/pack.d.ts

@@ -0,0 +1,9 @@
+import type { Pack } from "@ingestron/definitions";
+/**
+ * The Databricks pack, version 1. Name it in a standard as `pack: databricks@1`.
+ * @example
+ * const result = loadDefinitions("definitions", { pack: databricks, environment: "dev" });
+ * if (result.model) console.log(Object.keys(databricks.build(result.model)));
+ */
+export declare const databricks: Pack;
+//# sourceMappingURL=pack.d.ts.map

package/dist/pack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.d.ts","sourceRoot":"","sources":["../src/pack.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAUnD;;;;;GAKG;AACH,eAAO,MAAM,UAAU,EAAE,IAuDxB,CAAC"}

package/dist/pack.js

@@ -0,0 +1,73 @@
+// The pack: methods, smart defaults, rules and the build function.
+import { contract } from "./contract.js";
+import { connectors } from "./connectors.js";
+import { build } from "./build.js";
+import { request } from "./request.js";
+import { capabilities, tagProblems } from "./capabilities.js";
+import { lakeflowConnect } from "./connect-method.js";
+import { connectGroups, sharedProblems } from "./connect.js";
+import { files, materialisedView, sql, streamingTable } from "./methods.js";
+import { pyspark } from "./pyspark-method.js";
+/**
+ * The Databricks pack, version 1. Name it in a standard as `pack: databricks@1`.
+ * @example
+ * const result = loadDefinitions("definitions", { pack: databricks, environment: "dev" });
+ * if (result.model) console.log(Object.keys(databricks.build(result.model)));
+ */
+export const databricks = {
+    id: "databricks",
+    version: 1,
+    defaults: {
+        naming: {
+            catalog: "{{ environment }}",
+            schema: "{{ layer }}",
+            table: "{{ table }}",
+            pipeline: "{{ project }}",
+            job: "{{ project }}_refresh",
+        },
+        layers: ["bronze", "silver", "gold"],
+        types: {
+            string: "STRING",
+            integer: "BIGINT",
+            number: "DOUBLE",
+            boolean: "BOOLEAN",
+            date: "DATE",
+            timestamp: "TIMESTAMP",
+        },
+        quality: { required: "warn", default: "warn" },
+    },
+    rules: {
+        ingest: {
+            default: "files",
+            when: [
+                { connector: "sqlserver", use: "lakeflow-connect" },
+                { connector: "salesforce", use: "lakeflow-connect" },
+                { connector: "kafka", use: "lakeflow-connect", with: { mode: "append" } },
+            ],
+        },
+        transform: { default: "sql" },
+        store: { default: "streaming-table", when: [{ input: "query", use: "materialised-view" }] },
+    },
+    methods: {
+        files,
+        "lakeflow-connect": lakeflowConnect,
+        sql,
+        pyspark,
+        "streaming-table": streamingTable,
+        "materialised-view": materialisedView,
+    },
+    connectors,
+    capabilities,
+    contract,
+    check(model) {
+        const problems = [];
+        if (!model.platform.compute.serverless)
+            problems.push("classic compute is not supported yet; leave compute.serverless at true");
+        if (model.tables.length === 0)
+            problems.push("there is nothing to generate: add a flow or a transform");
+        return [...problems, ...tagProblems(model), ...sharedProblems(connectGroups(model))];
+    },
+    build,
+    request,
+};
+//# sourceMappingURL=pack.js.map

package/dist/pack.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.js","sourceRoot":"","sources":["../src/pack.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAE9C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAS;IAC9B,EAAE,EAAE,YAAY;IAChB,OAAO,EAAE,CAAC;IACV,QAAQ,EAAE;QACR,MAAM,EAAE;YACN,OAAO,EAAE,mBAAmB;YAC5B,MAAM,EAAE,aAAa;YACrB,KAAK,EAAE,aAAa;YACpB,QAAQ,EAAE,eAAe;YACzB,GAAG,EAAE,uBAAuB;SAC7B;QACD,MAAM,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;QACpC,KAAK,EAAE;YACL,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,SAAS;YAClB,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,WAAW;SACvB;QACD,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KAC/C;IACD,KAAK,EAAE;QACL,MAAM,EAAE;YACN,OAAO,EAAE,OAAO;YAChB,IAAI,EAAE;gBACJ,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,kBAAkB,EAAE;gBACnD,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,EAAE,kBAAkB,EAAE;gBACpD,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,kBAAkB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;aAC1E;SACF;QACD,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;QAC7B,KAAK,EAAE,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,mBAAmB,EAAE,CAAC,EAAE;KAC5F;IACD,OAAO,EAAE;QACP,KAAK;QACL,kBAAkB,EAAE,eAAe;QACnC,GAAG;QACH,OAAO;QACP,iBAAiB,EAAE,cAAc;QACjC,mBAAmB,EAAE,gBAAgB;KACtC;IACD,UAAU;IACV,YAAY;IACZ,QAAQ;IACR,KAAK,CAAC,KAAK;QACT,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU;YACpC,QAAQ,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;QAC1F,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QAC3E,OAAO,CAAC,GAAG,QAAQ,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,EAAE,GAAG,cAAc,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvF,CAAC;IACD,KAAK;IACL,OAAO;CACR,CAAC"}

package/dist/pyspark-method.d.ts

@@ -0,0 +1,4 @@
+import type { Method } from "@ingestron/definitions";
+/** A PySpark transform: the code defines a function that gets a DataFrame per input. */
+export declare const pyspark: Method;
+//# sourceMappingURL=pyspark-method.d.ts.map

package/dist/pyspark-method.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pyspark-method.d.ts","sourceRoot":"","sources":["../src/pyspark-method.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,wFAAwF;AACxF,eAAO,MAAM,OAAO,EAAE,MA4BrB,CAAC"}

package/dist/pyspark-method.js

@@ -0,0 +1,41 @@
+/** A PySpark transform: the code defines a function that gets a DataFrame per input. */
+export const pyspark = {
+    action: "transform",
+    description: "Calls a function in the transform's Python code with a dict of input DataFrames; it returns one DataFrame.",
+    accepts: ["table"],
+    produces: "query",
+    defaults: { function: "transform" },
+    settings: {
+        type: "object",
+        additionalProperties: false,
+        properties: {
+            function: {
+                type: "string",
+                pattern: "^[A-Za-z_][A-Za-z0-9_]*$",
+                description: "The function to call. Default: transform.",
+                examples: ["build_daily_sales"],
+            },
+        },
+    },
+    check: (table) => {
+        const code = table.transform?.code ?? "";
+        const name = String(table.transform?.settings.function);
+        const defined = new RegExp(`^def ${name}\\(`, "m").test(code);
+        return [
+            ...(defined ? [] : [`the code has no function "def ${name}(tables)"`]),
+            ...unknownInputs(code, Object.keys(table.transform?.inputs ?? {})),
+        ];
+    },
+};
+/**
+ * Each `tables["x"]` in the code whose `x` is not an input: the function gets
+ * a dict with one DataFrame per input, so reading another key fails.
+ */
+function unknownInputs(code, inputs) {
+    const lines = code.split("\n").map((line) => line.replace(/#.*/, ""));
+    const read = [...lines.join("\n").matchAll(/\btables\[\s*["']([^"']+)["']\s*\]/g)].map((match) => match[1]);
+    return [...new Set(read)]
+        .filter((key) => !inputs.includes(key))
+        .map((key) => `the code reads tables["${key}"], which is not an input; the inputs are ${inputs.map((input) => `"${input}"`).join(", ") || "none"}`);
+}
+//# sourceMappingURL=pyspark-method.js.map

package/dist/pyspark-method.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pyspark-method.js","sourceRoot":"","sources":["../src/pyspark-method.ts"],"names":[],"mappings":"AAGA,wFAAwF;AACxF,MAAM,CAAC,MAAM,OAAO,GAAW;IAC7B,MAAM,EAAE,WAAW;IACnB,WAAW,EACT,4GAA4G;IAC9G,OAAO,EAAE,CAAC,OAAO,CAAC;IAClB,QAAQ,EAAE,OAAO;IACjB,QAAQ,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE;IACnC,QAAQ,EAAE;QACR,IAAI,EAAE,QAAQ;QACd,oBAAoB,EAAE,KAAK;QAC3B,UAAU,EAAE;YACV,QAAQ,EAAE;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,0BAA0B;gBACnC,WAAW,EAAE,2CAA2C;gBACxD,QAAQ,EAAE,CAAC,mBAAmB,CAAC;aAChC;SACF;KACF;IACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,QAAQ,IAAI,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,OAAO;YACL,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,iCAAiC,IAAI,WAAW,CAAC,CAAC;YACtE,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;SACnE,CAAC;IACJ,CAAC;CACF,CAAC;AAEF;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,MAAgB;IACnD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC,CAAC,CAAC,GAAG,CACpF,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAE,CACrB,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;SACtB,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;SACtC,GAAG,CACF,CAAC,GAAG,EAAE,EAAE,CACN,0BAA0B,GAAG,6CAA6C,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CACvI,CAAC;AACN,CAAC"}

package/dist/python-data.d.ts

@@ -0,0 +1,22 @@
+import { type ResolvedTable } from "@ingestron/definitions";
+/** Everything materialised-view.py.liquid needs. */
+export declare function pythonData(table: ResolvedTable, header: string): {
+    header: string;
+    code: string;
+    name: string;
+    comment: string;
+    cluster: string;
+    properties: string;
+    expectations: {
+        decorator: "expect_all" | "expect_all_or_drop" | "expect_all_or_fail";
+        checks: string;
+        count: number;
+    }[];
+    wrapper: string;
+    function: string;
+    inputs: {
+        name: string;
+        table: string;
+    }[];
+};
+//# sourceMappingURL=python-data.d.ts.map

package/dist/python-data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"python-data.d.ts","sourceRoot":"","sources":["../src/python-data.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAQ1E,oDAAoD;AACpD,wBAAgB,UAAU,CAAC,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM;;;;;;;;;;;;;;;;;;EA4B9D"}

package/dist/python-data.js

@@ -0,0 +1,50 @@
+// Template data for a PySpark transform's Python file. Python string and
+// dict literals are written as JSON, which Python reads the same way.
+import { fullNameText } from "@ingestron/definitions";
+const decorators = {
+    warn: "expect_all",
+    drop: "expect_all_or_drop",
+    fail: "expect_all_or_fail",
+};
+/** Everything materialised-view.py.liquid needs. */
+export function pythonData(table, header) {
+    const store = table.store?.settings ?? {};
+    const transform = table.transform;
+    const properties = (store.properties ?? {});
+    return {
+        header,
+        code: transform.code.trimEnd(),
+        name: JSON.stringify(fullNameText(table.fullName)),
+        comment: table.contract.description ? JSON.stringify(table.contract.description) : "",
+        cluster: clusterOf(store.cluster_by),
+        properties: Object.keys(properties).length > 0 ? pythonDict(Object.entries(properties)) : "",
+        expectations: ["warn", "drop", "fail"]
+            .map((action) => {
+            const checks = table.expectations.filter((item) => item.action === action);
+            return {
+                decorator: decorators[action],
+                checks: pythonDict(checks.map((item) => [item.name, item.check])),
+                count: checks.length,
+            };
+        })
+            .filter((group) => group.count > 0),
+        wrapper: `ingestron_${table.name}`,
+        function: String(transform.settings.function),
+        inputs: Object.entries(transform.inputs).map(([name, full]) => ({
+            name: JSON.stringify(name),
+            table: JSON.stringify(full),
+        })),
+    };
+}
+function clusterOf(value) {
+    if (value === "auto")
+        return "cluster_by_auto=True";
+    if (Array.isArray(value))
+        return `cluster_by=${JSON.stringify(value)}`;
+    return "";
+}
+/** `{"a": "b", …}`: a Python dict of strings, written the way Python code usually is. */
+function pythonDict(entries) {
+    return `{${entries.map(([key, value]) => `${JSON.stringify(key)}: ${JSON.stringify(value)}`).join(", ")}}`;
+}
+//# sourceMappingURL=python-data.js.map

package/dist/python-data.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"python-data.js","sourceRoot":"","sources":["../src/python-data.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,sEAAsE;AACtE,OAAO,EAAE,YAAY,EAAsB,MAAM,wBAAwB,CAAC;AAE1E,MAAM,UAAU,GAAG;IACjB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,oBAAoB;IAC1B,IAAI,EAAE,oBAAoB;CAClB,CAAC;AAEX,oDAAoD;AACpD,MAAM,UAAU,UAAU,CAAC,KAAoB,EAAE,MAAc;IAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,SAAU,CAAC;IACnC,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAA2B,CAAC;IACtE,OAAO;QACL,MAAM;QACN,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;QAC9B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAClD,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE;QACrF,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC;QACpC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QAC5F,YAAY,EAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAW;aAC9C,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACd,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;YAC3E,OAAO;gBACL,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC;gBAC7B,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjE,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC;QACJ,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;QACrC,OAAO,EAAE,aAAa,KAAK,CAAC,IAAI,EAAE;QAClC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC7C,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC5B,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,KAAK,MAAM;QAAE,OAAO,sBAAsB,CAAC;IACpD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,cAAc,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;IACvE,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,yFAAyF;AACzF,SAAS,UAAU,CAAC,OAA2B;IAC7C,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;AAC7G,CAAC"}

package/dist/query.d.ts

@@ -0,0 +1,9 @@
+/** The SQL with inputs replaced, or a problem to report. A trailing `;` is removed. */
+export declare function renderQuery(code: string, inputs: Record<string, string>): {
+    sql: string;
+} | {
+    problem: string;
+};
+/** `a.b.c` → `` `a`.`b`.`c` ``, so any valid name is safe in SQL. */
+export declare function quoteName(fullName: string): string;
+//# sourceMappingURL=query.d.ts.map

package/dist/query.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAMA,uFAAuF;AACvF,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAavC;AAED,qEAAqE;AACrE,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKlD"}

package/dist/query.js

@@ -0,0 +1,26 @@
+// Renders a transform's SQL: `{{ tables.<input> }}` becomes the input's full name.
+import { Liquid } from "liquidjs";
+// Strict, so an input that the transform did not declare is an error.
+const engine = new Liquid({ strictVariables: true, strictFilters: true });
+/** The SQL with inputs replaced, or a problem to report. A trailing `;` is removed. */
+export function renderQuery(code, inputs) {
+    const quoted = Object.fromEntries(Object.entries(inputs).map(([name, full]) => [name, quoteName(full)]));
+    try {
+        const sql = engine.parseAndRenderSync(code, { tables: quoted });
+        return { sql: sql.trim().replace(/;\s*$/, "") };
+    }
+    catch (cause) {
+        const reason = cause instanceof Error ? cause.message.split("\n")[0] : String(cause);
+        return {
+            problem: `the SQL cannot be rendered (${reason}); inputs are written {{ tables.<name> }} and must be listed in inputs`,
+        };
+    }
+}
+/** `a.b.c` → `` `a`.`b`.`c` ``, so any valid name is safe in SQL. */
+export function quoteName(fullName) {
+    return fullName
+        .split(".")
+        .map((part) => `\`${part.replaceAll("`", "``")}\``)
+        .join(".");
+}
+//# sourceMappingURL=query.js.map

package/dist/query.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query.js","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,sEAAsE;AACtE,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;AAE1E,uFAAuF;AACvF,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,MAA8B;IAE9B,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CACtE,CAAC;IACF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,kBAAkB,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAW,CAAC;QAC1E,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrF,OAAO;YACL,OAAO,EAAE,+BAA+B,MAAM,wEAAwE;SACvH,CAAC;IACJ,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,SAAS,CAAC,QAAgB;IACxC,OAAO,QAAQ;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;SAClD,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC"}

package/dist/request.d.ts

@@ -0,0 +1,4 @@
+import type { RequestSection, ResolvedModel } from "@ingestron/definitions";
+/** The Databricks sections of the platform request. Nothing here is checked. */
+export declare function request(model: ResolvedModel): RequestSection[];
+//# sourceMappingURL=request.d.ts.map

package/dist/request.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAE5E,gFAAgF;AAChF,wBAAgB,OAAO,CAAC,KAAK,EAAE,aAAa,GAAG,cAAc,EAAE,CAS9D"}

package/dist/request.js

@@ -0,0 +1,47 @@
+/** The Databricks sections of the platform request. Nothing here is checked. */
+export function request(model) {
+    return [
+        {
+            title: "Must exist before deploying",
+            intro: "Ingestron has not checked these; the generated files expect them.",
+            lines: prerequisites(model),
+        },
+        { title: "Schedules", lines: [schedule(model)] },
+    ];
+}
+function prerequisites(model) {
+    const { platform } = model;
+    const names = model.tables.map((table) => table.fullName);
+    const catalogs = unique(names.map((name) => name.catalog));
+    const schemas = platform.manage.schemas
+        ? []
+        : unique(names.map((name) => `${name.catalog}.${name.schema}`));
+    const folders = unique(model.tables.flatMap((table) => {
+        const folder = table.ingest?.settings.landing ?? table.ingest?.connection.settings.path;
+        return typeof folder === "string" ? [folder] : [];
+    }));
+    return [
+        ...catalogs.map((catalog) => `Catalog \`${catalog}\`, which the deploying identity can use.`),
+        ...schemas.map((schema) => `Schema \`${schema}\`, in which it can create tables and views.`),
+        ...folders.map((folder) => `Files at \`${folder}\`, which the pipeline can read.`),
+        platform.compute.serverless
+            ? "Serverless compute for pipelines and jobs, turned on in the workspace."
+            : "Classic compute that the pipelines may use.",
+        ...(platform.workspace.host
+            ? []
+            : [
+                "No workspace host is set, so `databricks bundle deploy` uses the workspace of the Databricks CLI profile that runs it.",
+            ]),
+    ];
+}
+function schedule(model) {
+    const { schedule } = model.platform;
+    if (!schedule)
+        return "No schedule: the pipelines run only when someone starts them.";
+    const state = schedule.paused ? "paused" : "runs";
+    return `Job \`${model.names.job}\`: \`${schedule.cron}\` (${schedule.timezone}), ${state}.`;
+}
+function unique(values) {
+    return [...new Set(values)];
+}
+//# sourceMappingURL=request.js.map

package/dist/request.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAIA,gFAAgF;AAChF,MAAM,UAAU,OAAO,CAAC,KAAoB;IAC1C,OAAO;QACL;YACE,KAAK,EAAE,6BAA6B;YACpC,KAAK,EAAE,mEAAmE;YAC1E,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC;SAC5B;QACD,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE;KACjD,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAoB;IACzC,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO;QACrC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,MAAM,CACpB,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC;QACxF,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,CAAC,CAAC,CACH,CAAC;IACF,OAAO;QACL,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,OAAO,2CAA2C,CAAC;QAC7F,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,MAAM,8CAA8C,CAAC;QAC5F,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,MAAM,kCAAkC,CAAC;QAClF,QAAQ,CAAC,OAAO,CAAC,UAAU;YACzB,CAAC,CAAC,wEAAwE;YAC1E,CAAC,CAAC,6CAA6C;QACjD,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI;YACzB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC;gBACE,wHAAwH;aACzH,CAAC;KACP,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAoB;IACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;IACpC,IAAI,CAAC,QAAQ;QAAE,OAAO,+DAA+D,CAAC;IACtF,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;IAClD,OAAO,SAAS,KAAK,CAAC,KAAK,CAAC,GAAG,SAAS,QAAQ,CAAC,IAAI,OAAO,QAAQ,CAAC,QAAQ,MAAM,KAAK,GAAG,CAAC;AAC9F,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9B,CAAC"}