@ingestron/core 0.12.0

package/dist/core/config.js

@@ -0,0 +1,233 @@
+import { readFileSync, realpathSync, statSync } from "node:fs";
+import { dirname, relative, resolve, isAbsolute } from "node:path";
+import { parseDocument, LineCounter } from "yaml";
+import { check, Problem, digest, isMap } from "./errors.js";
+export class Configuration {
+    environmentVariables;
+    root;
+    files = {};
+    locations = new WeakMap();
+    inputs = {};
+    pending = [];
+    bytes = 0;
+    nodes = 0;
+    documents = new Map();
+    constructor(root, environmentVariables = process.env) {
+        this.environmentVariables = environmentVariables;
+        this.root = realpathSync(root);
+    }
+    path(name, from = resolve(this.root, "project.yaml")) {
+        let file;
+        try {
+            file = realpathSync(resolve(dirname(from), name));
+        }
+        catch {
+            throw new Problem("FILE", `Missing file: ${name}`, from);
+        }
+        const rel = relative(this.root, file);
+        check(rel !== ".." && !rel.startsWith("../") && !isAbsolute(rel), "PATH", "Reference leaves the project", from);
+        check(statSync(file).isFile(), "FILE", "Expected a regular file", file);
+        return file;
+    }
+    text(name, from) {
+        const file = this.path(name, from), size = statSync(file).size;
+        check(size <= 10 * 1024 * 1024, "LIMIT", "A source file must be at most 10 MiB", file);
+        if (!Object.hasOwn(this.files, relative(this.root, file)))
+            this.bytes += size;
+        check(this.bytes <= 50 * 1024 * 1024, "LIMIT", "Project inputs exceed 50 MiB", file);
+        const value = readFileSync(file, "utf8");
+        this.files[relative(this.root, file)] = digest(value);
+        return value;
+    }
+    document(file) {
+        if (!this.documents.has(file)) {
+            const lines = new LineCounter(), doc = parseDocument(this.text(file), {
+                uniqueKeys: true,
+                lineCounter: lines,
+                strict: true,
+            });
+            check(!doc.errors.length && !doc.warnings.length, "YAML", `${doc.errors[0]?.message ?? doc.warnings[0]?.message}`, file);
+            let value;
+            try {
+                value = doc.toJS({ maxAliasCount: 50 });
+            }
+            catch {
+                throw new Problem("LIMIT", "YAML aliases exceed the limit", file);
+            }
+            this.documents.set(file, { doc, lines, value });
+        }
+        return this.documents.get(file);
+    }
+    load(name = "project.yaml", from = resolve(this.root, "project.yaml"), stack = [], depth = 0) {
+        check(depth < 64, "LIMIT", "Includes exceed 64 levels", from);
+        check(!/^[a-z]+:/i.test(name), "REFERENCE", "Only local configuration includes are allowed", from);
+        const index = name.indexOf("#"), path = index < 0 ? name : name.slice(0, index);
+        let pointer = "";
+        try {
+            pointer = index < 0 ? "" : decodeURIComponent(name.slice(index + 1));
+        }
+        catch {
+            throw new Problem("POINTER", "Invalid fragment encoding", from);
+        }
+        check(!pointer || pointer.startsWith("/"), "POINTER", "Expected a JSON Pointer fragment", from);
+        const file = path ? this.path(path, from) : from, key = file + "#" + pointer;
+        check(!stack.includes(key), "CYCLE", `Reference cycle: ${[...stack, key].join(" → ")}`, file);
+        let value = this.document(file).value;
+        for (const token of pointer ? pointer.slice(1).split("/") : []) {
+            check(!/~(?![01])/.test(token), "POINTER", "Invalid pointer escape", file);
+            const part = token.replaceAll("~1", "/").replaceAll("~0", "~");
+            check(!["__proto__", "constructor", "prototype"].includes(part) &&
+                value != null &&
+                typeof value === "object" &&
+                Object.hasOwn(value, part) &&
+                (!Array.isArray(value) || /^(0|[1-9]\d*)$/.test(part)), "POINTER", `Missing pointer ${pointer}`, file);
+            value = value[part];
+        }
+        return this.walk(value, file, pointer, [...stack, key], depth + 1);
+    }
+    walk(value, file, pointer, stack, depth) {
+        check(++this.nodes <= 200000 && depth <= 64, "LIMIT", "Configuration exceeds structural limits", file);
+        if (Array.isArray(value))
+            return value.map((v, i) => this.walk(v, file, `${pointer}/${i}`, stack, depth + 1));
+        if (!isMap(value))
+            return value;
+        for (const key of Object.keys(value))
+            check(!["__proto__", "constructor", "prototype"].includes(key), "KEY", "Unsafe mapping key", file);
+        if (Object.hasOwn(value, "$resolve")) {
+            check(Object.keys(value).length === 1 && typeof value.$resolve === "string", "REFERENCE", "$resolve must be the only key", file);
+            return this.load(value.$resolve, file, stack, depth);
+        }
+        const out = {};
+        for (const [key, child] of Object.entries(value)) {
+            if (["sqlFile", "pythonFile", "moduleFile"].includes(key) &&
+                typeof child === "string") {
+                const target = this.path(child, file);
+                this.text(target);
+                out[key] = relative(this.root, target);
+            }
+            else if (key === "projectDirectory" && typeof child === "string") {
+                const target = realpathSync(resolve(dirname(file), child)), rel = relative(this.root, target);
+                check(rel !== ".." &&
+                    !rel.startsWith("../") &&
+                    !isAbsolute(rel) &&
+                    statSync(target).isDirectory(), "PATH", "Project directory must stay inside the project", file);
+                out[key] = rel;
+            }
+            else if (key === "uses" &&
+                typeof child === "string" &&
+                child.startsWith(".")) {
+                const target = this.path(child, file);
+                this.text(target);
+                out[key] = "./" + relative(this.root, target);
+            }
+            else
+                out[key] = this.walk(child, file, pointer + "/" + key.replaceAll("~", "~0").replaceAll("/", "~1"), stack, depth + 1);
+        }
+        const entry = this.document(file), keys = pointer
+            ? pointer
+                .slice(1)
+                .split("/")
+                .map((k) => k.replaceAll("~1", "/").replaceAll("~0", "~"))
+            : [];
+        const node = keys.length ? entry.doc.getIn(keys, true) : entry.doc.contents;
+        this.locations.set(out, {
+            file: relative(this.root, file),
+            pointer,
+            ...entry.lines.linePos(node?.range?.[0] ?? 0),
+            column: entry.lines.linePos(node?.range?.[0] ?? 0).col,
+        });
+        return out;
+    }
+    parse(schema, value) {
+        const result = schema.safeParse(value);
+        if (result.success)
+            return result.data;
+        const issue = result.error.issues[0];
+        let owner = value;
+        for (const key of issue.path.slice(0, -1))
+            if (owner && typeof owner === "object")
+                owner = owner[key];
+        const where = this.locations.get(owner) ??
+            (isMap(value) ? this.locations.get(value) : undefined);
+        throw new Problem("SCHEMA", `${issue.path.join(".")}: ${issue.message}`, where?.file, where?.pointer, "Use config_explain to inspect the originating value");
+    }
+    values(value, context, options = {}) {
+        const resolving = new Set();
+        const lookup = (name) => {
+            check(!resolving.has(name), "CYCLE", `Value cycle at ${name}`);
+            const parts = name.split(".");
+            let current = context;
+            for (const key of parts) {
+                check(!["__proto__", "constructor", "prototype"].includes(key), "KEY", "Unsafe value reference");
+                current =
+                    isMap(current) && Object.hasOwn(current, key)
+                        ? current[key]
+                        : undefined;
+            }
+            check(current !== undefined, "VALUE", `Unknown value {{${name}}}`);
+            resolving.add(name);
+            const result = visit(current);
+            resolving.delete(name);
+            return result;
+        };
+        const visit = (v) => {
+            if (Array.isArray(v))
+                return v.map(visit);
+            if (isMap(v)) {
+                if (Object.hasOwn(v, "$env")) {
+                    check(Object.keys(v).length === 1 &&
+                        typeof v.$env === "string" &&
+                        /^[A-Za-z_][A-Za-z0-9_]*$/.test(v.$env), "ENV", "$env requires one environment variable name");
+                    const supplied = this.environmentVariables[v.$env];
+                    if (supplied === undefined || supplied === "") {
+                        const location = this.locations.get(v);
+                        this.pending.push({
+                            name: v.$env,
+                            file: location?.file,
+                            pointer: location?.pointer,
+                        });
+                        check(options.draft, "INPUT", `Missing environment variable ${v.$env}`, location?.file);
+                        return { $env: v.$env };
+                    }
+                    this.inputs[v.$env] = digest(supplied);
+                    return supplied;
+                }
+                if (Object.hasOwn(v, "$secret")) {
+                    check(Object.keys(v).length === 1 && isMap(v.$secret), "SECRET", "$secret must describe a runtime lookup");
+                    return structuredClone(v);
+                }
+                return Object.fromEntries(Object.entries(v).map(([k, child]) => [
+                    k,
+                    ["sql", "query", "python", "script"].includes(k)
+                        ? child
+                        : visit(child),
+                ]));
+            }
+            if (typeof v !== "string")
+                return v;
+            const all = [...v.matchAll(/{{\s*([A-Za-z_][\w.]*)\s*}}/g)];
+            if (!all.length)
+                return v;
+            const tokenValue = (name) => {
+                if (name.startsWith("run.") && options.runtime) {
+                    check(["run.id", "run.date"].includes(name), "VALUE", `Unknown runtime placeholder ${name}; use run.id or run.date`);
+                    return `{{${name}}}`;
+                }
+                if ((name.startsWith("table.") || name.startsWith("step.")) &&
+                    options.table)
+                    return `{{${name}}}`;
+                return lookup(name);
+            };
+            if (all.length === 1 && all[0][0] === v)
+                return tokenValue(all[0][1]);
+            return v.replace(/{{\s*([A-Za-z_][\w.]*)\s*}}/g, (_, name) => {
+                const resolved = tokenValue(name);
+                if (options.draft && isMap(resolved) && resolved.$env)
+                    return `{{${name}}}`;
+                check(["string", "number", "boolean"].includes(typeof resolved), "VALUE", `Cannot embed non-scalar ${name} in text`);
+                return String(resolved);
+            });
+        };
+        return visit(value);
+    }
+}

package/dist/core/connections.d.ts

@@ -0,0 +1,19 @@
+export declare function prepareConnection(root: string, environment: string, flowId: string, validateOnly?: boolean, reviewFile?: string, configureExecution?: (execution: Record<string, any>) => Record<string, any>): {
+    project: string;
+    environment: string;
+    configuration: string;
+    provider: string;
+    version: string;
+    commit: string;
+    apiVersion: string;
+    command: string;
+    execution: string;
+    inputDigest: string;
+    reportLock?: Record<string, unknown> | undefined;
+    result: import("../plugins/provider-renderer.js").ProviderAssets;
+} | {
+    valid: boolean;
+    specificationSha256: string;
+    connection: string;
+    flow: string;
+};

package/dist/core/connections.js

@@ -0,0 +1,203 @@
+import { readFileSync } from "node:fs";
+import { fence } from "./packages.js";
+import { connectorPackageSchema } from "./connector-package.js";
+import { contractColumns } from "./contracts.js";
+/** Project-owned configuration; schemas and runtime assets remain plugin-owned. */
+import { Ajv2020 } from "ajv/dist/2020.js";
+import { Configuration } from "./config.js";
+import { projectSchema, flowSchema } from "./schema.js";
+import { canonical, check, digest, isMap } from "./errors.js";
+import { canonicalPackageReference, resolvePackage, packageYaml, providerPackageSchema, } from "./packages.js";
+import { checkProviderCompatibility } from "../plugins/compatibility.js";
+import { runProviderCommand, validateSchemaShape, } from "./provider-commands.js";
+function validate(schema, value, label) {
+    validateSchemaShape(schema);
+    const ajv = new Ajv2020({ strict: true, validateFormats: false });
+    const fn = ajv.compile(schema);
+    check(fn(value), "CONNECTION", `${label}: ${ajv.errorsText(fn.errors)}`);
+}
+export function prepareConnection(root, environment, flowId, validateOnly = false, reviewFile, configureExecution) {
+    // Do not interpolate $env here: connection credentials must never enter compiler memory/output.
+    const reader = new Configuration(root, {});
+    const project = reader.parse(projectSchema, reader.load());
+    const profile = project.environments[environment];
+    check(profile, "ENVIRONMENT", `Unknown environment ${environment}`);
+    check(!profile.environment || profile.environment === environment, "ENVIRONMENT", "Environment name differs from selected profile");
+    const flows = project.flows.map((f) => reader.parse(flowSchema, f));
+    check(new Set(flows.map((f) => f.id)).size === flows.length, "FLOW", "Duplicate flow IDs");
+    const flow = flows.find((f) => f.id === flowId);
+    check(flow && flow.kind === "ingestion" && !flow.steps, "FLOW", "Select an ingestion flow");
+    const ingestion = flow.ingestion;
+    check(isMap(ingestion) &&
+        Object.keys(ingestion).every((k) => ["connection", "execution", "timeoutSeconds"].includes(k)), "CONNECTION", "Connection ingestion accepts connection, execution and timeoutSeconds only");
+    check(!Object.keys(flow.requires).length && !Object.keys(flow.publishes).length, "CONNECTION", "Downstream dataset handovers require reviewed contracts");
+    const connection = project.connections[String(ingestion.connection)];
+    check(connection, "CONNECTION", "Unknown ingestion connection");
+    const configured = project.providers.configurations[flow.provider ?? project.defaults.provider ?? ""];
+    check(configured, "PROVIDER", "Select the execution provider configuration");
+    const owner = project.providers.packages[connection.package];
+    const executor = project.providers.packages[configured.package];
+    check(owner && executor, "PACKAGE", "Unknown connection/execution package");
+    const load = (pkg) => {
+        const ref = canonicalPackageReference(`${pkg.source}@${pkg.version}`);
+        const locked = resolvePackage(root, ref);
+        const manifest = providerPackageSchema.parse(packageYaml(locked.file));
+        checkProviderCompatibility(manifest);
+        return { ref, locked, manifest };
+    };
+    const sourceRef = canonicalPackageReference(`${owner.source}@${owner.version}`);
+    const sourceLock = resolvePackage(root, sourceRef);
+    const sourceRaw = packageYaml(sourceLock.file);
+    check(sourceRaw.apiVersion === "ingestron.connector/v1", "CONNECTION", "Select an independently installed source connector package");
+    const connectorPackage = connectorPackageSchema.parse(sourceRaw);
+    const source = { ref: sourceRef, locked: sourceLock };
+    const target = load(executor);
+    const connector = connectorPackage.connector;
+    check(!connection.connector || connection.connector === connector, "CONNECTION", "Connection must use the exact installed connector identity");
+    check(connectorPackage.upstream.licenceStatus === "evidenced", "LICENCE", "Connector licence evidence is unresolved");
+    const descriptor = connectorPackage.definition;
+    const runtimeContract = connectorPackage.runtime.contract;
+    const adapter = target.manifest.connectorRuntimes?.[runtimeContract];
+    check(adapter &&
+        adapter.executionPlatforms.includes(target.manifest.platform) &&
+        Object.hasOwn(connectorPackage.execution, target.manifest.platform), "CONNECTION", "Selected execution provider does not support this connector runtime");
+    const binding = connection.binding
+        ? profile.bindings[connection.binding]
+        : {};
+    check(binding, "CONNECTION", "Missing connection environment binding");
+    let settings = { ...connection.settings, ...binding };
+    const rejectEnvironment = (v) => {
+        if (Array.isArray(v))
+            return v.forEach(rejectEnvironment);
+        if (!isMap(v))
+            return;
+        check(!Object.hasOwn(v, "$env"), "SECRET", "Use runtime $secret references, not compiler $env values, in connections");
+        Object.values(v).forEach(rejectEnvironment);
+    };
+    rejectEnvironment(settings);
+    const context = {
+        env: environment,
+        project: { id: project.id },
+        values: profile.values,
+    };
+    settings = reader.values(settings, context);
+    rejectEnvironment(settings);
+    validate(descriptor.settingsSchema, settings, "Connection settings");
+    check(flow.tables && Object.keys(flow.tables).length > 0, "CONNECTION", "Select at least one table with an ODCS contract");
+    const streams = new Set();
+    const tables = Object.fromEntries(Object.entries(flow.tables).map(([name, table]) => {
+        check(Object.keys(table.source).length === 1 &&
+            typeof table.source.stream === "string", "CONNECTION", `${name}: source must identify one connector stream`);
+        check(!streams.has(table.source.stream), "CONNECTION", "Duplicate source stream");
+        streams.add(table.source.stream);
+        check(!table.ingestion && !Object.keys(table.steps).length, "CONNECTION", "Connection tables do not accept ingestion/step overrides");
+        return [
+            name,
+            {
+                source: table.source,
+                contract: table.contract,
+                columns: contractColumns(table.contract),
+            },
+        ];
+    }));
+    rejectEnvironment(ingestion.execution);
+    let execution = reader.values(ingestion.execution ?? { mode: "local" }, context);
+    rejectEnvironment(execution);
+    if (configureExecution)
+        execution = configureExecution(execution);
+    else if (target.manifest.projectAssembly) {
+        const configuration = flow.provider ?? project.defaults.provider;
+        const configured = project.providers.configurations[configuration];
+        const result = runProviderCommand(root, environment, {
+            configuration,
+            command: target.manifest.projectAssembly.command,
+            input: {
+                phase: "configure",
+                project: project.id,
+                environment,
+                configuration,
+                flow: flowId,
+                scope: { id: "full", partial: false },
+                execution,
+                binding: profile.bindings[configured.binding],
+                options: configured.options,
+            },
+        });
+        check(isMap(result.result?.execution), "PROVIDER", "Invalid project execution configuration");
+        execution = result.result.execution;
+    }
+    rejectEnvironment(execution);
+    validate(adapter.executionSchema, execution, "Execution settings");
+    check(connectorPackage.execution[target.manifest.platform]?.modes.includes(execution.mode), "CONNECTION", "Connector package does not support the selected execution mode");
+    const timeoutSeconds = ingestion.timeoutSeconds ?? 600;
+    check(Number.isInteger(timeoutSeconds) &&
+        Number(timeoutSeconds) >= 1 &&
+        Number(timeoutSeconds) <= 604800, "CONNECTION", "Timeout must be 1–604800 seconds");
+    const asset = connectorPackage.runtime;
+    check(Object.hasOwn(sourceLock.entry.files, asset.path), "PACKAGE", "Runtime asset is not locked");
+    const content = readFileSync(fence(sourceLock.root, asset.path), "utf8");
+    check(digest(content) === asset.sha256, "PACKAGE", "Runtime asset digest mismatch");
+    const parsed = JSON.parse(content);
+    check(isMap(parsed) &&
+        Object.keys(parsed).length <= 100 &&
+        Object.entries(parsed).every(([name, value]) => /^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(name) &&
+            typeof value === "string"), "PACKAGE", "Invalid runtime asset map");
+    const runtimeAssets = parsed;
+    const specification = {
+        runtimeContract,
+        runtimeAssetSha256: connectorPackage.runtime.sha256,
+        apiVersion: "ingestron.connection-request/v1",
+        project: project.id,
+        environment,
+        flow: flowId,
+        connection: String(ingestion.connection),
+        connector,
+        sourceId: connection.sourceId,
+        tenantId: connection.tenantId,
+        settings,
+        tables,
+        execution,
+        timeoutSeconds,
+        sourcePackage: {
+            reference: source.ref,
+            commit: source.locked.entry.commit,
+        },
+        executionPackage: {
+            reference: target.ref,
+            commit: target.locked.entry.commit,
+        },
+    };
+    const input = {
+        ...specification,
+        specificationSha256: digest(canonical(specification)),
+    };
+    if (reviewFile) {
+        const review = JSON.parse(reader.text(reviewFile));
+        check(review.status === "approved" &&
+            review.identity?.projectSpecSha256 === input.specificationSha256, "CONNECTION", "Review does not match the current project specification");
+        check(canonical(review.contracts) ===
+            canonical(Object.fromEntries(Object.entries(flow.tables).map(([name, table]) => [
+                name,
+                table.contract,
+            ]))), "CONNECTION", "Review selection differs from project");
+        check(adapter.contractsCommand, "CONNECTION", "Provider does not declare contract export");
+        return runProviderCommand(root, environment, {
+            configuration: flow.provider ?? project.defaults.provider,
+            command: adapter.contractsCommand,
+            input: { review },
+        });
+    }
+    const result = runProviderCommand(root, environment, {
+        configuration: flow.provider ?? project.defaults.provider,
+        command: adapter.prepareCommand,
+        input: { ...input, runtimeAssets },
+    });
+    return validateOnly
+        ? {
+            valid: true,
+            specificationSha256: input.specificationSha256,
+            connection: input.connection,
+            flow: flowId,
+        }
+        : result;
+}

package/dist/core/connector-package.d.ts

@@ -0,0 +1,37 @@
+/** Source definitions are data-only; execution remains with a selected provider. */
+import { z } from "zod";
+export declare const connectorPackageSchema: z.ZodObject<{
+    apiVersion: z.ZodLiteral<"ingestron.connector/v1">;
+    id: z.ZodString;
+    version: z.ZodString;
+    description: z.ZodString;
+    connector: z.ZodString;
+    documentation: z.ZodString;
+    upstream: z.ZodObject<{
+        ecosystem: z.ZodString;
+        variant: z.ZodString;
+        package: z.ZodString;
+        version: z.ZodString;
+        repository: z.ZodString;
+        licence: z.ZodString;
+        licenceFile: z.ZodString;
+        licenceStatus: z.ZodEnum<{
+            conflict: "conflict";
+            evidenced: "evidenced";
+            unknown: "unknown";
+        }>;
+    }, z.core.$strict>;
+    runtime: z.ZodObject<{
+        contract: z.ZodString;
+        path: z.ZodString;
+        sha256: z.ZodString;
+    }, z.core.$strict>;
+    definition: z.ZodObject<{
+        settingsSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        selectionSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$strict>;
+    execution: z.ZodRecord<z.ZodString, z.ZodObject<{
+        modes: z.ZodArray<z.ZodString>;
+        evidence: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;

package/dist/core/connector-package.js

@@ -0,0 +1,43 @@
+/** Source definitions are data-only; execution remains with a selected provider. */
+import { z } from "zod";
+const schema = z.record(z.string(), z.unknown());
+export const connectorPackageSchema = z
+    .object({
+    apiVersion: z.literal("ingestron.connector/v1"),
+    id: z.string().regex(/^[a-z][a-z0-9-]*$/),
+    version: z.string().regex(/^\d+\.\d+\.\d+$/),
+    description: z.string().min(1),
+    connector: z
+        .string()
+        .regex(/^[a-z][a-z0-9-]*:[a-z][a-z0-9-]*@\d+\.\d+\.\d+$/),
+    documentation: z.string().url(),
+    upstream: z
+        .object({
+        ecosystem: z.string().regex(/^[a-z][a-z0-9-]*$/),
+        variant: z.string().min(1),
+        package: z.string().min(1),
+        version: z.string().regex(/^\d+\.\d+\.\d+$/),
+        repository: z.string().url(),
+        licence: z.string().min(1),
+        licenceFile: z.string().min(1),
+        licenceStatus: z.enum(["evidenced", "unknown", "conflict"]),
+    })
+        .strict(),
+    runtime: z
+        .object({
+        contract: z.string().min(1),
+        path: z.string().min(1),
+        sha256: z.string().regex(/^[a-f0-9]{64}$/),
+    })
+        .strict(),
+    definition: z
+        .object({ settingsSchema: schema, selectionSchema: schema })
+        .strict(),
+    execution: z.record(z.string(), z
+        .object({
+        modes: z.array(z.string().regex(/^[a-z][a-z0-9-]*$/)).min(1),
+        evidence: z.string().min(1),
+    })
+        .strict()),
+})
+    .strict();

package/dist/core/contracts.d.ts

@@ -0,0 +1,8 @@
+export declare function validateContract(contract: Record<string, any>, file?: string): void;
+export interface Column {
+    name: string;
+    type: string;
+    required: boolean;
+    key: boolean;
+}
+export declare function contractColumns(contract: Record<string, any>, file?: string): Column[];

package/dist/core/contracts.js

@@ -0,0 +1,45 @@
+import { assetPath } from "./installation.js";
+import { readFileSync } from "node:fs";
+import { Ajv2019 } from "ajv/dist/2019.js";
+import addFormats from "ajv-formats";
+import { check } from "./errors.js";
+const ajv = new Ajv2019({ allErrors: true, strict: false });
+addFormats(ajv);
+const validator = ajv.compile(JSON.parse(readFileSync(assetPath("schemas/odcs-3.1.0.json"), "utf8")));
+export function validateContract(contract, file) {
+    check(contract.apiVersion === "v3.1.0", "ODCS", "Use pinned ODCS v3.1.0; other versions are not silently converted", file);
+    check(validator(contract), "ODCS", ajv.errorsText(validator.errors, { separator: "; " }), file);
+}
+export function contractColumns(contract, file) {
+    validateContract(contract, file);
+    check(contract.schema?.length === 1 && contract.schema[0].properties?.length, "ODCS", "A dataset contract must contain one table schema with columns", file);
+    const names = new Set();
+    return contract.schema[0].properties.map((p) => {
+        const name = p.physicalName ?? p.name;
+        check(typeof name === "string" &&
+            /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
+            !names.has(name), "ODCS", `Invalid or duplicate column ${name}`, file);
+        names.add(name);
+        const type = (p.physicalType ??
+            {
+                string: "STRING",
+                integer: "BIGINT",
+                number: "DOUBLE",
+                boolean: "BOOLEAN",
+                date: "DATE",
+                timestamp: "TIMESTAMP",
+            }[p.logicalType])?.toUpperCase();
+        check(typeof type === "string" &&
+            /^(STRING|BIGINT|INT|INTEGER|SMALLINT|DOUBLE|FLOAT|BOOLEAN|DATE|TIMESTAMP|BINARY|DECIMAL\(\d{1,2},\s*\d{1,2}\))$/.test(type), "ODCS", `Unsupported native type ${type} for ${name}`, file);
+        if (type.startsWith("DECIMAL")) {
+            const [precision, scale] = type.match(/\d+/g).map(Number);
+            check(precision >= 1 && precision <= 38 && scale <= precision, "ODCS", `Invalid decimal type ${type}`);
+        }
+        return {
+            name,
+            type,
+            required: p.required === true,
+            key: p.primaryKey === true,
+        };
+    });
+}

package/dist/core/deliveries.d.ts

@@ -0,0 +1,16 @@
+import { z } from "zod";
+/** Producer/consumer boundary; not an assertion that an upstream delivery exists. */
+export declare const deliveryIndexSchema: z.ZodObject<{
+    apiVersion: z.ZodLiteral<"ingestron.delivery-index/v1">;
+    dataset: z.ZodString;
+    deliveries: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        version: z.ZodNumber;
+        capturedAt: z.ZodISODateTime;
+        contractVersion: z.ZodString;
+        complete: z.ZodLiteral<true>;
+        scope: z.ZodLiteral<"full-table">;
+        rowCount: z.ZodNumber;
+        path: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;

package/dist/core/deliveries.js

@@ -0,0 +1,28 @@
+import { z } from "zod";
+/** Producer/consumer boundary; not an assertion that an upstream delivery exists. */
+export const deliveryIndexSchema = z
+    .object({
+    apiVersion: z.literal("ingestron.delivery-index/v1"),
+    dataset: z
+        .string()
+        .regex(/^[A-Za-z_][\w-]*\.[A-Za-z_][\w-]*\.[A-Za-z_][\w-]*$/),
+    deliveries: z
+        .array(z
+        .object({
+        id: z.string().min(1),
+        version: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
+        capturedAt: z.iso.datetime({ offset: true }),
+        contractVersion: z.string().min(1),
+        complete: z.literal(true),
+        scope: z.literal("full-table"),
+        rowCount: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
+        path: z
+            .string()
+            .min(1)
+            .max(4096)
+            .refine((value) => !value.includes("\0"), "Delivery location cannot contain NUL"),
+    })
+        .strict())
+        .min(1),
+})
+    .strict();

package/dist/core/delivery-exports.d.ts

@@ -0,0 +1,20 @@
+import type { Plan } from "./schema.js";
+/** Partition compilation only. This never schedules jobs or asserts data exists. */
+export declare function deliveryExports(plan: Plan): {
+    id: string;
+    directory: string;
+    needs: string[];
+    imports: {
+        dataset: string;
+        producer: string;
+        kind: "files" | "relation";
+        location: Record<string, unknown>;
+        contract: Record<string, unknown>;
+    }[];
+    resources: {
+        scope: string;
+        kind: string;
+        name: string;
+    }[];
+    plan: Plan;
+}[];