@rex0220/ksql-flownet 1.0.0

package/dist/executor/run-subprocess.js

@@ -0,0 +1,175 @@
+import { spawn as nodeSpawn } from "node:child_process";
+import { mkdir } from "node:fs/promises";
+import { isAbsolute, join, resolve } from "node:path";
+export class RunSubprocess {
+    options;
+    spawn;
+    constructor(options) {
+        this.options = options;
+        if (!isAbsolute(options.executionDirectory))
+            throw new Error("executionDirectory must be absolute");
+        if ((options.timeoutMs !== null &&
+            (!Number.isFinite(options.timeoutMs) || options.timeoutMs < 0)) ||
+            !Number.isFinite(options.gracePeriodMs) ||
+            options.gracePeriodMs < 0)
+            throw new Error("timeoutMs and gracePeriodMs must be non-negative");
+        this.spawn = options.spawn ?? spawnRunProcess;
+    }
+    async run(request) {
+        if (!isAbsolute(request.sqlPath))
+            throw new Error("sqlPath must be absolute");
+        for (const [name, value] of [
+            ["correlationId", request.correlationId],
+            ["attemptId", request.attemptId],
+            ["expectedJobId", request.expectedJobId],
+        ])
+            if (!/^[A-Za-z0-9._:-]{1,128}$/.test(value))
+                throw new Error(`${name} must match ^[A-Za-z0-9._:-]{1,128}$`);
+        const metadataDirectory = join(resolve(this.options.executionDirectory), "metadata");
+        await mkdir(metadataDirectory, { recursive: true });
+        const resultJsonPath = join(metadataDirectory, `${safeSegment(request.attemptId)}.json`);
+        const contractArgs = [
+            "run",
+            "-f",
+            request.sqlPath,
+            "--profile",
+            request.profile,
+            "--config",
+            request.configPath,
+            "--as-of",
+            request.asOf,
+            "--result-json",
+            resultJsonPath,
+            "--correlation-id",
+            request.correlationId,
+            "--attempt-id",
+            request.attemptId,
+            "--expected-job-id",
+            request.expectedJobId,
+        ];
+        for (const input of [...(request.imports ?? [])].sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0)) {
+            if (input.name.length === 0 ||
+                input.name.includes(":") ||
+                input.name.includes("=") ||
+                [...input.name].some((character) => {
+                    const code = character.codePointAt(0);
+                    return code <= 0x1f || code === 0x7f;
+                }) ||
+                !isAbsolute(input.path) ||
+                !/^[a-f0-9]{64}$/u.test(input.sha256)) {
+                throw new Error("import source must have a safe name, absolute path, and SHA-256");
+            }
+            contractArgs.push("--import-csv", `${input.name}=${input.path}`, "--expected-import-sha256", `${input.name}=${input.sha256}`);
+        }
+        for (const output of [...(request.exports ?? [])].sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0)) {
+            if (!isSafeIoName(output.name) || !isAbsolute(output.path)) {
+                throw new Error("export sink must have a safe name and absolute path");
+            }
+            contractArgs.push("--export-csv", `${output.name}=${output.path}`);
+        }
+        const args = [...(this.options.binArgs ?? []), ...contractArgs];
+        let stdout = "";
+        let stderr = "";
+        let handle;
+        try {
+            handle = this.spawn({
+                command: this.options.command,
+                args,
+                onStdout: (chunk) => (stdout += chunk),
+                onStderr: (chunk) => (stderr += chunk),
+            });
+        }
+        catch (error) {
+            return {
+                exitCode: null,
+                stdout,
+                stderr: `${stderr}${diagnostic(error)}`,
+                resultJsonPath,
+                timedOut: false,
+                forced: false,
+                launchFailureConfirmed: true,
+            };
+        }
+        const first = this.options.timeoutMs === null
+            ? await handle.completion
+            : await raceExit(handle.completion, this.options.timeoutMs);
+        if (first)
+            return completed(first, stdout, stderr, resultJsonPath, false, false);
+        await handle.gracefulStop();
+        const graceful = await raceExit(handle.completion, this.options.gracePeriodMs);
+        if (graceful)
+            return completed(graceful, stdout, stderr, resultJsonPath, true, false);
+        await handle.forceStop();
+        const forced = await raceExit(handle.completion, this.options.forcedExitWaitMs ?? this.options.gracePeriodMs);
+        return completed(forced ?? { exitCode: null }, stdout, stderr, resultJsonPath, true, true);
+    }
+}
+function isSafeIoName(value) {
+    return (value.length >= 1 &&
+        value.length <= 128 &&
+        value !== "__net__" &&
+        !value.includes(":") &&
+        !value.includes("=") &&
+        ![...value].some((character) => {
+            const code = character.codePointAt(0);
+            return code <= 0x1f || code === 0x7f;
+        }));
+}
+export const spawnRunProcess = (request) => {
+    const child = nodeSpawn(request.command, request.args, {
+        shell: false,
+        stdio: ["ignore", "pipe", "pipe"],
+        windowsHide: true,
+    });
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", request.onStdout);
+    child.stderr.on("data", request.onStderr);
+    const completion = new Promise((resolveCompletion) => {
+        child.once("error", (error) => resolveCompletion({
+            exitCode: null,
+            launchFailureConfirmed: true,
+            launchError: diagnostic(error),
+        }));
+        child.once("close", (exitCode) => resolveCompletion({ exitCode }));
+    });
+    return {
+        completion,
+        gracefulStop: () => {
+            child.kill(process.platform === "win32" ? "SIGINT" : "SIGTERM");
+        },
+        forceStop: () => {
+            child.kill("SIGKILL");
+        },
+    };
+};
+function completed(exit, stdout, stderr, resultJsonPath, timedOut, forced) {
+    return {
+        ...exit,
+        stdout,
+        stderr: `${stderr}${exit.launchError ?? ""}`,
+        resultJsonPath,
+        timedOut,
+        forced,
+        launchFailureConfirmed: exit.launchFailureConfirmed === true,
+    };
+}
+function raceExit(completion, timeoutMs) {
+    return new Promise((resolveRace, reject) => {
+        const timer = setTimeout(() => resolveRace(null), timeoutMs);
+        completion.then((result) => {
+            clearTimeout(timer);
+            resolveRace(result);
+        }, (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+    });
+}
+function safeSegment(value) {
+    return value.replace(/[^A-Za-z0-9._-]/g, "_");
+}
+function diagnostic(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=run-subprocess.js.map

package/dist/io/input-baseline.d.ts

@@ -0,0 +1,19 @@
+import type { InputBaseline } from "./io-path.js";
+import type { ExecutionOutputFile } from "../executor/result-classifier.js";
+export declare const INPUT_AUDIT_SUMMARY_MAX_LENGTH = 10000;
+export interface StoredInputBaseline {
+    readonly source: string;
+    readonly sha256: string;
+    readonly bytes: number;
+}
+export interface InputFileReceipt extends StoredInputBaseline {
+    readonly rows: number;
+    readonly encoding: string;
+}
+export declare function serializeInputBaseline(inputs: readonly Pick<InputBaseline, "name" | "sha256" | "bytes">[]): string;
+export declare function parseInputBaseline(summary: string | null): StoredInputBaseline[] | null;
+export declare function serializeInputAuditSummary(baseline: readonly Pick<InputBaseline, "name" | "sha256" | "bytes">[], receipts: readonly InputFileReceipt[]): string;
+export declare function serializeOutputAuditSummary(outputs: readonly ExecutionOutputFile[]): string;
+export declare function serializeIoAuditSummary(baseline: readonly Pick<InputBaseline, "name" | "sha256" | "bytes">[], inputReceipts: readonly InputFileReceipt[], outputReceipts: readonly ExecutionOutputFile[]): string;
+export declare function inputBaselinesEqual(expected: readonly StoredInputBaseline[], actual: readonly Pick<InputBaseline, "name" | "sha256" | "bytes">[]): boolean;
+//# sourceMappingURL=input-baseline.d.ts.map

package/dist/io/input-baseline.js

@@ -0,0 +1,214 @@
+export const INPUT_AUDIT_SUMMARY_MAX_LENGTH = 10_000;
+export function serializeInputBaseline(inputs) {
+    const normalized = normalizeBaseline(inputs);
+    const summary = checkedJson({
+        version: 1,
+        kind: "KSQL_FLOWNET_INPUT_BASELINE",
+        inputs: normalized,
+    });
+    // Reserve enough room for the terminal receipt form before accepting the
+    // pre-execution baseline into the fixed-size safe summary field.
+    checkedJson({
+        version: 1,
+        kind: "KSQL_FLOWNET_INPUT_AUDIT",
+        baseline: normalized,
+        input_files: normalized.map((input) => ({
+            source: input.source,
+            type: "IMPORT",
+            sha256: input.sha256.slice(0, 12),
+            bytes: input.bytes,
+            rows: Number.MAX_SAFE_INTEGER,
+            encoding: "x".repeat(32),
+        })),
+    });
+    return summary;
+}
+export function parseInputBaseline(summary) {
+    if (summary === null || summary.length > INPUT_AUDIT_SUMMARY_MAX_LENGTH)
+        return null;
+    let value;
+    try {
+        value = JSON.parse(summary);
+    }
+    catch {
+        return null;
+    }
+    if (!isRecord(value))
+        return null;
+    if (value.kind === "KSQL_FLOWNET_IO_AUDIT") {
+        if (!hasOnlyKeys(value, [
+            "version",
+            "kind",
+            "baseline",
+            "input_files",
+            "output_files",
+        ]))
+            throw new Error("IO audit summary has unknown fields");
+        if (value.version !== 1 || !Array.isArray(value.baseline))
+            throw new Error("IO audit summary has an invalid version or baseline");
+        return strictBaseline(value.baseline);
+    }
+    if (value.kind === "KSQL_FLOWNET_INPUT_AUDIT") {
+        if (!hasOnlyKeys(value, ["version", "kind", "baseline", "input_files"]))
+            throw new Error("input audit summary has unknown fields");
+        if (value.version !== 1 || !Array.isArray(value.baseline))
+            throw new Error("input audit summary has an invalid version or baseline");
+        return strictBaseline(value.baseline);
+    }
+    if (value.kind !== "KSQL_FLOWNET_INPUT_BASELINE")
+        return null;
+    if (!hasOnlyKeys(value, ["version", "kind", "inputs"]))
+        throw new Error("input baseline summary has unknown fields");
+    if (value.version !== 1 || !Array.isArray(value.inputs))
+        throw new Error("input baseline summary has an invalid version or inputs");
+    return strictBaseline(value.inputs);
+}
+export function serializeInputAuditSummary(baseline, receipts) {
+    const normalizedBaseline = normalizeBaseline(baseline);
+    const normalizedReceipts = [...receipts]
+        .sort((left, right) => compareText(left.source, right.source))
+        .map((receipt) => {
+        assertStoredBaseline(receipt);
+        if (!Number.isSafeInteger(receipt.rows) || receipt.rows < 0)
+            throw new Error("input receipt rows must be a non-negative integer");
+        if (!/^[A-Za-z0-9._-]{1,32}$/u.test(receipt.encoding))
+            throw new Error("input receipt encoding is invalid");
+        return {
+            source: receipt.source,
+            type: "IMPORT",
+            sha256: receipt.sha256.slice(0, 12),
+            bytes: receipt.bytes,
+            rows: receipt.rows,
+            encoding: receipt.encoding,
+        };
+    });
+    return checkedJson({
+        version: 1,
+        kind: "KSQL_FLOWNET_INPUT_AUDIT",
+        baseline: normalizedBaseline,
+        input_files: normalizedReceipts,
+    });
+}
+export function serializeOutputAuditSummary(outputs) {
+    return checkedJson({
+        version: 1,
+        kind: "KSQL_FLOWNET_OUTPUT_AUDIT",
+        output_files: normalizeOutputReceipts(outputs),
+    });
+}
+export function serializeIoAuditSummary(baseline, inputReceipts, outputReceipts) {
+    const inputAudit = JSON.parse(serializeInputAuditSummary(baseline, inputReceipts));
+    return checkedJson({
+        version: 1,
+        kind: "KSQL_FLOWNET_IO_AUDIT",
+        baseline: inputAudit.baseline,
+        input_files: inputAudit.input_files,
+        output_files: normalizeOutputReceipts(outputReceipts),
+    });
+}
+function normalizeOutputReceipts(outputs) {
+    const normalized = outputs.map((output) => {
+        if (!isSafeSourceName(output.name) ||
+            !/^[a-f0-9]{64}$/u.test(output.sha256) ||
+            !Number.isSafeInteger(output.bytes) ||
+            output.bytes < 0 ||
+            !Number.isSafeInteger(output.rows) ||
+            output.rows < 0 ||
+            !["utf8", "sjis"].includes(output.encoding))
+            throw new Error("output receipt contains invalid audit metadata");
+        return {
+            sink: output.name,
+            type: "EXPORT",
+            sha256: output.sha256,
+            bytes: output.bytes,
+            rows: output.rows,
+            encoding: output.encoding,
+        };
+    });
+    if (new Set(normalized.map((output) => output.sink)).size !== normalized.length)
+        throw new Error("output receipt sink names must be unique");
+    return normalized;
+}
+export function inputBaselinesEqual(expected, actual) {
+    return serializeStoredBaseline(expected) === serializeInputBaseline(actual);
+}
+function serializeStoredBaseline(inputs) {
+    for (const input of inputs)
+        assertStoredBaseline(input);
+    return checkedJson({
+        version: 1,
+        kind: "KSQL_FLOWNET_INPUT_BASELINE",
+        inputs: [...inputs].sort((left, right) => compareText(left.source, right.source)),
+    });
+}
+function normalizeBaseline(inputs) {
+    const normalized = inputs
+        .map((input) => ({
+        source: input.name,
+        sha256: input.sha256,
+        bytes: input.bytes,
+    }))
+        .sort((left, right) => compareText(left.source, right.source));
+    for (const input of normalized)
+        assertStoredBaseline(input);
+    if (new Set(normalized.map((input) => input.source)).size !== normalized.length)
+        throw new Error("input baseline source names must be unique");
+    return normalized;
+}
+function strictBaseline(value) {
+    const inputs = value.map((item) => {
+        if (!isRecord(item) || !hasOnlyKeys(item, ["source", "sha256", "bytes"]))
+            throw new Error("input baseline entry has an invalid shape");
+        const baseline = {
+            source: item.source,
+            sha256: item.sha256,
+            bytes: item.bytes,
+        };
+        assertStoredBaseline(baseline);
+        return baseline;
+    });
+    const canonical = [...inputs].sort((left, right) => compareText(left.source, right.source));
+    if (JSON.stringify(inputs) !== JSON.stringify(canonical))
+        throw new Error("input baseline entries are not in canonical source order");
+    if (new Set(inputs.map((input) => input.source)).size !== inputs.length)
+        throw new Error("input baseline source names must be unique");
+    return inputs;
+}
+function assertStoredBaseline(value) {
+    if (typeof value.source !== "string" ||
+        !isSafeSourceName(value.source) ||
+        typeof value.sha256 !== "string" ||
+        !/^[a-f0-9]{64}$/u.test(value.sha256) ||
+        !Number.isSafeInteger(value.bytes) ||
+        value.bytes < 0)
+        throw new Error("input baseline accepts only source, SHA-256, and byte count");
+}
+function isSafeSourceName(value) {
+    return (value.length >= 1 &&
+        value.length <= 128 &&
+        value !== "__net__" &&
+        !value.includes(":") &&
+        !value.includes("=") &&
+        ![...value].some((character) => {
+            const code = character.codePointAt(0);
+            return code <= 0x1f || code === 0x7f;
+        }));
+}
+function checkedJson(value) {
+    const summary = JSON.stringify(value);
+    if (summary.length > INPUT_AUDIT_SUMMARY_MAX_LENGTH)
+        throw new Error(`input audit summary exceeds ${INPUT_AUDIT_SUMMARY_MAX_LENGTH} characters`);
+    return summary;
+}
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function hasOnlyKeys(value, keys) {
+    const actual = Object.keys(value).sort(compareText);
+    const expected = [...keys].sort(compareText);
+    return JSON.stringify(actual) === JSON.stringify(expected);
+}
+function compareText(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+//# sourceMappingURL=input-baseline.js.map

package/dist/io/io-config.d.ts

@@ -0,0 +1,6 @@
+export interface IoConfig {
+    readonly root: string;
+    readonly retentionDays: number;
+}
+export declare function loadIoConfig(environment?: NodeJS.ProcessEnv): IoConfig;
+//# sourceMappingURL=io-config.d.ts.map

package/dist/io/io-config.js

@@ -0,0 +1,33 @@
+import { lstatSync } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
+export function loadIoConfig(environment = process.env) {
+    const configuredRoot = environment.KSQL_FLOWNET_IO_DIR;
+    if (configuredRoot === undefined || configuredRoot.trim() === "") {
+        throw new Error("KSQL_FLOWNET_IO_DIR is required for networks with inputs or outputs");
+    }
+    if (!isAbsolute(configuredRoot)) {
+        throw new Error("KSQL_FLOWNET_IO_DIR must be an absolute path");
+    }
+    const root = resolve(configuredRoot);
+    let stat;
+    try {
+        stat = lstatSync(root);
+    }
+    catch (error) {
+        throw new Error("KSQL_FLOWNET_IO_DIR must be an existing directory", {
+            cause: error,
+        });
+    }
+    if (!stat.isDirectory()) {
+        throw new Error("KSQL_FLOWNET_IO_DIR must be an existing directory");
+    }
+    const rawRetention = environment.KSQL_FLOWNET_IO_RETENTION_DAYS;
+    const retentionDays = rawRetention === undefined || rawRetention.trim() === ""
+        ? 90
+        : Number(rawRetention);
+    if (!Number.isSafeInteger(retentionDays) || retentionDays <= 0) {
+        throw new Error("KSQL_FLOWNET_IO_RETENTION_DAYS must be a positive integer");
+    }
+    return { root, retentionDays };
+}
+//# sourceMappingURL=io-config.js.map

package/dist/io/io-path.d.ts

@@ -0,0 +1,37 @@
+export type InputPathErrorCode = "INPUT_PATH_REJECTED" | "INPUT_FILE_MISSING" | "INPUT_FILE_MUTATED";
+export declare class InputPathError extends Error {
+    readonly code: InputPathErrorCode;
+    constructor(code: InputPathErrorCode, message: string, options?: ErrorOptions);
+}
+export declare class OutputPathError extends Error {
+    readonly code: "OUTPUT_PATH_REJECTED";
+    constructor(message: string, options?: ErrorOptions);
+}
+export interface InputBaseline {
+    readonly name: string;
+    readonly path: string;
+    readonly sha256: string;
+    readonly bytes: number;
+}
+export interface ResolveNodeInputsInput {
+    readonly ioRoot: string;
+    readonly patterns: Readonly<Record<string, string>>;
+    readonly businessKey: string;
+    readonly profile: string;
+}
+export interface ResolvedOutput {
+    readonly name: string;
+    readonly path: string;
+}
+export interface ResolveNodeOutputsInput {
+    readonly ioRoot: string;
+    readonly patterns: Readonly<Record<string, string>>;
+    readonly businessKey: string;
+    readonly profile: string;
+    readonly runId: string;
+    readonly nodeId: string;
+}
+export declare function resolveNodeOutputs(input: ResolveNodeOutputsInput): Promise<readonly ResolvedOutput[]>;
+export declare function resolveNodeInputs(input: ResolveNodeInputsInput): Promise<readonly InputBaseline[]>;
+export declare function percentEncodePathSegment(value: string): string;
+//# sourceMappingURL=io-path.d.ts.map