@rex0220/ksql-flownet 1.0.0

package/dist/domain/persistence-model.d.ts

@@ -0,0 +1,198 @@
+import type { CanonicalRecordKey } from "./canonical-record-key.js";
+export type NodeStateStatus = "WAITING" | "RUNNING" | "SUCCESS" | "FAILED" | "BLOCKED" | "SKIPPED" | "CANCELLED" | "UNKNOWN";
+export type NodeAttemptStatus = "RUNNING" | "SUCCESS" | "FAILED" | "CANCELLED" | "UNKNOWN";
+export type NetworkRunStatus = "CREATED" | "RUNNING" | "SUCCESS" | "FAILED" | "CANCELLED" | "UNKNOWN";
+export type RunInvocationStatus = "RUNNING" | "SUCCESS" | "FAILED" | "CANCELLED" | "UNKNOWN";
+export type CancelRequestState = "REQUESTED" | "ACCEPTED" | "RELEASED";
+export interface CancelRequest {
+    run_id: string;
+    state: CancelRequestState;
+    requested_by: string;
+    reason: string;
+    requested_at: string;
+    accepted_at: string | null;
+    released_at: string | null;
+    release_reason: string | null;
+    release_requested_by: string | null;
+}
+export interface ResolvedProfileSnapshot {
+    profile: string;
+    base_url: string;
+    guest_space_id: number | null;
+    timezone: string;
+    apps: Record<string, number>;
+    limits: {
+        max_api_calls: number | null;
+        max_read_rows: number | null;
+        batch_timeout_sec: number | null;
+    };
+}
+export interface NetworkRun {
+    run_id: string;
+    network_id: string;
+    business_key: string;
+    max_active_runs: number;
+    status: NetworkRunStatus;
+    lifecycle_status: "ACTIVE" | "ARCHIVED";
+    resume_allowed: boolean;
+    as_of: string | null;
+    definition_schema_version: number;
+    definition_sha256: string;
+    source_bundle_sha256: string;
+    source_bundle_attachment: string;
+    resolved_profile_snapshot: ResolvedProfileSnapshot;
+    resolved_profile_sha256: string;
+    ksql_flow_version: string;
+    engine_version: string;
+    dialect: number;
+    created_at: string;
+    started_at: string | null;
+    finished_at: string | null;
+    updated_at: string;
+}
+export interface RunInvocation {
+    invocation_id: string;
+    run_id: string;
+    mode: "NEW" | "RESUME" | "RERUN_FROM";
+    requested_by: string;
+    host: string;
+    started_at: string;
+    finished_at: string | null;
+    status: RunInvocationStatus;
+    result_code: string;
+    selected_node_ids: string[];
+    preserved_node_ids: string[];
+    blocked_node_ids: string[];
+    reason: string;
+}
+export interface NodeState {
+    node_state_id: string;
+    node_state_key: CanonicalRecordKey<"S1">;
+    run_id: string;
+    node_id: string;
+    job_id: string;
+    status: NodeStateStatus;
+    latest_attempt_no: number;
+    active_attempt_id: string | null;
+    revision: number;
+    idempotent: boolean;
+    trigger_rule: "all_success";
+    blocked_by: string[];
+    status_reason: string | null;
+    started_at: string | null;
+    finished_at: string | null;
+    updated_at: string;
+}
+export interface NodeAttempt {
+    node_attempt_id: string;
+    attempt_key: CanonicalRecordKey<"A1">;
+    run_id: string;
+    node_id: string;
+    job_id: string;
+    invocation_id: string;
+    attempt_no: number;
+    status: NodeAttemptStatus;
+    result_code: string;
+    execution_started_at: string | null;
+    runner_execution_started_at: string | null;
+    execution_id: string | null;
+    finished_at: string | null;
+    duration_sec: number | null;
+    error_message: string | null;
+    read_count: number;
+    written_count: number;
+    last_successful_chunk_no: number | null;
+    last_written_key: string | null;
+    /** D-09 start-protocol evidence. */
+    state_revision_before: number | null;
+}
+export interface AttemptResolution {
+    event_type: "ATTEMPT_RESOLVED";
+    resolution_type: "OUTCOME_CONFIRMED" | "NODE_MANUAL_COMPLETION_CONFIRMED" | "NODE_COMPENSATION_COMPLETED";
+    attempt_id: string;
+    resolved_outcome: "SUCCESS" | "FAILED" | "CANCELLED";
+    reason: string;
+    evidence_ref: string;
+    service_principal: string;
+    requested_by: string;
+    approved_by: string;
+    stop_confirmed_by: string;
+    stop_evidence_ref: string;
+    resolved_at: string;
+}
+export interface ReconciliationOperationAudit {
+    event_id: string;
+    event_type: "RECONCILIATION_REPAIR";
+    repair_type: "TERMINAL_ATTEMPT_APPLIED" | "ATTEMPT_RESOLUTION_APPLIED" | "RUN_AGGREGATE_RECOMPUTED" | "INVOCATION_FINALIZED";
+    run_id: string;
+    target_type: "NODE_STATE" | "NETWORK_RUN" | "RUN_INVOCATION";
+    target_id: string;
+    before: Record<string, unknown>;
+    after: Record<string, unknown>;
+    basis: string;
+    occurred_at: string;
+}
+export type LockRecoveryOutcome = "RELEASED" | "NOT_FOUND" | "NOT_RUNNING" | "CONFLICT" | "UNCONFIRMED";
+export interface LockRecoveryResult {
+    kind: "LOCK_RECOVERY_RESULT";
+    formatVersion: 1;
+    jobKey: string;
+    recordId: string | null;
+    outcome: LockRecoveryOutcome;
+    before: {
+        batchId: string;
+        startedAt: string;
+        host: string;
+    } | null;
+    executedAt: string;
+    nextAction?: string;
+}
+export interface JobLockRecoveryOperationAudit {
+    event_id: string;
+    event_type: "JOB_LOCK_FORCE_UNLOCK_RECORDED";
+    run_id: string;
+    node_id: string;
+    job_id: string;
+    service_principal: string;
+    requested_by: string;
+    stop_confirmed_by: string;
+    reason: string;
+    evidence_ref: string;
+    recorded_at: string;
+    lock_recovery_result: LockRecoveryResult;
+}
+export interface NetworkLockForceReleaseOperationAudit {
+    event_id: string;
+    event_type: "NETWORK_LOCK_FORCE_RELEASED";
+    network_id: string;
+    profile: string;
+    lock_key: string;
+    record_id: string;
+    previous_owner_invocation_id: string;
+    previous_lease_token: string;
+    previous_heartbeat_at: string;
+    previous_lease_expires_at: string;
+    service_principal: string;
+    requested_by: string;
+    stop_confirmed_by: string;
+    stop_method: string;
+    stop_evidence_ref: string;
+    reason: string;
+    evidence_ref: string;
+    released_at: string;
+    post_release_revision: number;
+}
+export interface RunArchivedOperationAudit {
+    event_id: `archive_${string}`;
+    event_type: "RUN_ARCHIVED";
+    run_id: string;
+    result_code: "RUN_ARCHIVED";
+    requested_by: string;
+    reason: string;
+    archived_at: string;
+    previous_status: "FAILED" | "CANCELLED";
+    run_revision_before: number;
+    service_principal: string;
+}
+export type OperationAudit = ReconciliationOperationAudit | JobLockRecoveryOperationAudit | NetworkLockForceReleaseOperationAudit | RunArchivedOperationAudit;
+//# sourceMappingURL=persistence-model.d.ts.map

package/dist/domain/persistence-model.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=persistence-model.js.map

package/dist/domain/run-aggregate.d.ts

@@ -0,0 +1,3 @@
+import type { NetworkRunStatus, NodeStateStatus } from "./persistence-model.js";
+export declare function computeRunAggregateStatus(nodeStates: readonly NodeStateStatus[], startedAt: string | null): NetworkRunStatus;
+//# sourceMappingURL=run-aggregate.d.ts.map

package/dist/domain/run-aggregate.js

@@ -0,0 +1,15 @@
+export function computeRunAggregateStatus(nodeStates, startedAt) {
+    if (nodeStates.includes("UNKNOWN"))
+        return "UNKNOWN";
+    if (nodeStates.includes("RUNNING"))
+        return "RUNNING";
+    if (nodeStates.some((status) => status === "FAILED" || status === "BLOCKED"))
+        return "FAILED";
+    if (nodeStates.includes("CANCELLED"))
+        return "CANCELLED";
+    if (nodeStates.length > 0 &&
+        nodeStates.every((status) => status === "SUCCESS"))
+        return "SUCCESS";
+    return startedAt === null ? "CREATED" : "RUNNING";
+}
+//# sourceMappingURL=run-aggregate.js.map

package/dist/domain/validate-network-path.d.ts

@@ -0,0 +1,11 @@
+import type { NetworkDefinition, ValidationError } from "./network-definition.js";
+export interface NetworkPathValidationResult {
+    readonly definition?: NetworkDefinition;
+    readonly errors: readonly ValidationError[];
+}
+export declare function validateSqlFiles(networkPath: string, nodes: readonly {
+    readonly id: string;
+    readonly sql: string;
+}[]): ValidationError[];
+export declare function validateNetworkPath(networkPath: string): NetworkPathValidationResult;
+//# sourceMappingURL=validate-network-path.d.ts.map

package/dist/domain/validate-network-path.js

@@ -0,0 +1,33 @@
+import { accessSync, constants, statSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { loadNetworkDefinition } from "./load-network.js";
+export function validateSqlFiles(networkPath, nodes) {
+    const errors = [];
+    for (const [index, node] of nodes.entries()) {
+        const sqlPath = resolve(dirname(networkPath), node.sql);
+        try {
+            accessSync(sqlPath, constants.R_OK);
+            if (!statSync(sqlPath).isFile())
+                throw new Error("path is not a file");
+        }
+        catch (error) {
+            errors.push({
+                code: "SQL_FILE_UNREADABLE",
+                path: `$.nodes/${index}/sql`,
+                message: `SQL file for node '${node.id}' is not readable: ${sqlPath} (${String(error)})`,
+            });
+        }
+    }
+    return errors;
+}
+export function validateNetworkPath(networkPath) {
+    const loaded = loadNetworkDefinition(networkPath);
+    const errors = [...loaded.errors];
+    if (loaded.definition !== undefined) {
+        errors.push(...validateSqlFiles(networkPath, loaded.definition.nodes));
+    }
+    return loaded.definition === undefined
+        ? { errors }
+        : { definition: loaded.definition, errors };
+}
+//# sourceMappingURL=validate-network-path.js.map

package/dist/domain/validate-network.d.ts

@@ -0,0 +1,3 @@
+import type { ValidationResult } from "./network-definition.js";
+export declare function validateNetworkDefinition(input: unknown): ValidationResult;
+//# sourceMappingURL=validate-network.d.ts.map

package/dist/domain/validate-network.js

@@ -0,0 +1,180 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { Ajv2020 } from "ajv/dist/2020.js";
+import { stableTopologicalSort } from "../dag/topological-sort.js";
+import { validateScheduledPeriodPolicy } from "./business-key.js";
+const schemaPath = fileURLToPath(new URL("../../schemas/network-definition.schema.json", import.meta.url));
+const schema = JSON.parse(readFileSync(schemaPath, "utf8"));
+const ajv = new Ajv2020({ allErrors: true, strict: true });
+const validateSchema = ajv.compile(schema);
+function schemaError(error) {
+    const path = error.instancePath === "" ? "$" : `$${error.instancePath}`;
+    if (error.keyword === "additionalProperties") {
+        const property = String(error.params.additionalProperty);
+        return {
+            code: "UNKNOWN_PROPERTY",
+            path,
+            message: `unknown property '${property}'`,
+        };
+    }
+    return {
+        code: "SCHEMA_INVALID",
+        path,
+        message: error.message ?? "is invalid",
+    };
+}
+function semanticErrors(definition) {
+    const errors = [];
+    const nodeIds = new Set();
+    if (definition.business_key_policy.type === "scheduled_period") {
+        errors.push(...validateScheduledPeriodPolicy(definition.business_key_policy));
+    }
+    const { lease_duration_sec: lease, heartbeat_interval_sec: heartbeat } = definition.network_lock;
+    if (heartbeat >= lease) {
+        errors.push({
+            code: "NETWORK_LOCK_HEARTBEAT_NOT_LESS_THAN_LEASE",
+            path: "$.network_lock.heartbeat_interval_sec",
+            message: "must be less than lease_duration_sec",
+        });
+    }
+    if (heartbeat > lease / 3) {
+        errors.push({
+            code: "NETWORK_LOCK_HEARTBEAT_EXCEEDS_ONE_THIRD",
+            path: "$.network_lock.heartbeat_interval_sec",
+            message: "must be at most lease_duration_sec / 3",
+        });
+    }
+    definition.nodes.forEach((node, nodeIndex) => {
+        const nodePath = `$.nodes/${nodeIndex}`;
+        if (nodeIds.has(node.id)) {
+            errors.push({
+                code: "DUPLICATE_NODE_ID",
+                path: `${nodePath}/id`,
+                message: `duplicate node id '${node.id}'`,
+            });
+        }
+        nodeIds.add(node.id);
+        for (const [sourceName, pattern] of Object.entries(node.inputs ?? {})) {
+            const problem = inputPatternProblem(pattern);
+            if (problem !== null) {
+                errors.push({
+                    code: "INPUT_PATTERN_INVALID",
+                    path: `${nodePath}/inputs/${escapeJsonPointer(sourceName)}`,
+                    message: problem,
+                });
+            }
+        }
+        for (const [sinkName, pattern] of Object.entries(node.outputs ?? {})) {
+            const problem = outputPatternProblem(pattern);
+            if (problem !== null) {
+                errors.push({
+                    code: "OUTPUT_PATTERN_INVALID",
+                    path: `${nodePath}/outputs/${escapeJsonPointer(sinkName)}`,
+                    message: problem,
+                });
+            }
+        }
+        if (node.trigger_rule !== "all_success") {
+            errors.push({
+                code: "TRIGGER_RULE_UNSUPPORTED",
+                path: `${nodePath}/trigger_rule`,
+                message: `'${node.trigger_rule}' is reserved and not supported in Phase 1`,
+            });
+        }
+        const dependencies = new Set();
+        node.depends_on.forEach((dependency, dependencyIndex) => {
+            const dependencyPath = `${nodePath}/depends_on/${dependencyIndex}`;
+            if (dependency === node.id) {
+                errors.push({
+                    code: "SELF_DEPENDENCY",
+                    path: dependencyPath,
+                    message: "self-dependency is not allowed",
+                });
+            }
+            if (dependencies.has(dependency)) {
+                errors.push({
+                    code: "DUPLICATE_DEPENDENCY",
+                    path: dependencyPath,
+                    message: `duplicate dependency '${dependency}'`,
+                });
+            }
+            dependencies.add(dependency);
+        });
+    });
+    definition.nodes.forEach((node, nodeIndex) => {
+        node.depends_on.forEach((dependency, dependencyIndex) => {
+            if (!nodeIds.has(dependency)) {
+                errors.push({
+                    code: "UNKNOWN_DEPENDENCY",
+                    path: `$.nodes/${nodeIndex}/depends_on/${dependencyIndex}`,
+                    message: `unknown dependency '${dependency}'`,
+                });
+            }
+        });
+    });
+    if (errors.some((error) => error.code === "DUPLICATE_NODE_ID")) {
+        return errors;
+    }
+    if (errors.some((error) => error.code === "UNKNOWN_DEPENDENCY")) {
+        return errors;
+    }
+    const sorted = stableTopologicalSort(definition.nodes);
+    if (sorted.cyclicNodeIds.length > 0) {
+        errors.push({
+            code: "CYCLE_DETECTED",
+            path: "$.nodes",
+            message: `cycle detected; nodes not sortable: ${sorted.cyclicNodeIds.join(", ")}`,
+        });
+    }
+    return errors;
+}
+const INPUT_PLACEHOLDERS = new Set(["business_key", "profile"]);
+const OUTPUT_PLACEHOLDERS = new Set([
+    "business_key",
+    "profile",
+    "run_id",
+    "node_id",
+]);
+function inputPatternProblem(pattern) {
+    const literal = pattern.replace(/\{([^{}]*)\}/gu, (_match, name) => INPUT_PLACEHOLDERS.has(name) ? "placeholder" : `{${name}}`);
+    return relativePatternProblem(pattern, literal, "input");
+}
+function outputPatternProblem(pattern) {
+    const literal = pattern.replace(/\{([^{}]*)\}/gu, (_match, name) => OUTPUT_PLACEHOLDERS.has(name) ? "placeholder" : `{${name}}`);
+    return relativePatternProblem(pattern, literal, "output");
+}
+function relativePatternProblem(pattern, literal, context) {
+    if (pattern.length === 0)
+        return `${context} pattern must not be empty`;
+    if (pattern.includes("\0"))
+        return `${context} pattern must not contain NUL`;
+    if (pattern.startsWith("/") ||
+        pattern.startsWith("\\") ||
+        /^[A-Za-z]:/u.test(pattern))
+        return `${context} pattern must be a relative path without a drive or UNC prefix`;
+    if (literal.includes("{") || literal.includes("}")) {
+        return `${context} pattern contains an unknown or malformed placeholder`;
+    }
+    if (pattern
+        .split(/[\\/]/u)
+        .some((segment) => segment === "." || segment === "..")) {
+        return `${context} pattern must not contain '.' or '..' path segments`;
+    }
+    return null;
+}
+function escapeJsonPointer(value) {
+    return value.replace(/~/gu, "~0").replace(/\//gu, "~1");
+}
+export function validateNetworkDefinition(input) {
+    if (!validateSchema(input)) {
+        return { errors: (validateSchema.errors ?? []).map(schemaError) };
+    }
+    const raw = input;
+    const definition = {
+        ...raw,
+        max_active_runs: raw.max_active_runs ?? 1,
+    };
+    const errors = semanticErrors(definition);
+    return { definition, errors };
+}
+//# sourceMappingURL=validate-network.js.map

package/dist/executor/attempt-executor.d.ts

@@ -0,0 +1,43 @@
+import type { NodeAttempt, NodeState } from "../domain/persistence-model.js";
+import type { PersistenceRepository, Versioned } from "../persistence/repository.js";
+import { type JobLogReader } from "./job-log-reader.js";
+import { type ResultClassification, type ResultFileReader } from "./result-classifier.js";
+import type { RunRequest, RunSubprocess, SubprocessRunResult } from "./run-subprocess.js";
+export interface AttemptExecutorInput extends RunRequest {
+    readonly attempt: Versioned<NodeAttempt>;
+    readonly nodeState: Versioned<NodeState>;
+    readonly executionStartedAt: string;
+    /** D-29 gate evaluated after subprocess completion and before result writes. */
+    readonly authorizeResultPersistence?: () => Promise<boolean>;
+    /** Scheduler-scoped retry for transport failures after subprocess completion. */
+    readonly runControlPlaneOperation?: <T>(operation: () => Promise<T>) => Promise<T>;
+}
+export interface AttemptExecutorOptions {
+    readonly repository: PersistenceRepository;
+    readonly runner: Pick<RunSubprocess, "run">;
+    readonly jobLogReader: JobLogReader;
+    readonly resultFileReader?: ResultFileReader;
+    readonly now?: () => string;
+}
+export interface AttemptExecutionOutcome {
+    readonly classification: ResultClassification;
+    readonly process: SubprocessRunResult;
+    readonly attempt: Versioned<NodeAttempt>;
+    readonly nodeState: Versioned<NodeState>;
+    readonly invocationResultCode: string | null;
+}
+export declare class AttemptResultPersistenceDeferredError extends Error {
+    readonly classification: ResultClassification;
+    readonly process: SubprocessRunResult;
+    readonly code = "NETWORK_LEASE_INTERRUPTED";
+    constructor(classification: ResultClassification, process: SubprocessRunResult);
+}
+/** FN-10 entry point for one already-created RUNNING attempt and node state. */
+export declare class AttemptExecutor {
+    private readonly options;
+    private readonly now;
+    constructor(options: AttemptExecutorOptions);
+    execute(input: AttemptExecutorInput): Promise<AttemptExecutionOutcome>;
+    private readMarker;
+}
+//# sourceMappingURL=attempt-executor.d.ts.map