@rex0220/ksql-flownet 1.0.0

package/dist/orchestration/job-lock-audit.js

@@ -0,0 +1,123 @@
+import { randomUUID } from "node:crypto";
+const OUTCOMES = new Set([
+    "RELEASED",
+    "NOT_FOUND",
+    "NOT_RUNNING",
+    "CONFLICT",
+    "UNCONFIRMED",
+]);
+export class JobLockAuditError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.name = "JobLockAuditError";
+        this.code = code;
+    }
+}
+/** Records the kSQL-Flow result only. This module has no Job lock mutation dependency. */
+export async function recordJobUnlock(input) {
+    requireText("runId", input.runId);
+    requireText("nodeId", input.nodeId);
+    requireText("reason", input.reason);
+    requireText("evidenceRef", input.evidenceRef);
+    requireText("servicePrincipal", input.servicePrincipal);
+    requireText("requestedBy", input.requestedBy);
+    requireText("stopConfirmedBy", input.stopConfirmedBy);
+    const result = parseLockRecoveryResult(input.result);
+    const run = await input.repository.getRun(input.runId);
+    const states = (await input.repository.getNodeStates(input.runId)).filter(({ value }) => value.node_id === input.nodeId);
+    if (states.length !== 1) {
+        throw new JobLockAuditError("NODE_STATE_NOT_UNIQUE", `expected exactly one Node State for ${input.runId}/${input.nodeId}, found ${states.length}`);
+    }
+    const state = states[0].value;
+    const expectedJobKey = `${run.value.resolved_profile_snapshot.profile}:${state.job_id}`;
+    if (result.jobKey !== expectedJobKey) {
+        throw new JobLockAuditError("JOB_KEY_MISMATCH", `LOCK_RECOVERY_RESULT jobKey ${result.jobKey} does not match ${expectedJobKey}`);
+    }
+    const audit = {
+        event_id: `job_unlock_${(input.uuid ?? randomUUID)()}`,
+        event_type: "JOB_LOCK_FORCE_UNLOCK_RECORDED",
+        run_id: input.runId,
+        node_id: input.nodeId,
+        job_id: state.job_id,
+        service_principal: input.servicePrincipal.trim(),
+        requested_by: input.requestedBy.trim(),
+        stop_confirmed_by: input.stopConfirmedBy.trim(),
+        reason: input.reason.trim(),
+        evidence_ref: input.evidenceRef.trim(),
+        recorded_at: (input.now ?? (() => new Date()))().toISOString(),
+        lock_recovery_result: result,
+    };
+    await input.repository.appendOperationAudit(audit);
+    return audit;
+}
+export function parseLockRecoveryResult(value) {
+    const record = object(value, "LOCK_RECOVERY_RESULT must be a JSON object");
+    if (record.kind !== "LOCK_RECOVERY_RESULT" || record.formatVersion !== 1) {
+        throw new JobLockAuditError("INVALID_LOCK_RECOVERY_RESULT", "kind must be LOCK_RECOVERY_RESULT and formatVersion must be 1");
+    }
+    const jobKey = requiredString(record.jobKey, "jobKey");
+    const outcome = requiredString(record.outcome, "outcome");
+    if (!OUTCOMES.has(outcome)) {
+        throw new JobLockAuditError("INVALID_LOCK_RECOVERY_RESULT", `unsupported outcome ${outcome}`);
+    }
+    const recordId = nullableString(record.recordId, "recordId");
+    const executedAt = requiredTimestamp(record.executedAt, "executedAt");
+    let before = null;
+    if (record.before !== null) {
+        const beforeValue = object(record.before, "before must be an object or null");
+        before = {
+            batchId: requiredString(beforeValue.batchId, "before.batchId"),
+            startedAt: requiredTimestamp(beforeValue.startedAt, "before.startedAt"),
+            host: requiredString(beforeValue.host, "before.host"),
+        };
+    }
+    if (outcome === "NOT_FOUND" && (recordId !== null || before !== null)) {
+        throw new JobLockAuditError("INVALID_LOCK_RECOVERY_RESULT", "NOT_FOUND requires null recordId and before");
+    }
+    if ((outcome === "RELEASED" ||
+        outcome === "NOT_RUNNING" ||
+        outcome === "CONFLICT") &&
+        (recordId === null || before === null)) {
+        throw new JobLockAuditError("INVALID_LOCK_RECOVERY_RESULT", `${outcome} requires recordId and before`);
+    }
+    const nextAction = record.nextAction === undefined
+        ? undefined
+        : requiredString(record.nextAction, "nextAction");
+    return {
+        kind: "LOCK_RECOVERY_RESULT",
+        formatVersion: 1,
+        jobKey,
+        recordId,
+        outcome: outcome,
+        before,
+        executedAt,
+        ...(nextAction === undefined ? {} : { nextAction }),
+    };
+}
+function requireText(name, value) {
+    if (value.trim() === "")
+        throw new JobLockAuditError("AUDIT_INPUT_REQUIRED", `${name} is required`);
+}
+function object(value, message) {
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        throw new JobLockAuditError("INVALID_LOCK_RECOVERY_RESULT", message);
+    return value;
+}
+function requiredString(value, name) {
+    if (typeof value !== "string" || value.trim() === "")
+        throw new JobLockAuditError("INVALID_LOCK_RECOVERY_RESULT", `${name} must be a non-empty string`);
+    return value;
+}
+function nullableString(value, name) {
+    if (value === null)
+        return null;
+    return requiredString(value, name);
+}
+function requiredTimestamp(value, name) {
+    const timestamp = requiredString(value, name);
+    if (!Number.isFinite(Date.parse(timestamp)))
+        throw new JobLockAuditError("INVALID_LOCK_RECOVERY_RESULT", `${name} must be an ISO timestamp`);
+    return timestamp;
+}
+//# sourceMappingURL=job-lock-audit.js.map

package/dist/orchestration/orphan-attempt-adjudication.d.ts

@@ -0,0 +1,39 @@
+import type { NodeAttempt, NodeAttemptStatus } from "../domain/persistence-model.js";
+import type { JobLogAttemptResult, JobLogReader } from "../executor/job-log-reader.js";
+import type { PersistenceRepository } from "../persistence/repository.js";
+import type { Versioned } from "../persistence/repository.js";
+export interface OrphanAttemptAdjudicationInput {
+    readonly runId: string;
+    readonly invocationId: string;
+    readonly repository: PersistenceRepository;
+    readonly jobLogReader: Pick<JobLogReader, "findAttemptResult">;
+    readonly confirmWrite: () => Promise<boolean>;
+    readonly now?: () => string;
+    readonly attempts?: readonly Versioned<NodeAttempt>[];
+}
+export interface OrphanAttemptAdjudication {
+    readonly attemptId: string;
+    readonly nodeId: string;
+    readonly status: Exclude<NodeAttemptStatus, "RUNNING">;
+    readonly resultCode: string;
+}
+export interface AbandonedInvocationFinalization {
+    readonly invocationId: string;
+    readonly previousStatus: string;
+    readonly auditEventId: string;
+}
+interface TerminalDecision {
+    readonly status: "SUCCESS" | "FAILED" | "CANCELLED" | "UNKNOWN";
+    readonly resultCode: string;
+}
+/**
+ * Resolves RUNNING attempts owned by an older invocation before resume selects
+ * nodes. Job-log transport/schema failures deliberately escape (fail-closed).
+ */
+export declare function adjudicateOrphanRunningAttempts(input: OrphanAttemptAdjudicationInput): Promise<readonly OrphanAttemptAdjudication[]>;
+/** Finalizes invocations made unable to continue by the current lock owner. */
+export declare function finalizeAbandonedInvocations(input: Pick<OrphanAttemptAdjudicationInput, "runId" | "invocationId" | "repository" | "confirmWrite" | "now">): Promise<readonly AbandonedInvocationFinalization[]>;
+export declare function decideOrphanResult(jobLog: JobLogAttemptResult | null): TerminalDecision;
+export declare function orphanAdjudicationReason(adjudications: readonly OrphanAttemptAdjudication[]): string | undefined;
+export {};
+//# sourceMappingURL=orphan-attempt-adjudication.d.ts.map

package/dist/orchestration/orphan-attempt-adjudication.js

@@ -0,0 +1,167 @@
+import { randomUUID } from "node:crypto";
+import { NO_EXECUTION_RESULT } from "../executor/result-classifier.js";
+/**
+ * Resolves RUNNING attempts owned by an older invocation before resume selects
+ * nodes. Job-log transport/schema failures deliberately escape (fail-closed).
+ */
+export async function adjudicateOrphanRunningAttempts(input) {
+    const [states, attempts] = await Promise.all([
+        input.repository.getNodeStates(input.runId),
+        input.attempts ?? input.repository.getAttempts(input.runId),
+    ]);
+    const attemptsById = new Map(attempts.map((attempt) => [attempt.value.node_attempt_id, attempt]));
+    const adjudications = [];
+    const now = input.now ?? (() => new Date().toISOString());
+    const candidates = [];
+    for (const state of states) {
+        if (state.value.status !== "RUNNING" ||
+            state.value.active_attempt_id === null)
+            continue;
+        const attempt = attemptsById.get(state.value.active_attempt_id);
+        if (attempt === undefined)
+            throw new Error(`RUNNING Node State '${state.value.node_id}' has no active Attempt '${state.value.active_attempt_id}'`);
+        if (attempt.value.status !== "RUNNING" ||
+            attempt.value.invocation_id === input.invocationId)
+            continue;
+        candidates.push({ state, attempt });
+    }
+    // Complete every read before the first write. A transport/schema failure
+    // therefore stops this adjudication pass without guessing any outcome.
+    const evidence = await Promise.all(candidates.map(async ({ state, attempt }) => ({
+        state,
+        attempt,
+        jobLog: await input.jobLogReader.findAttemptResult(attempt.value.node_attempt_id),
+    })));
+    for (const { state, attempt, jobLog } of evidence) {
+        const decision = decideOrphanResult(jobLog);
+        const finishedAt = jobLog?.finishedAt ?? now();
+        if (!(await input.confirmWrite()))
+            throw leaseInterrupted();
+        await input.repository.finalizeAttempt(attempt.value.node_attempt_id, attempt.revision, {
+            status: decision.status,
+            result_code: decision.resultCode,
+            // D-11 preserves the Attempt's durable start marker. The JOB log is
+            // evidence for adjudication, not a surrogate write to this marker.
+            runner_execution_started_at: attempt.value.runner_execution_started_at,
+            execution_id: attempt.value.execution_id ?? jobLog?.executionId ?? null,
+            finished_at: finishedAt,
+            duration_sec: attempt.value.duration_sec,
+            error_message: decision.status === "UNKNOWN"
+                ? "orphan RUNNING attempt had no terminal execution result"
+                : attempt.value.error_message,
+            read_count: attempt.value.read_count,
+            written_count: attempt.value.written_count,
+            last_successful_chunk_no: attempt.value.last_successful_chunk_no,
+            last_written_key: attempt.value.last_written_key,
+        });
+        if (!(await input.confirmWrite()))
+            throw leaseInterrupted();
+        await input.repository.upsertNodeState({
+            expected_revision: state.revision,
+            value: terminalState(state.value, decision.status, decision.resultCode, finishedAt),
+        });
+        adjudications.push({
+            attemptId: attempt.value.node_attempt_id,
+            nodeId: state.value.node_id,
+            status: decision.status,
+            resultCode: decision.resultCode,
+        });
+    }
+    return adjudications;
+}
+/** Finalizes invocations made unable to continue by the current lock owner. */
+export async function finalizeAbandonedInvocations(input) {
+    const invocations = await input.repository.getInvocations(input.runId);
+    const now = input.now ?? (() => new Date().toISOString());
+    const results = [];
+    for (const invocation of invocations) {
+        const previousStatus = invocation.value.status;
+        if (invocation.value.invocation_id === input.invocationId ||
+            (previousStatus !== "RUNNING" && previousStatus !== "CREATED"))
+            continue;
+        if (!(await input.confirmWrite()))
+            throw leaseInterrupted();
+        const finishedAt = now();
+        const finalized = await input.repository.finalizeInvocation(invocation.value.invocation_id, invocation.revision, {
+            status: "CANCELLED",
+            result_code: "NETWORK_LEASE_INTERRUPTED",
+            finished_at: finishedAt,
+            reason: `${invocation.value.reason}; finalized as an abandoned invocation during reconciliation`,
+        });
+        if (!(await input.confirmWrite()))
+            throw leaseInterrupted();
+        const auditEventId = randomUUID();
+        const audit = {
+            event_id: auditEventId,
+            event_type: "RECONCILIATION_REPAIR",
+            repair_type: "INVOCATION_FINALIZED",
+            run_id: input.runId,
+            target_type: "RUN_INVOCATION",
+            target_id: invocation.value.invocation_id,
+            before: {
+                status: previousStatus,
+                result_code: invocation.value.result_code,
+                revision: invocation.revision,
+            },
+            after: {
+                status: finalized.value.status,
+                result_code: finalized.value.result_code,
+                revision: finalized.revision,
+            },
+            basis: "the current Network lock owner makes an older unterminated invocation unable to continue",
+            occurred_at: finishedAt,
+        };
+        await input.repository.appendOperationAudit(audit);
+        results.push({
+            invocationId: invocation.value.invocation_id,
+            previousStatus,
+            auditEventId,
+        });
+    }
+    return results;
+}
+export function decideOrphanResult(jobLog) {
+    if (jobLog === null)
+        return unknownDecision();
+    switch (jobLog.status.toUpperCase()) {
+        case "SUCCESS":
+            return { status: "SUCCESS", resultCode: "OK" };
+        case "NO_DATA":
+            return { status: "SUCCESS", resultCode: "NO_DATA" };
+        case "FAILED":
+            return { status: "FAILED", resultCode: "FAILED" };
+        case "ABORTED":
+            return { status: "FAILED", resultCode: "ASSERT_FAILED" };
+        case "TIMEOUT":
+            return { status: "FAILED", resultCode: "EXECUTION_TIMEOUT" };
+        case "CANCELLED":
+            return { status: "CANCELLED", resultCode: "CANCELLED" };
+        case "RUNNING":
+        default:
+            return unknownDecision();
+    }
+}
+export function orphanAdjudicationReason(adjudications) {
+    if (adjudications.length === 0)
+        return undefined;
+    return `orphan attempts adjudicated: ${adjudications
+        .map((item) => `${item.attemptId}=${item.status}(${item.resultCode})`)
+        .join(", ")}`;
+}
+function unknownDecision() {
+    return { status: "UNKNOWN", resultCode: NO_EXECUTION_RESULT };
+}
+function terminalState(state, status, resultCode, finishedAt) {
+    return {
+        ...state,
+        status,
+        active_attempt_id: null,
+        status_reason: resultCode,
+        finished_at: finishedAt,
+        updated_at: finishedAt,
+    };
+}
+function leaseInterrupted() {
+    return Object.assign(new Error("network lease was interrupted during orphan adjudication"), { code: "NETWORK_LEASE_INTERRUPTED" });
+}
+//# sourceMappingURL=orphan-attempt-adjudication.js.map

package/dist/orchestration/reconciliation.d.ts

@@ -0,0 +1,41 @@
+import type { ReconciliationOperationAudit } from "../domain/persistence-model.js";
+import type { PersistenceRepository, StatusReadRepository } from "../persistence/repository.js";
+export interface ReconciliationRepair {
+    type: ReconciliationOperationAudit["repair_type"];
+    targetType: ReconciliationOperationAudit["target_type"];
+    targetId: string;
+    before: Record<string, unknown>;
+    after: Record<string, unknown>;
+    basis: string;
+    auditEventId: string;
+}
+export interface ReconciliationInconsistency {
+    code: "ACTIVE_ATTEMPT_ID_MISSING" | "ACTIVE_ATTEMPT_NOT_FOUND" | "ATTEMPT_NODE_STATE_MISSING" | "STATE_TERMINAL_ATTEMPT_RUNNING" | "STATE_ATTEMPT_STATUS_MISMATCH" | "MULTIPLE_RUNNING_ATTEMPTS" | "REVISION_SERIES_INVERTED" | "REVISION_SERIES_UNDETERMINED" | "RESOLUTION_ATTEMPT_NOT_UNKNOWN" | "RESOLUTION_NOT_CURRENT_ATTEMPT" | "RESOLUTION_STATE_NOT_UNKNOWN" | "CONFLICTING_ATTEMPT_RESOLUTIONS";
+    nodeId: string;
+    detail: string;
+    attemptIds?: string[];
+}
+export interface ReconciliationResult {
+    repaired: ReconciliationRepair[];
+    inconsistencies: ReconciliationInconsistency[];
+    aggregateUpdated: boolean;
+}
+export interface ReconciliationDetection {
+    readonly inconsistencies: readonly ReconciliationDetectionItem[];
+}
+export interface ReconciliationDetectionItem {
+    readonly code: ReconciliationInconsistency["code"] | "TERMINAL_ATTEMPT_PENDING_APPLY" | "ATTEMPT_RESOLUTION_PENDING_APPLY" | "RUN_AGGREGATE_MISMATCH";
+    readonly nodeId?: string;
+    readonly detail: string;
+    readonly attemptIds?: readonly string[];
+}
+export declare class ReconciliationRequiredError extends Error {
+    readonly code = "RECONCILIATION_REQUIRED";
+    readonly result: ReconciliationResult;
+    constructor(result: ReconciliationResult);
+}
+/** Runs the D-09 inspections without exposing or invoking any repair method. */
+export declare function detectReconciliation(repository: StatusReadRepository, runId: string): Promise<ReconciliationDetection>;
+/** D-09 pre-DAG gate. Throws RECONCILIATION_REQUIRED after retaining safe repairs. */
+export declare function reconcileRun(repository: PersistenceRepository, runId: string): Promise<ReconciliationResult>;
+//# sourceMappingURL=reconciliation.d.ts.map