@rex0220/ksql-flownet 1.0.0

package/dist/persistence/network-lock-recovery.js

@@ -0,0 +1,392 @@
+import { randomUUID } from "node:crypto";
+import { hostname } from "node:os";
+import { networkLockKey } from "../domain/canonical-lock-key.js";
+import { KintoneApiError, KintoneClient, KintoneTransportError, } from "./kintone/client.js";
+import { KINTONE_DATETIME_TRUNCATION_MS } from "./kintone/design-notes.js";
+import { releaseTombstoneRecordKey } from "./network-lock.js";
+import { ownerInstanceIdFromStatusReason } from "./network-lock-reader.js";
+export const manualStopConfirmation = {
+    async confirm(input) {
+        const confirmed = input.stopMethod === "manual" &&
+            input.stopConfirmedBy.trim() !== "" &&
+            input.stopEvidenceRef.trim() !== "";
+        return {
+            confirmed,
+            method: "manual",
+            detail: confirmed
+                ? "manual stop evidence was provided"
+                : "manual stop confirmation fields are invalid",
+        };
+    },
+};
+const localPidPattern = /^local-pid:\/\/([^/]+)\/([^/]+)$/u;
+const cloudRunExecutionPattern = /^projects\/[^/]+\/locations\/[^/]+\/jobs\/[^/]+\/executions\/[^/]+$/u;
+const terminalCloudRunStates = new Set(["SUCCEEDED", "FAILED", "CANCELLED"]);
+const nonTerminalCloudRunStates = new Set(["RUNNING", "PENDING"]);
+function detailTime(now) {
+    return now().toISOString();
+}
+export function createLocalPidStopConfirmation(dependencies = {}) {
+    const host = dependencies.host ?? process.env.KSQL_FLOWNET_HOST ?? hostname();
+    const processKill = dependencies.processKill ?? ((pid, signal) => process.kill(pid, signal));
+    const now = dependencies.now ?? (() => new Date());
+    return {
+        async confirm(input) {
+            const match = localPidPattern.exec(input.ownerInstanceId);
+            if (match === null) {
+                return {
+                    confirmed: false,
+                    method: "local_pid",
+                    detail: "owner_instance_id is not a valid local-pid resource",
+                };
+            }
+            const ownerHost = match[1];
+            const pidText = match[2];
+            if (ownerHost !== host) {
+                return {
+                    confirmed: false,
+                    method: "local_pid",
+                    detail: `owner host '${ownerHost}' differs from current host '${host}'; use manual recovery because lease expiry is not stop confirmation`,
+                };
+            }
+            const pid = Number(pidText);
+            if (!/^[1-9]\d*$/u.test(pidText) || !Number.isSafeInteger(pid)) {
+                return {
+                    confirmed: false,
+                    method: "local_pid",
+                    detail: "owner_instance_id PID is not a positive safe integer",
+                };
+            }
+            try {
+                processKill(pid, 0);
+                return {
+                    confirmed: false,
+                    method: "local_pid",
+                    detail: `PID ${pid} exists on host '${host}'`,
+                };
+            }
+            catch (error) {
+                const code = errorCodeOf(error);
+                if (code === "ESRCH") {
+                    return {
+                        confirmed: true,
+                        method: "local_pid",
+                        detail: `PID ${pid} was absent at ${detailTime(now)}; PID reuse remains a residual risk`,
+                    };
+                }
+                return {
+                    confirmed: false,
+                    method: "local_pid",
+                    detail: code === "EPERM"
+                        ? `PID ${pid} may exist but permission was denied`
+                        : `PID ${pid} existence check failed${code === undefined ? "" : ` (${code})`}`,
+                };
+            }
+        },
+    };
+}
+export function createCloudRunJobExecutionStopConfirmation(dependencies = {}) {
+    const fetchImplementation = dependencies.fetch ?? globalThis.fetch;
+    const accessToken = dependencies.gcpAccessToken ?? process.env.KSQL_FLOWNET_GCP_ACCESS_TOKEN;
+    return {
+        async confirm(input) {
+            const resourceName = input.ownerInstanceId;
+            if (!cloudRunExecutionPattern.test(resourceName)) {
+                return cloudRunResult(false, "owner_instance_id is not a valid Cloud Run Job Execution resource");
+            }
+            if (accessToken === undefined || accessToken.trim() === "") {
+                return cloudRunResult(false, "KSQL_FLOWNET_GCP_ACCESS_TOKEN is not configured");
+            }
+            let response;
+            try {
+                response = await fetchImplementation(`https://run.googleapis.com/v2/${resourceName}`, {
+                    method: "GET",
+                    headers: { Authorization: `Bearer ${accessToken}` },
+                });
+            }
+            catch {
+                return cloudRunResult(false, "Cloud Run API request failed");
+            }
+            if (!response.ok) {
+                return cloudRunResult(false, `Cloud Run API returned HTTP ${response.status}`);
+            }
+            let body;
+            try {
+                body = await response.json();
+            }
+            catch {
+                return cloudRunResult(false, `Cloud Run API returned an unknown response shape (HTTP ${response.status})`);
+            }
+            if (!isRecord(body) || body.name !== resourceName) {
+                return cloudRunResult(false, `Cloud Run API returned an unknown response shape (HTTP ${response.status})`);
+            }
+            const state = executionState(body);
+            const stateDetail = state ?? "UNSPECIFIED";
+            const completionTime = body.completionTime;
+            if (completionTime === undefined ||
+                completionTime === null ||
+                completionTime === "") {
+                return cloudRunResult(false, `Cloud Run Execution is not terminal (state=${stateDetail}, completionTime=unset, HTTP ${response.status})`);
+            }
+            if (typeof completionTime !== "string" ||
+                !Number.isFinite(Date.parse(completionTime))) {
+                return cloudRunResult(false, `Cloud Run API returned an unknown completionTime shape (state=${stateDetail}, HTTP ${response.status})`);
+            }
+            if (state !== null && !terminalCloudRunStates.has(state)) {
+                const verdict = nonTerminalCloudRunStates.has(state)
+                    ? "not terminal"
+                    : "an unknown state";
+                return cloudRunResult(false, `Cloud Run Execution has ${verdict} (state=${state}, completionTime=${completionTime}, HTTP ${response.status})`);
+            }
+            const countDetail = executionCountDetail(body);
+            return cloudRunResult(true, `Cloud Run Execution is terminal (state=${stateDetail}, completionTime=${completionTime}, HTTP ${response.status}${countDetail})`);
+        },
+    };
+}
+function cloudRunResult(confirmed, detail) {
+    return { confirmed, method: "cloud_run_job_execution", detail };
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function errorCodeOf(error) {
+    return isRecord(error) && typeof error.code === "string"
+        ? error.code
+        : undefined;
+}
+function executionState(body) {
+    if (typeof body.state === "string")
+        return body.state;
+    if (typeof body.status === "string")
+        return body.status;
+    return isRecord(body.terminalCondition) &&
+        typeof body.terminalCondition.state === "string"
+        ? body.terminalCondition.state
+        : null;
+}
+function executionCountDetail(body) {
+    const names = [
+        "taskCount",
+        "succeededCount",
+        "failedCount",
+        "cancelledCount",
+    ];
+    const counts = names.map((name) => body[name]);
+    if (!counts.every((value) => Number.isSafeInteger(value) && Number(value) >= 0))
+        return "";
+    const [taskCount, succeededCount, failedCount, cancelledCount] = counts.map(Number);
+    const completed = succeededCount + failedCount + cancelledCount;
+    return `, taskCounts=${completed === taskCount ? "consistent" : "inconsistent"} (${completed}/${taskCount})`;
+}
+export function defaultStopConfirmations(dependencies = {}) {
+    return new Map([
+        ["manual", manualStopConfirmation],
+        ["local_pid", createLocalPidStopConfirmation(dependencies)],
+        [
+            "cloud_run_job_execution",
+            createCloudRunJobExecutionStopConfirmation(dependencies),
+        ],
+    ]);
+}
+export class NetworkLockRecoveryError extends Error {
+    code;
+    causeDetail;
+    constructor(code, message, causeDetail) {
+        super(message);
+        this.name = "NetworkLockRecoveryError";
+        this.code = code;
+        this.causeDetail = causeDetail;
+    }
+}
+const field = (value) => ({ value });
+const text = (record, code) => String(record[code]?.value ?? "");
+const revisionOf = (record) => Number(record.$revision?.value);
+function quote(value) {
+    return `"${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
+}
+function snapshot(record) {
+    const result = {
+        record,
+        recordId: text(record, "$id"),
+        revision: revisionOf(record),
+        businessRevision: Number(record.revision?.value),
+        ownerInvocationId: text(record, "owner_invocation_id"),
+        ownerInstanceId: ownerInstanceIdFromStatusReason(text(record, "status_reason")),
+        leaseToken: text(record, "lease_token"),
+        heartbeatAt: text(record, "heartbeat_at"),
+        leaseExpiresAt: text(record, "lease_expires_at"),
+    };
+    if (result.recordId === "" ||
+        !Number.isSafeInteger(result.revision) ||
+        !Number.isSafeInteger(result.businessRevision) ||
+        result.leaseToken === "" ||
+        !Number.isFinite(Date.parse(result.leaseExpiresAt))) {
+        throw new NetworkLockRecoveryError("LOCK_READ_FAILED", "network lock record is malformed");
+    }
+    return result;
+}
+async function getRecords(client, recordKey) {
+    return client.getRecords(`record_key in (${quote(recordKey)})`);
+}
+async function getInitialLock(client, recordKey) {
+    let records;
+    try {
+        records = await getRecords(client, recordKey);
+    }
+    catch (error) {
+        throw new NetworkLockRecoveryError("LOCK_READ_FAILED", "network lock read failed", error);
+    }
+    if (records.length === 0) {
+        throw new NetworkLockRecoveryError("LOCK_NOT_FOUND", "network lock was not found");
+    }
+    if (records.length !== 1) {
+        throw new NetworkLockRecoveryError("LOCK_READ_FAILED", `network lock read returned ${records.length} records`);
+    }
+    return snapshot(records[0]);
+}
+function requireNonEmpty(value, name) {
+    if (value.trim() === "") {
+        throw new NetworkLockRecoveryError("INVALID_ARGUMENT", `${name} must not be empty`);
+    }
+}
+function releaseConfirmed(record, initial, tombstone) {
+    // finished_at is DATETIME with minute precision, so do not use it for round-trip comparison.
+    return (text(record, "$id") === initial.recordId &&
+        text(record, "record_key") === tombstone &&
+        text(record, "lock_key") === "" &&
+        text(record, "owner_invocation_id") === "" &&
+        text(record, "lease_token") === "" &&
+        text(record, "status") === "CANCELLED" &&
+        text(record, "status_reason") === "NETWORK_LOCK_FORCE_RELEASED" &&
+        revisionOf(record) === initial.revision + 1);
+}
+export async function forceUnlockNetwork(input, dependencies) {
+    for (const [name, value] of Object.entries(input))
+        requireNonEmpty(value, name);
+    const now = dependencies.now ?? (() => new Date());
+    const uuid = dependencies.uuid ?? randomUUID;
+    const confirmations = dependencies.stopConfirmations ?? defaultStopConfirmations();
+    const lockKey = networkLockKey(input.profile, input.networkId);
+    const recordKey = `LOCK:${lockKey}`;
+    const client = new KintoneClient({
+        baseUrl: dependencies.baseUrl,
+        appId: dependencies.stateAppId,
+        apiToken: dependencies.stateApiToken,
+        ...(dependencies.fetch === undefined ? {} : { fetch: dependencies.fetch }),
+    });
+    const initial = await getInitialLock(client, recordKey);
+    if (initial.ownerInvocationId !== input.expectedOwnerInvocationId) {
+        throw new NetworkLockRecoveryError("OWNER_MISMATCH", "network lock owner does not match the expected invocation");
+    }
+    // The persisted lease expiry may be truncated by up to 59 seconds, so add the upper bound before allowing recovery.
+    if (Date.parse(initial.leaseExpiresAt) + KINTONE_DATETIME_TRUNCATION_MS >
+        now().getTime()) {
+        throw new NetworkLockRecoveryError("LEASE_STILL_ACTIVE", "network lock lease has not expired");
+    }
+    const confirmation = confirmations.get(input.stopMethod);
+    if (confirmation === undefined) {
+        throw new NetworkLockRecoveryError("STOP_NOT_CONFIRMED", `unknown stop confirmation method '${input.stopMethod}'`);
+    }
+    let confirmed;
+    try {
+        confirmed = await confirmation.confirm({
+            profile: input.profile,
+            networkId: input.networkId,
+            expectedOwnerInvocationId: input.expectedOwnerInvocationId,
+            ownerInstanceId: initial.ownerInstanceId,
+            stopConfirmedBy: input.stopConfirmedBy,
+            stopMethod: input.stopMethod,
+            stopEvidenceRef: input.stopEvidenceRef,
+        });
+    }
+    catch (error) {
+        throw new NetworkLockRecoveryError("STOP_NOT_CONFIRMED", "stop confirmation failed", error);
+    }
+    if (!confirmed.confirmed) {
+        throw new NetworkLockRecoveryError("STOP_NOT_CONFIRMED", confirmed.detail || "stop was not confirmed");
+    }
+    let checked;
+    try {
+        checked = await getInitialLock(client, recordKey);
+    }
+    catch (error) {
+        throw new NetworkLockRecoveryError("HEARTBEAT_ADVANCED", "network lock changed while stop confirmation was in progress", error);
+    }
+    if (checked.revision !== initial.revision ||
+        checked.heartbeatAt !== initial.heartbeatAt ||
+        checked.ownerInvocationId !== initial.ownerInvocationId ||
+        checked.leaseToken !== initial.leaseToken) {
+        throw new NetworkLockRecoveryError("HEARTBEAT_ADVANCED", "network lock changed while stop confirmation was in progress");
+    }
+    const releasedAt = now().toISOString();
+    const tombstone = releaseTombstoneRecordKey(`${lockKey}:${initial.leaseToken}`);
+    const terminalRecord = {
+        record_key: field(tombstone),
+        lock_key: field(""),
+        owner_invocation_id: field(""),
+        lease_token: field(""),
+        status: field("CANCELLED"),
+        status_reason: field("NETWORK_LOCK_FORCE_RELEASED"),
+        finished_at: field(releasedAt),
+        revision: field(initial.businessRevision + 1),
+    };
+    let postReleaseRevision;
+    try {
+        postReleaseRevision = await client.putRecordById(initial.recordId, initial.revision, terminalRecord);
+    }
+    catch (error) {
+        if (error instanceof KintoneApiError &&
+            (error.status === 409 || error.apiCode === "GAIA_CO02")) {
+            throw new NetworkLockRecoveryError("REVISION_CONFLICT", "network lock release revision conflicted", error);
+        }
+        if (!(error instanceof KintoneTransportError)) {
+            throw new NetworkLockRecoveryError("RELEASE_UNCONFIRMED", "network lock release failed", error);
+        }
+        let records;
+        try {
+            records = await getRecords(client, tombstone);
+        }
+        catch (adjudicationError) {
+            throw new NetworkLockRecoveryError("RELEASE_UNCONFIRMED", "network lock release outcome could not be confirmed", { release: error, adjudication: adjudicationError });
+        }
+        if (records.length !== 1 ||
+            !releaseConfirmed(records[0], initial, tombstone)) {
+            throw new NetworkLockRecoveryError("RELEASE_UNCONFIRMED", "network lock release outcome could not be confirmed", { release: error, adjudication: "released tombstone did not match" });
+        }
+        postReleaseRevision = revisionOf(records[0]);
+    }
+    const audit = {
+        event_id: `net_unlock_${uuid()}`,
+        event_type: "NETWORK_LOCK_FORCE_RELEASED",
+        network_id: input.networkId,
+        profile: input.profile,
+        lock_key: lockKey,
+        record_id: initial.recordId,
+        previous_owner_invocation_id: initial.ownerInvocationId,
+        previous_lease_token: initial.leaseToken,
+        previous_heartbeat_at: initial.heartbeatAt,
+        previous_lease_expires_at: initial.leaseExpiresAt,
+        service_principal: input.servicePrincipal,
+        requested_by: input.requestedBy,
+        stop_confirmed_by: input.stopConfirmedBy,
+        stop_method: confirmed.method,
+        stop_evidence_ref: input.stopEvidenceRef,
+        reason: input.reason,
+        evidence_ref: input.evidenceRef,
+        released_at: releasedAt,
+        post_release_revision: postReleaseRevision,
+    };
+    try {
+        await dependencies.repository.appendOperationAudit(audit);
+    }
+    catch (error) {
+        throw new NetworkLockRecoveryError("AUDIT_FAILED", "network lock was released but its audit could not be confirmed", error);
+    }
+    return {
+        lockKey,
+        previousOwnerInvocationId: initial.ownerInvocationId,
+        postReleaseRevision,
+        audit,
+    };
+}
+//# sourceMappingURL=network-lock-recovery.js.map

package/dist/persistence/network-lock.d.ts

@@ -0,0 +1,97 @@
+import { type CanonicalLockKey } from "../domain/canonical-lock-key.js";
+export type NetworkLockErrorCode = "LOCK_CONFLICT" | "LOCK_UNAVAILABLE" | "LEASE_TOKEN_MISMATCH" | "LEASE_REVISION_MISMATCH" | "LEASE_REVISION_CONFLICT";
+export declare class NetworkLockError extends Error {
+    readonly code: NetworkLockErrorCode;
+    readonly causeDetail: unknown;
+    constructor(code: NetworkLockErrorCode, message: string, causeDetail?: unknown);
+}
+export interface NetworkLockReference {
+    readonly lockKey: CanonicalLockKey<"N1">;
+    readonly recordId: string;
+    readonly originalRecordKey: string;
+    readonly leaseToken: string;
+    readonly ownerInvocationId: string;
+    readonly leaseDurationSec: number;
+    revision: number;
+    businessRevision: number;
+    heartbeatAt: string;
+    leaseExpiresAt: string;
+}
+export interface NetworkLockManagerConfig {
+    baseUrl: string;
+    appId: number;
+    apiToken: string;
+    profile: string;
+    networkId: string;
+    ownerInvocationId: string;
+    ownerInstanceId: string;
+    leaseDurationSec: number;
+    fetch?: typeof fetch;
+    now?: () => Date;
+    uuid?: () => string;
+}
+export declare function releaseTombstoneRecordKey(uniqueValue: string): string;
+export declare class NetworkLockManager {
+    private readonly client;
+    private readonly config;
+    private readonly now;
+    private readonly uuid;
+    readonly lockKey: CanonicalLockKey<"N1">;
+    readonly recordKey: string;
+    constructor(config: NetworkLockManagerConfig);
+    acquire(): Promise<NetworkLockReference>;
+    heartbeat(reference: NetworkLockReference): Promise<NetworkLockReference>;
+    release(reference: NetworkLockReference, status?: string, resultCode?: string): Promise<{
+        released: true;
+        recordKey: string;
+    }>;
+    private getUniqueAfterAcquireFailure;
+    private getUnique;
+    private assertFence;
+}
+export type LeaseMonitorState = "HELD" | "LEASE_UNCERTAIN" | "FINAL_WRITE_CONFIRMED";
+export interface LeaseMonitorEvent {
+    previous: LeaseMonitorState;
+    current: LeaseMonitorState;
+    reason: "FAILURE_THRESHOLD" | "LEASE_SAFETY_THRESHOLD" | "FENCE_LOST" | "FINAL_WRITE_CONFIRMED";
+    at: string;
+}
+export interface LeaseMonitorConfig {
+    leaseDurationSec: number;
+    heartbeatIntervalSec: number;
+    consecutiveFailureThreshold?: number;
+    remainingLeaseSafetySec?: number;
+    now?: () => Date;
+    schedule?: (callback: () => void, intervalMs: number) => unknown;
+    cancelSchedule?: (handle: unknown) => void;
+}
+export declare class LeaseMonitor {
+    private readonly manager;
+    private readonly reference;
+    private readonly config;
+    private state;
+    private failures;
+    private timer;
+    private tickInFlight;
+    private readonly listeners;
+    private readonly now;
+    private readonly failureThreshold;
+    private readonly safetyMs;
+    private readonly schedule;
+    private readonly cancelSchedule;
+    constructor(manager: NetworkLockManager, reference: NetworkLockReference, config: LeaseMonitorConfig);
+    get currentState(): LeaseMonitorState;
+    canStartNewNode(): boolean;
+    canPersistResults(): boolean;
+    requiresNetworkLeaseInterruptedFinalization(): boolean;
+    markControlPlaneUnreachable(): void;
+    subscribe(listener: (event: LeaseMonitorEvent) => void): () => void;
+    tick(): Promise<boolean>;
+    private performTick;
+    confirmLeaseForFinalWrite(): Promise<boolean>;
+    start(): void;
+    stop(): void;
+    private remainingLeaseMs;
+    private transition;
+}
+//# sourceMappingURL=network-lock.d.ts.map