@botiverse/k-carrier 0.1.8 → 0.2.0

package/core/src/lifecycle/hostAdapter.ts

@@ -10,22 +10,31 @@
  *    equivalent — including when resume() happens on the ROLLED-BACK slot.
  *  - healthProbe(): evidence must be bound to one live process (same-PID /
  *    startId), never assembled from files or caches. A probe that cannot
- *    prove which process answered is not a probe.
- *  - start(): may return BEFORE the process exists. Some hosts cannot start
- *    themselves at all -- a service that is replaced by exiting, letting its
- *    supervisor respawn it from the new bytes, is started by that supervisor,
- *    asynchronously. So start() means "the successor has been asked for",
- *    never "the successor is running": only healthProbe() can say that.
+ *    prove which process answered is not a probe. K probes once before
+ *    handover and refuses readback evidence that repeats that startId.
+ *  - start(): may return before readiness. The external controller asks
+ *    the service manager to start the selected artifact; healthProbe()
+ *    then proves that the requested incarnation is actually running.
  *
- * A consequence worth stating, because it decides who finishes an upgrade:
- * on such hosts the process driving the transaction DIES on the success path.
- * The successor finds a journal that stops mid-handover -- indistinguishable
- * from a crash -- and K resolves it by EVIDENCE (a live process reporting the
- * experiment version from a different incarnation), never by a flag saying the
- * restart was planned. A crash could set that flag just as easily.
+ * K runs outside the resident process and service unit, so stopping the
+ * application does not kill the transaction owner. A runner crash before
+ * durable promote intent restores stable; after that intent, recovery
+ * replays the commit. A live candidate alone never authorizes promotion.
  */
 export interface HostAdapter {
-  /** Park all hosted workloads durably. Idempotent. */
+  /**
+   * Drain or fence effects left by an earlier worker/controller before replay.
+   * Called under K's transaction lock. Throw if isolation cannot be proved.
+   * Required for adapters whose effects can outlive their worker; purely
+   * in-process adapters without surviving effects may omit it.
+   */
+  fence?(): Promise<void>;
+
+  /**
+   * Park all hosted workloads durably. Idempotent. A stateless service
+   * acknowledges and returns; the obligation covers only workloads the
+   * product promises to preserve across an upgrade.
+   */
   quiesce(): Promise<void>;
 
   /** Stop the resident service process tree for the given slot. */
@@ -40,7 +49,10 @@ export interface HostAdapter {
    */
   healthProbe(): Promise<ProcessEvidence>;
 
-  /** Resume workloads parked by quiesce(). Must also work after rollback. */
+  /**
+   * Resume workloads parked by quiesce(). Must also work after rollback,
+   * i.e. on the previous version. Trivial when quiesce() is trivial.
+   */
   resume(): Promise<void>;
 }
 
@@ -60,8 +72,8 @@ export interface HostAdapter {
  * reconstructed after a failure, it was never disturbed.
  *
  * ⚠️ `stable` here is a POSITION, not the name of a release channel. If your
- * product also has a channel called "stable" (ours does), they are unrelated:
- * a channel says which stream you follow, a slot says which copy on disk.
+ * product also has a channel called "stable", they are unrelated: a channel
+ * says which stream you follow, a slot says which copy on disk.
  */
 export type Slot = "stable" | "experiment";
 

package/core/src/operation.ts

@@ -4,10 +4,11 @@
  * Hosts may project this record into their own UI or transport, but they do
  * not maintain a second upgrade state machine. The operation receipt is the
  * single durable answer to: what is running, which version was requested,
- * what stable version can be restored, and whether the terminal receipt has
- * already been acknowledged by the host transport.
+ * what stable version can be restored, and the outcome. Transport delivery
+ * tracking belongs to the caller and cannot block another transaction.
  */
 import { promises as fs } from "node:fs";
+import { createHash } from "node:crypto";
 import * as path from "node:path";
 import { platformOpsFor } from "./platform/index.ts";
 import type { ProvenanceIdentity } from "./upgrader.ts";
@@ -75,8 +76,6 @@ export interface OperationRecord {
   reason: string | null;
   provenance: ProvenanceIdentity | null;
   metadata: Record<string, string>;
-  /** Host transport receipt, not a second transaction outcome. */
-  acknowledgedAtMs: number | null;
 }
 
 export type OperationRead =
@@ -97,7 +96,7 @@ function validIdentity(value: unknown): value is ProvenanceIdentity {
   );
 }
 
-function parseOperation(text: string): OperationRecord {
+export function parseOperation(text: string): OperationRecord {
   const parsed = JSON.parse(text) as Partial<OperationRecord>;
   if (
     parsed.formatVersion !== OPERATION_FORMAT_VERSION
@@ -122,7 +121,6 @@ function parseOperation(text: string): OperationRecord {
     || parsed.metadata === null
     || Array.isArray(parsed.metadata)
     || Object.values(parsed.metadata).some((value) => typeof value !== "string")
-    || !(parsed.acknowledgedAtMs === null || typeof parsed.acknowledgedAtMs === "number")
   ) {
     throw new Error("operation record has an invalid shape");
   }
@@ -160,23 +158,47 @@ export async function loadOperation(stateDir: string): Promise<OperationRead> {
   }
 }
 
-export async function acknowledgeOperation(
-  stateDir: string,
-  operationId: string,
-  acknowledgedAtMs: number,
-): Promise<"acknowledged" | "not-terminal" | "not-found" | "changed"> {
-  const current = await loadOperation(stateDir);
-  if (current.kind === "genesis") return "not-found";
-  if (current.kind === "unreadable") throw new Error(current.reason);
-  if (current.operation.id !== operationId) return "changed";
-  if (current.operation.outcome === null) return "not-terminal";
-  // Exact replay is idempotent. The first delivery time is part of the audit
-  // receipt; a retry must not rewrite it or manufacture a later delivery.
-  if (current.operation.acknowledgedAtMs !== null) return "acknowledged";
-  await persistOperation(stateDir, {
-    ...current.operation,
-    updatedAtMs: acknowledgedAtMs,
-    acknowledgedAtMs,
-  });
-  return "acknowledged";
+/** A completed request is replayed from its durable receipt, never executed twice. */
+export class OperationReplay extends Error {
+  readonly operation: OperationRecord;
+  constructor(operation: OperationRecord) {
+    super(`OPERATION_REPLAY: ${operation.id}`);
+    this.operation = operation;
+    this.name = "OperationReplay";
+  }
+}
+
+function archivePath(stateDir: string, id: string): string {
+  return path.join(stateDir, "receipts", `${createHash("sha256").update(id).digest("hex")}.json`);
+}
+
+/** Durable history is independent of transport acknowledgement. Called under K's lock. */
+export async function archiveOperation(stateDir: string, operation: OperationRecord): Promise<void> {
+  if (operation.outcome === null) throw new Error("cannot archive an active operation");
+  const target = archivePath(stateDir, operation.id);
+  const existing = await loadArchivedOperation(stateDir, operation.id);
+  if (existing.kind === "unreadable") throw new Error(existing.reason);
+  if (existing.kind === "observed") {
+    if (existing.operation.targetVersion !== operation.targetVersion || existing.operation.outcome !== operation.outcome) {
+      throw new Error("OPERATION_ARCHIVE_CONFLICT");
+    }
+    return;
+  }
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  const tmp = `${target}.tmp`;
+  const handle = await fs.open(tmp, "w", 0o600);
+  try { await handle.writeFile(JSON.stringify(operation)); await handle.sync(); }
+  finally { await handle.close(); }
+  await platformOpsFor().renamePath(tmp, target);
+}
+
+export async function loadArchivedOperation(stateDir: string, id: string): Promise<OperationRead> {
+  try {
+    const operation = parseOperation(await fs.readFile(archivePath(stateDir, id), "utf8"));
+    if (operation.id !== id || operation.outcome === null) throw new Error("invalid archived receipt identity/outcome");
+    return { kind: "observed", operation };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return { kind: "genesis" };
+    return { kind: "unreadable", reason: "cannot read archived K receipt" };
+  }
 }

package/core/src/operationLifecycle.ts

@@ -2,7 +2,7 @@ import type { Clock } from "./clock.ts";
 import { phaseAtRest, type TxnState } from "./txn/state.ts";
 import type { ProvenanceIdentity } from "./upgrader.ts";
 import {
-  acknowledgeOperation,
+  archiveOperation,
   loadOperation,
   persistOperation,
   type OperationDescriptor,
@@ -27,7 +27,6 @@ export interface OperationLifecycle {
   settleRecovery(): Promise<void>;
   reset(): void;
   read(): Promise<OperationRead>;
-  acknowledge(operationId: string): Promise<"acknowledged" | "not-terminal" | "not-found" | "changed">;
 }
 
 export function createOperationLifecycle(
@@ -58,9 +57,7 @@ export function createOperationLifecycle(
         if (existing.operation.outcome === null) {
           throw new Error(`OPERATION_IN_PROGRESS: ${existing.operation.id}`);
         }
-        if (existing.operation.acknowledgedAtMs === null) {
-          throw new Error(`OPERATION_RECEIPT_PENDING: ${existing.operation.id}`);
-        }
+        await archiveOperation(stateDir, existing.operation);
       }
       if (existing.kind === "observed" && existing.operation.id === descriptor.id) {
         record = existing.operation;
@@ -79,7 +76,6 @@ export function createOperationLifecycle(
         reason: null,
         provenance: descriptor.provenance ?? provenance,
         metadata: { ...descriptor.metadata },
-        acknowledgedAtMs: null,
       };
       await persistOperation(stateDir, record);
     },
@@ -108,6 +104,5 @@ export function createOperationLifecycle(
 
     read: () => loadOperation(stateDir),
 
-    acknowledge: (operationId) => acknowledgeOperation(stateDir, operationId, clock.nowMs()),
   };
 }

package/core/src/platform/ops.ts

@@ -36,6 +36,13 @@ export interface PlatformOps {
    * RUNNING image; this one is plain state.
    */
   renamePath(from: string, to: string): Promise<void>;
+  /**
+   * Make directory entries durable. A rename or file creation is only
+   * crash-safe once the PARENT directory is fsync'd; `fh.sync()` on the file
+   * alone leaves the new name in the page cache. No-op where the OS does not
+   * support fsync on a directory handle (Windows).
+   */
+  syncDirectory(dir: string): Promise<void>;
   /** Identifier used to select a manifest target, e.g. "linux-x64". */
   platformKey(): string;
 }

package/core/src/platform/posix.ts

@@ -3,9 +3,16 @@
  * The only place in core allowed to name rename/signals.
  */
 import { execFileSync } from "node:child_process";
-import { promises as fs } from "node:fs";
+import { promises as fs, readFileSync } from "node:fs";
+import path from "node:path";
 import type { PlatformOps } from "./ops.ts";
 
+/** fsync a directory so entries created/renamed inside it survive power loss. */
+async function syncDirectory(dir: string): Promise<void> {
+  const fh = await fs.open(dir, "r");
+  try { await fh.sync(); } finally { await fh.close(); }
+}
+
 async function atomicReplace(filePath: string, data: Uint8Array): Promise<void> {
   const tmpPath = `${filePath}.tmp`;
   try {
@@ -28,6 +35,7 @@ async function atomicReplace(filePath: string, data: Uint8Array): Promise<void>
     }
     // Atomic on POSIX: a reader sees either the old file or the whole new one.
     await fs.rename(tmpPath, filePath);
+    await syncDirectory(path.dirname(filePath)); // the new name itself must be durable
   } catch (err) {
     await fs.unlink(tmpPath).catch(() => {});
     throw err;
@@ -77,19 +85,31 @@ export function platformKeyFor(
 export const posixOps: PlatformOps = {
   swapExecutable: atomicReplace,
   isProcessAlive(pid) {
-    try {
-      process.kill(pid, 0);
-      return true;
-    } catch {
-      return false;
+    try { process.kill(pid, 0); }
+    catch (error) { return (error as NodeJS.ErrnoException).code !== "ESRCH"; }
+    if (process.platform === "linux") {
+      try {
+        const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
+        // A zombie may stay in a container's process table, but cannot execute.
+        if (stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")) return false;
+      } catch (error) {
+        // Access restrictions are uncertainty, never proof of death.
+        return (error as NodeJS.ErrnoException).code !== "ENOENT";
+      }
     }
+    return true;
   },
   killProcess(pid) {
     process.kill(pid, "SIGKILL");
   },
   async renamePath(from, to) {
     await fs.rename(from, to);
+    const target = path.dirname(to);
+    await syncDirectory(target);
+    const source = path.dirname(from);
+    if (source !== target) await syncDirectory(source);
   },
+  syncDirectory,
   async makeExecutable(filePath) {
     await fs.chmod(filePath, 0o755);
   },

package/core/src/platform/windows.ts

@@ -51,8 +51,8 @@ export const windowsOps: PlatformOps = {
     try {
       process.kill(pid, 0);
       return true;
-    } catch {
-      return false;
+    } catch (error) {
+      return (error as NodeJS.ErrnoException).code !== "ESRCH";
     }
   },
   killProcess(pid) {
@@ -63,6 +63,12 @@ export const windowsOps: PlatformOps = {
     // Plain state moves DO work on Windows (unlike replacing a running .exe).
     await fs.rename(from, to);
   },
+  async syncDirectory() {
+    // Node does not provide a portable directory flush here. This no-op
+    // supplies no power-loss durability guarantee; NTFS journaling alone
+    // is not an acknowledgement that these particular writes reached disk.
+    await Promise.resolve();
+  },
   async makeExecutable() {
     // No executable bit on Windows; nothing to do (a real no-op).
     await Promise.resolve();

package/core/src/protocol/runner.ts

@@ -0,0 +1,81 @@
+import { parseOperation, type OperationRead } from "../operation.ts";
+
+/** Wire version is independent of the helper and application release versions. */
+export const RUNNER_PROTOCOL_VERSION = 1;
+export type RunnerRequest =
+  | { protocolVersion: 1; action: "upgrade"; id: string; targetVersion: string; consented: boolean }
+  | { protocolVersion: 1; action: "recover"; expected?: { id: string; targetVersion: string } }
+  | { protocolVersion: 1; action: "status" }
+;
+
+export interface RunnerResponse {
+  protocolVersion: 1;
+  action: RunnerRequest["action"];
+  /** 0 = completed command, 1 = failure/unknown, 2 = hold, 3 = recovery needed. */
+  exitCode: 0 | 1 | 2 | 3;
+  result: string;
+  /** Verbatim K receipt; an exception never manufactures a rollback receipt. */
+  operation: OperationRead;
+  error: string | null;
+}
+
+function objectValue(value: unknown): Record<string, unknown> {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("RUNNER_PROTOCOL_INVALID: expected an object");
+  }
+  return value as Record<string, unknown>;
+}
+
+function textField(value: unknown, field: string): string {
+  if (typeof value !== "string" || value.trim() !== value || value.length === 0 || value.length > 256) {
+    throw new Error(`RUNNER_PROTOCOL_INVALID: invalid ${field}`);
+  }
+  return value;
+}
+
+/** Reject unknown versions and fields before loading an adapter or touching disk. */
+export function parseRunnerRequest(input: unknown): RunnerRequest {
+  const value = objectValue(input);
+  if (value.protocolVersion !== RUNNER_PROTOCOL_VERSION) throw new Error("RUNNER_PROTOCOL_UNSUPPORTED");
+  const keys = value.action === "upgrade" ? ["protocolVersion", "action", "id", "targetVersion", "consented"] : value.action === "recover" ? ["protocolVersion", "action", "expected"] : ["protocolVersion", "action"];
+  if (Object.keys(value).some((key) => !keys.includes(key))) throw new Error("RUNNER_PROTOCOL_INVALID: unknown field");
+  switch (value.action) {
+    case "upgrade":
+      if (typeof value.consented !== "boolean") throw new Error("RUNNER_PROTOCOL_INVALID: consented must be boolean");
+      return { protocolVersion: 1, action: "upgrade", id: textField(value.id, "id"),
+        targetVersion: textField(value.targetVersion, "targetVersion"), consented: value.consented };
+    case "recover": {
+      if (value.expected === undefined) return { protocolVersion: 1, action: "recover" };
+      const expected = objectValue(value.expected);
+      if (Object.keys(expected).some((key) => !["id", "targetVersion"].includes(key))) throw new Error("RUNNER_PROTOCOL_INVALID: unknown expected field");
+      return { protocolVersion: 1, action: "recover", expected: {
+        id: textField(expected.id, "id"), targetVersion: textField(expected.targetVersion, "targetVersion"),
+      } };
+    }
+    case "status":
+      return { protocolVersion: 1, action: value.action };
+    default: throw new Error("RUNNER_PROTOCOL_INVALID: unknown action");
+  }
+}
+
+/** The supervisor accepts only a complete, typed runner response. */
+export function parseRunnerResponse(input: unknown): RunnerResponse {
+  const value = objectValue(input);
+  if (value.protocolVersion !== 1 || !["upgrade", "recover", "status"].includes(String(value.action)) ||
+      ![0, 1, 2, 3].includes(Number(value.exitCode)) || typeof value.exitCode !== "number" ||
+      typeof value.result !== "string" || !(value.error === null || typeof value.error === "string")) {
+    throw new Error("RUNNER_RESPONSE_INVALID");
+  }
+  const read = objectValue(value.operation);
+  let operation: OperationRead;
+  switch (read.kind) {
+    case "genesis": operation = { kind: "genesis" }; break;
+    case "unreadable":
+      if (typeof read.reason !== "string") throw new Error("RUNNER_RESPONSE_INVALID");
+      operation = { kind: "unreadable", reason: read.reason }; break;
+    case "observed": operation = { kind: "observed", operation: parseOperation(JSON.stringify(read.operation)) }; break;
+    default: throw new Error("RUNNER_RESPONSE_INVALID");
+  }
+  return { protocolVersion: 1, action: value.action as RunnerRequest["action"],
+    exitCode: value.exitCode as RunnerResponse["exitCode"], result: value.result, error: value.error, operation };
+}

package/core/src/provenance/journal.ts

@@ -174,7 +174,7 @@ export interface ProvenanceSummary {
 }
 
 /**
- * Record one reconcile in the journal (used by createUpgrader before the
+ * Record one reconcile in the journal (used by createRunner before the
  * transaction). The VERSION comes from the release the reconcile drove —
  * the journal records what was attempted, not the caller's claim. Identity
  * defaults to the local operator.

package/core/src/runner/cli.ts

@@ -0,0 +1,27 @@
+import type { Readable, Writable } from "node:stream";
+import { parseRunnerRequest } from "../protocol/runner.ts";
+import { executeRequest, type RunnerUpgrader } from "./execute.ts";
+
+/** Build-time adapter factory; request JSON can never name code to import. */
+export async function serveRunner(
+  create: () => Promise<RunnerUpgrader> | RunnerUpgrader,
+  input: Readable = process.stdin, output: Writable = process.stdout,
+): Promise<number> {
+  try {
+    let text = "";
+    for await (const chunk of input) {
+      text += String(chunk);
+      if (text.length > 16_384) throw new Error("RUNNER_REQUEST_TOO_LARGE");
+    }
+    const request = parseRunnerRequest(JSON.parse(text));
+    const response = await executeRequest(await create(), request);
+    await new Promise<void>((resolve, reject) => {
+      output.write(`${JSON.stringify(response)}\n`, (error) => error ? reject(error) : resolve());
+    });
+    return response.exitCode;
+  } catch (error) {
+    output.write(`${JSON.stringify({ protocolVersion: 1, result: "failed", exitCode: 1,
+      error: error instanceof Error ? error.message : "runner failed" })}\n`);
+    return 1;
+  }
+}

package/core/src/runner/execute.ts

@@ -0,0 +1,68 @@
+import type { Upgrader } from "../upgrader.ts";
+import { OperationReplay, type OperationRead } from "../operation.ts";
+import { systemClock, type Clock } from "../clock.ts";
+import { parseRunnerRequest, type RunnerRequest, type RunnerResponse } from "../protocol/runner.ts";
+
+export type RunnerUpgrader = Pick<Upgrader, "upgradeTo" | "recover" | "operation">;
+
+async function readOperation(upgrader: RunnerUpgrader): Promise<OperationRead> {
+  try { return await upgrader.operation(); }
+  catch { return { kind: "unreadable", reason: "cannot read K operation" }; }
+}
+
+function receiptCode(read: OperationRead): RunnerResponse["exitCode"] {
+  if (read.kind !== "observed") return read.kind === "genesis" ? 0 : 1;
+  switch (read.operation.outcome) {
+    case "promoted": case "up-to-date": return 0;
+    case "held": return 2;
+    case null: return 3;
+    default: return 1;
+  }
+}
+
+/** Runs in the disposable helper, never in the resident application's process. */
+export async function executeRequest(
+  upgrader: RunnerUpgrader, input: RunnerRequest, clock: Clock = systemClock,
+): Promise<RunnerResponse> {
+  const request = parseRunnerRequest(input);
+  let result: string;
+  let exitCode: RunnerResponse["exitCode"] = 0;
+  try {
+    switch (request.action) {
+      case "status": result = "observed"; break;
+      case "recover": await upgrader.recover(request.expected); result = "recovered"; break;
+      case "upgrade": {
+        const outcome = await upgrader.upgradeTo(request.targetVersion, {
+          consented: request.consented,
+          operation: { id: request.id, startedAtMs: clock.nowMs(),
+            provenance: { who: "local-operator", carrier: "external-runner" } },
+        });
+        result = outcome.result;
+        exitCode = outcome.result === "held" ? 2 : outcome.result === "rolled-back" ? 1 : 0;
+        break;
+      }
+    }
+    const operation = await readOperation(upgrader);
+    if (request.action === "upgrade" && exitCode === 0 &&
+        (operation.kind !== "observed" || operation.operation.id !== request.id ||
+         operation.operation.targetVersion !== request.targetVersion)) {
+      throw new Error("RUNNER_RECEIPT_MISMATCH");
+    }
+    if (request.action !== "status" && exitCode === 0) {
+      exitCode = receiptCode(operation);
+    }
+    if (operation.kind === "unreadable") exitCode = 1;
+    return { protocolVersion: 1, action: request.action, result, exitCode, operation, error: null };
+  } catch (error) {
+    if (error instanceof OperationReplay) {
+      const operation: OperationRead = { kind: "observed", operation: error.operation };
+      return { protocolVersion: 1, action: request.action, result: "replayed", operation,
+        exitCode: receiptCode(operation), error: null };
+    }
+    const operation = await readOperation(upgrader);
+    const recovering = operation.kind === "observed" && operation.operation.outcome === null;
+    return { protocolVersion: 1, action: request.action, result: recovering ? "recovery-required" : "failed",
+      exitCode: recovering ? 3 : 1, operation,
+      error: error instanceof Error ? error.message : "executor failed" };
+  }
+}