@botiverse/k-carrier 0.1.0

package/harness/src/artifact/m3.ts

@@ -0,0 +1,296 @@
+import assert from "node:assert/strict";
+import * as path from "node:path";
+import { type ToothContext } from "../teeth/registry.ts";
+import { PLAIN_DAEMON_SOURCE } from "../../../examples/service-daemon/source.ts";
+import { serveRelease } from "./m1.ts";
+import { runCommand } from "../artifact-factory/run.ts";
+import { processAlive } from "../fake-host/daemon.ts";
+import {
+  buildServiceDaemon,
+  serviceEnv,
+  readIncarnation,
+  killIncarnation,
+  respawnUntilUp,
+  seedService,
+  waitDead,
+  HOST_SHAPES,
+  type HostShape,
+} from "./m3Hosts.ts";
+import { readState } from "./m1.ts";
+
+// ---------------------------------------------------------------------------
+// m3.service-upgrade (both host shapes)
+// ---------------------------------------------------------------------------
+
+export async function checkM3ServiceUpgrade(
+  ctx: ToothContext,
+  opts: { serveBadVersion?: boolean } = {},
+): Promise<void> {
+  for (const shape of HOST_SHAPES) {
+    await runUpgradeShape(ctx, shape, opts.serveBadVersion === true);
+  }
+}
+
+async function runUpgradeShape(
+  ctx: ToothContext,
+  shape: HostShape,
+  serveBadVersion: boolean,
+): Promise<void> {
+  const shapeCtx: ToothContext = { profile: ctx.profile, sandboxDir: path.join(ctx.sandboxDir, shape) };
+  const binPath = await buildServiceDaemon(shapeCtx);
+  const seed = await serveRelease(shapeCtx, {
+    version: "1.0.0",
+    behavior: "ok",
+    name: "seed",
+    source: PLAIN_DAEMON_SOURCE,
+  });
+  const target = await serveRelease(shapeCtx, {
+    version: "2.0.0",
+    behavior: serveBadVersion ? "crash-on-start" : "ok",
+    name: "target",
+    source: PLAIN_DAEMON_SOURCE,
+  });
+  const { seedChild, seedPid } = await seedService(shapeCtx, shape, binPath, seed);
+  const env = serviceEnv(shapeCtx, shape, target.url);
+  let respawned: Awaited<ReturnType<typeof respawnUntilUp>> | null = null;
+  try {
+    const before = await readIncarnation(shapeCtx);
+    assert.ok(before, "a running incarnation must exist before the upgrade");
+    const oldPid = before!.pid;
+    const oldStartId = before!.startId;
+    assert.ok(processAlive(oldPid), "the pre-upgrade service must be alive");
+
+    const up = await runCommand(binPath, ["self", "upgrade"], { env, timeoutMs: 30000 });
+    assert.equal(
+      up.code,
+      0,
+      `self upgrade must exit 0 (${up.stderr.trim()}; serveBadVersion mutation => RED)`,
+    );
+    if (shape === "respawn") {
+      // Tracked, not discarded: if an assertion below fails, an untracked
+      // live service keeps the test runner alive and a red degrades into a
+      // hang -- the failure mode is then indistinguishable from a wedge.
+      respawned = await respawnUntilUp(shapeCtx, env);
+    }
+
+    // signal-sent ≠ dead: the OLD pid must be VERIFIED gone.
+    assert.ok(!processAlive(oldPid), `[${shape}] the old incarnation must be verified dead`);
+
+    // the new incarnation runs the new version, with a FRESH startId.
+    const after = await readIncarnation(shapeCtx);
+    assert.ok(after, `[${shape}] a new service must be running after the upgrade`);
+    assert.equal(after!.version, "2.0.0", `[${shape}] the running service must be the new version`);
+    assert.ok(processAlive(after!.pid), `[${shape}] the new service must be alive`);
+    assert.notEqual(
+      after!.startId,
+      oldStartId,
+      `[${shape}] the new incarnation must have a fresh startId (not the old one)`,
+    );
+
+    const state = await readState(env);
+    assert.equal(state.stableVersion, "2.0.0", `[${shape}] stable slot must hold the new version`);
+    assert.equal(state.phase, "promoted", `[${shape}] journal must end at promoted`);
+  } finally {
+    if (respawned && respawned.child.pid && processAlive(respawned.child.pid)) {
+      respawned.child.kill("SIGKILL");
+    }
+    await killIncarnation(shapeCtx);
+    if (processAlive(seedPid)) seedChild.kill("SIGKILL");
+    await seed.stop();
+    await target.stop();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// m3.service-rollback (both host shapes)
+// ---------------------------------------------------------------------------
+
+export async function checkM3ServiceRollback(
+  ctx: ToothContext,
+  opts: { serveGoodVersion?: boolean } = {},
+): Promise<void> {
+  for (const shape of HOST_SHAPES) {
+    await runRollbackShape(ctx, shape, opts.serveGoodVersion === true);
+  }
+}
+
+async function runRollbackShape(
+  ctx: ToothContext,
+  shape: HostShape,
+  serveGoodVersion: boolean,
+): Promise<void> {
+  const shapeCtx: ToothContext = { profile: ctx.profile, sandboxDir: path.join(ctx.sandboxDir, shape) };
+  const binPath = await buildServiceDaemon(shapeCtx);
+  const seed = await serveRelease(shapeCtx, {
+    version: "1.0.0",
+    behavior: "ok",
+    name: "seed",
+    source: PLAIN_DAEMON_SOURCE,
+  });
+  const target = await serveRelease(shapeCtx, {
+    version: "2.0.0",
+    behavior: serveGoodVersion ? "ok" : "crash-on-start",
+    name: "target",
+    source: PLAIN_DAEMON_SOURCE,
+  });
+  const { seedChild, seedPid } = await seedService(shapeCtx, shape, binPath, seed);
+  const env = serviceEnv(shapeCtx, shape, target.url);
+  let respawned: Awaited<ReturnType<typeof respawnUntilUp>> | null = null;
+  try {
+    const before = await readIncarnation(shapeCtx);
+    assert.ok(before, "a running incarnation must exist before the upgrade");
+    const oldPid = before!.pid;
+
+    const up = await runCommand(binPath, ["self", "upgrade"], { env, timeoutMs: 30000 });
+    if (shape === "respawn") {
+      // Tracked, not discarded: if an assertion below fails, an untracked
+      // live service keeps the test runner alive and a red degrades into a
+      // hang -- the failure mode is then indistinguishable from a wedge.
+      respawned = await respawnUntilUp(shapeCtx, env);
+    }
+
+    const state = await readState(env);
+    if (serveGoodVersion) {
+      // mutation: a GOOD version is served — it promotes, so the rollback
+      // expectation must go RED
+      assert.equal(
+        state.phase,
+        "rolled-back",
+        `[${shape}] a bad version must roll back (good => RED)`,
+      );
+      return;
+    }
+    if (shape === "spawn") {
+      // the spawn-shape driver survives and returns the outcome itself
+      assert.equal(
+        up.code,
+        1,
+        `[${shape}] upgrade to a bad version must exit non-zero (${up.stderr.trim()})`,
+      );
+    }
+    // (the respawn-shape driver exits 0 to request the successor; the
+    // outcome is decided by the successor's recovery below)
+    assert.equal(state.phase, "rolled-back", `[${shape}] journal must end at rolled-back`);
+    assert.equal(state.stableVersion, "1.0.0", `[${shape}] stable slot must hold the old version`);
+    assert.equal(state.experimentVersion, null, `[${shape}] experiment slot must be cleared`);
+
+    // The old version must be PULLED BACK AND ACTUALLY RUNNING — verified
+    // alive, not just the slots reverted.
+    const running = await readIncarnation(shapeCtx);
+    assert.ok(running, `[${shape}] a service must be running after the rollback`);
+    assert.equal(running!.version, "1.0.0", `[${shape}] the pulled-back service must be the old version`);
+    assert.ok(processAlive(running!.pid), `[${shape}] the pulled-back service must be ACTUALLY running`);
+    assert.notEqual(
+      running!.startId,
+      before!.startId,
+      `[${shape}] the pulled-back service is a fresh incarnation (new startId)`,
+    );
+    assert.ok(!processAlive(oldPid), `[${shape}] the pre-upgrade incarnation must stay dead`);
+  } finally {
+    if (respawned && respawned.child.pid && processAlive(respawned.child.pid)) {
+      respawned.child.kill("SIGKILL");
+    }
+    await killIncarnation(shapeCtx);
+    if (processAlive(seedPid)) seedChild.kill("SIGKILL");
+    await seed.stop();
+    await target.stop();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// m3.stuck-driver-evidence-recovery (respawn shape)
+// ---------------------------------------------------------------------------
+
+/**
+ * A wedged driver (host.stop() never returns) must not wedge the updater:
+ * the host-call budget times the driver out, the upgrade lock is released,
+ * the owner respawns the successor, and the successor's recovery decides by
+ * EVIDENCE (a live v2 with a fresh startId) — never by a "this restart was
+ * planned" flag. `skipRecovery` simulates the must-red mutation (the
+ * successor never recovers, so the transaction stays pending).
+ */
+export async function checkM3StuckDriverEvidence(
+  ctx: ToothContext,
+  opts: { skipRecovery?: boolean } = {},
+): Promise<void> {
+  const shape: HostShape = "respawn";
+  const shapeCtx: ToothContext = { profile: ctx.profile, sandboxDir: path.join(ctx.sandboxDir, shape) };
+  const binPath = await buildServiceDaemon(shapeCtx);
+  const seed = await serveRelease(shapeCtx, {
+    version: "1.0.0",
+    behavior: "ok",
+    name: "seed",
+    source: PLAIN_DAEMON_SOURCE,
+  });
+  const target = await serveRelease(shapeCtx, {
+    version: "2.0.0",
+    behavior: "ok",
+    name: "target",
+    source: PLAIN_DAEMON_SOURCE,
+  });
+  const { seedChild, seedPid } = await seedService(shapeCtx, shape, binPath, seed);
+  const env = serviceEnv(shapeCtx, shape, target.url);
+  let oldPid: number | null = null;
+  try {
+    const before = await readIncarnation(shapeCtx);
+    assert.ok(before, "a running incarnation must exist before the upgrade");
+    oldPid = before!.pid;
+
+    // The driver is wedged at stop(): it must time out (host-call budget),
+    // release the lock, and exit — never wedge the tooth.
+    const stuckEnv = { ...env, K_STUCK_DRIVER: "1" };
+    const up = await runCommand(binPath, ["self", "upgrade"], {
+      env: stuckEnv,
+      timeoutMs: 30000,
+    });
+    assert.notEqual(up.code, 0, "the wedged driver must fail (budget timeout), not hang");
+
+    // The owner respawns the successor from the new bytes; its recovery
+    // acquires the (released) lock and decides by evidence.
+    const respawnEnv = { ...env, ...(opts.skipRecovery === true ? { K_SKIP_RECOVERY: "1" } : {}) };
+    const { info: successor, child } = await respawnUntilUp(shapeCtx, respawnEnv);
+    try {
+      if (opts.skipRecovery) {
+        // mutation: the successor never recovers — the transaction stays
+        // pending at handing-over, so the promoted assertion goes RED
+        const st = await readState(env);
+        assert.equal(st.phase, "promoted", "the successor's recovery must finish the transaction (skipRecovery => RED)");
+        return;
+      }
+
+      // ② the successor is up (respawned from the experiment bytes)
+      assert.equal(successor.version, "2.0.0", "the successor must run the new version");
+      assert.ok(processAlive(successor.pid), "the successor must be alive");
+      assert.notEqual(successor.startId, before!.startId, "the successor is a fresh incarnation");
+
+      // ③ the recovery's judgment matched the evidence (v2 + fresh startId):
+      // the transaction reached promoted, so the lock WAS released (the
+      // recovery acquired it) and the evidence said handover succeeded.
+      const state = await readState(env);
+      assert.equal(state.phase, "promoted", "recovery must finish at promoted (evidence, not a flag)");
+      assert.equal(state.stableVersion, "2.0.0", "stable must hold the new version");
+    } finally {
+      if (processAlive(successor.pid)) {
+        try {
+          process.kill(successor.pid, "SIGKILL");
+        } catch {
+          // already gone
+        }
+        await waitDead(successor.pid, 5000);
+      }
+      void child;
+    }
+  } finally {
+    await killIncarnation(shapeCtx);
+    if (oldPid !== null && processAlive(oldPid)) {
+      try {
+        process.kill(oldPid, "SIGKILL"); // the wedged stop never killed it
+      } catch {
+        // already gone
+      }
+    }
+    if (processAlive(seedPid)) seedChild.kill("SIGKILL");
+    await seed.stop();
+    await target.stop();
+  }
+}

package/harness/src/artifact/m3Hosts.ts

@@ -0,0 +1,241 @@
+import assert from "node:assert/strict";
+import * as path from "node:path";
+import { sandboxMarkerFor } from "../scenario/sandbox.ts";
+import { promises as fs } from "node:fs";
+import { spawn, type ChildProcess } from "node:child_process";
+import { type ToothContext } from "../teeth/registry.ts";
+import { currentPlatformKey } from "../../../core/src/artifact/staticManifestSource.ts";
+import { FakeServer } from "../fake-server/server.ts";
+import { ArtifactFactory } from "../artifact-factory/factory.ts";
+import { processAlive } from "../fake-host/daemon.ts";
+import { commandForArtifact, runCommand } from "../artifact-factory/run.ts";
+import { PLAIN_DAEMON_SOURCE } from "../../../examples/service-daemon/source.ts";
+import { coreUpgraderUrl, readState } from "./m1.ts";
+
+/**
+ * M3 host orchestration helpers — shared by both service-profile teeth
+ * (m3.service-upgrade / m3.service-rollback), each of which runs on BOTH
+ * host shapes:
+ *  - spawn:   self-starting — the driver spawns the successor and finishes
+ *             the transaction itself.
+ *  - respawn: computer's shape — the driver DIES mid-handover (it cannot
+ *             start itself; the owner respawns it from the new bytes), and
+ *             the SUCCESSOR finishes the transaction via recovery, proving
+ *             the handover by a fresh startId (a flag could be set by the
+ *             crash path; a different incarnation is something that
+ *             happened).
+ */
+
+export type HostShape = "spawn" | "respawn";
+export const HOST_SHAPES: HostShape[] = ["spawn", "respawn"];
+
+const INCARNATION_FILE = "incarnation.json";
+
+/** Build the service-daemon demo binary (v1.0.0, ok) into <sandbox>/app. */
+export async function buildServiceDaemon(ctx: ToothContext): Promise<string> {
+  const factory = new ArtifactFactory({
+    cacheDir: path.join(ctx.sandboxDir, "cache"),
+    demoSource: PLAIN_DAEMON_SOURCE,
+  });
+  const buildServer = new FakeServer({ storeDir: path.join(ctx.sandboxDir, "build") });
+  await buildServer.start();
+  try {
+    const rel = await factory.makeRelease({
+      version: "1.0.0",
+      behavior: "ok",
+      store: buildServer.store,
+      platform: currentPlatformKey(),
+    });
+    const appDir = path.join(ctx.sandboxDir, "app");
+    await fs.mkdir(appDir, { recursive: true });
+    const binPath = path.join(appDir, "service-daemon");
+    await fs.writeFile(binPath, rel.artifactBytes, { mode: 0o755 });
+    return binPath;
+  } finally {
+    await buildServer.stop();
+  }
+}
+
+function stateDir(ctx: ToothContext): string {
+  return path.join(ctx.sandboxDir, "state");
+}
+
+export function serviceEnv(
+  ctx: ToothContext,
+  shape: HostShape,
+  baseUrl: string,
+): Record<string, string> {
+  return {
+    K_RELEASE_BASE: baseUrl,
+    K_STATE_DIR: stateDir(ctx),
+    K_HOST_SHAPE: shape,
+    K_CORE_UPGRADER: coreUpgraderUrl(),
+    // Every process this tooth spawns — and, via inheritance, every process
+    // the driver spawns — carries the sandbox marker, so the sandbox
+    // teardown's verifyProcessTreeDead is the backstop for any leak: a "red"
+    // can never degrade into a hang.
+    // Resolved back to the SANDBOX id: ctx here is a per-shape NESTED context,
+    // and basename() of that is "spawn"/"respawn" -- a marker no teardown scan
+    // matches, which turns the backstop into a scan that always finds zero.
+    K_SANDBOX_MARKER: sandboxMarkerFor(ctx.sandboxDir),
+  };
+}
+
+export interface Incarnation {
+  version: string;
+  pid: number;
+  startId: string;
+}
+
+export async function readIncarnation(ctx: ToothContext): Promise<Incarnation | null> {
+  try {
+    return JSON.parse(
+      await fs.readFile(path.join(stateDir(ctx), INCARNATION_FILE), "utf8"),
+    ) as Incarnation;
+  } catch {
+    return null;
+  }
+}
+
+export function spawnService(binPath: string, env: Record<string, string>): { child: ChildProcess; pid: number } {
+  // Windows cannot spawn a shebang artifact directly (see commandForArtifact).
+  const routed = commandForArtifact(binPath, []);
+  const child = spawn(routed.cmd, routed.args, {
+    env: { ...process.env, ...env },
+    stdio: ["pipe", "pipe", "ignore"],
+  });
+  if (!child.pid) throw new Error("service did not get a pid");
+  return { child, pid: child.pid };
+}
+
+function readLine(child: ChildProcess, prefix: string, timeoutMs = 5000): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let buffer = "";
+    const timer = setTimeout(() => {
+      cleanup();
+      reject(new Error(`timed out waiting for "${prefix}" from service`));
+    }, timeoutMs);
+    const onData = (chunk: Buffer): void => {
+      buffer += chunk.toString("utf8");
+      // Only COMPLETE lines (newline-terminated) are protocol messages.
+      for (;;) {
+        const nl = buffer.indexOf("\n");
+        if (nl === -1) break;
+        const line = buffer.slice(0, nl);
+        buffer = buffer.slice(nl + 1);
+        if (line.startsWith(`${prefix} `)) {
+          cleanup();
+          resolve(line.slice(prefix.length + 1));
+          return;
+        }
+      }
+    };
+    const cleanup = (): void => {
+      clearTimeout(timer);
+      child.stdout?.off("data", onData);
+    };
+    child.stdout?.on("data", onData);
+  });
+}
+
+/** Ready line, or null fast when the child exits / times out. */
+export function tryReady(child: ChildProcess, timeoutMs: number): Promise<Incarnation | null> {
+  return new Promise((resolve) => {
+    let done = false;
+    const finish = (v: Incarnation | null): void => {
+      if (!done) {
+        done = true;
+        resolve(v);
+      }
+    };
+    readLine(child, "ready", timeoutMs)
+      .then((line) => finish(JSON.parse(line) as Incarnation))
+      .catch(() => finish(null));
+    child.once("exit", () => finish(null));
+  });
+}
+
+export async function waitDead(pid: number, timeoutMs = 5000): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (processAlive(pid)) {
+    if (Date.now() > deadline) throw new Error(`pid ${pid} still alive`);
+    await new Promise((r) => {
+      setTimeout(r, 10);
+    });
+  }
+}
+
+/** Kill the running incarnation and prove it is gone (cleanup discipline). */
+export async function killIncarnation(ctx: ToothContext): Promise<void> {
+  const inc = await readIncarnation(ctx);
+  if (inc && processAlive(inc.pid)) {
+    try {
+      process.kill(inc.pid, "SIGKILL");
+    } catch {
+      // already gone
+    }
+    await waitDead(inc.pid, 5000);
+  }
+}
+
+/**
+ * The OWNER (respawn shape): the driver exits mid-handover; the owner
+ * respawns the service from the new bytes (experiment first), falling back
+ * to stable when the new version cannot start, and respawning again when a
+ * recovery asks for a successor by exiting. A bounded loop: the service
+ * eventually stays up or the owner gives up.
+ */
+export async function respawnUntilUp(
+  ctx: ToothContext,
+  env: Record<string, string>,
+): Promise<{ info: Incarnation; child: ChildProcess }> {
+  const dir = stateDir(ctx);
+  for (let attempt = 0; attempt < 6; attempt++) {
+    const slot: "experiment" | "stable" = attempt === 0 ? "experiment" : "stable";
+    const artifact = path.join(dir, "slots", slot, "artifact.bin");
+    try {
+      await fs.access(artifact);
+    } catch {
+      continue;
+    }
+    const child = spawn(process.execPath, [artifact], {
+      env: { ...process.env, ...env },
+      stdio: ["pipe", "pipe", "ignore"],
+    });
+    const info = await tryReady(child, 2000);
+    if (info !== null) return { info, child };
+    if (child.pid && processAlive(child.pid)) child.kill("SIGKILL");
+  }
+  throw new Error("owner: the service never came up");
+}
+
+/**
+ * Seed the service world: spawn the v1 service (registers), then a real
+ * driver upgrade to 1.0.0 lands stable v1 with a running incarnation
+ * (driver-completed on spawn, successor-completed on respawn).
+ */
+export async function seedService(
+  ctx: ToothContext,
+  shape: HostShape,
+  binPath: string,
+  seedServer: FakeServer,
+): Promise<{ seedChild: ChildProcess; seedPid: number }> {
+  const env = serviceEnv(ctx, shape, seedServer.url);
+  const { child, pid } = spawnService(binPath, env);
+  const ready = JSON.parse(await readLine(child, "ready")) as Incarnation;
+  assert.equal(ready.version, "1.0.0", "seed service must run v1");
+  assert.ok(processAlive(ready.pid), "seed service must be alive");
+
+  const up = await runCommand(binPath, ["self", "upgrade"], { env, timeoutMs: 30000 });
+  assert.equal(up.code, 0, `seed upgrade must exit 0 (${up.stderr.trim()})`);
+  if (shape === "respawn") {
+    await respawnUntilUp(ctx, env); // the driver died; the owner brings the successor up
+  }
+  const seeded = await readState(env);
+  assert.equal(seeded.stableVersion, "1.0.0", "seed upgrade must land stable 1.0.0");
+  const running = await readIncarnation(ctx);
+  assert.ok(running, "a service must be running after the seed upgrade");
+  assert.equal(running.version, "1.0.0");
+  assert.ok(processAlive(running.pid), "the seeded service must be alive");
+  return { seedChild: child, seedPid: pid };
+}