hilos-agent 0.10.1 → 0.11.2

package/src/iterate-claim-recovery.mjs

@@ -0,0 +1,574 @@
+// Durable recovery records for local-daemon Iterate claims. A claim changes the
+// canonical run generation before its HTTP response reaches this machine, so a
+// response-lost process must retain the exact owner token across restarts.
+
+import * as nodeFs from "node:fs";
+import { execFileSync } from "node:child_process";
+import { createHash, randomUUID } from "node:crypto";
+import { homedir, platform } from "node:os";
+import { join } from "node:path";
+
+export const ITERATE_CLAIM_RECOVERY_DIR = join(homedir(), ".hilos", "iterate-claims");
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const INTRINSIC_SELF_ID_PREFIX = "intrinsic-self:";
+const SELF_PROCESS_INSTANCE_ID = `${INTRINSIC_SELF_ID_PREFIX}${platform()}:${Math.round(
+  Date.now() - process.uptime() * 1000,
+)}:${randomUUID()}`;
+
+function safeSegment(value) {
+  return String(value || "").replace(/[^a-zA-Z0-9-]/g, "");
+}
+
+export function canonicalIterateClaimServerIdentity(url) {
+  try {
+    const parsed = new URL(String(url || "").trim());
+    parsed.hash = "";
+    parsed.username = "";
+    parsed.password = "";
+    parsed.pathname = parsed.pathname.replace(/\/+$/, "") || "/";
+    parsed.searchParams.sort();
+    // URL normalizes scheme/host casing for us. Preserve the path's casing:
+    // unlike hostnames, HTTP paths can name different resources by case.
+    return parsed.toString().replace(/\/$/, "");
+  } catch {
+    return String(url || "").trim().replace(/\/+$/, "").toLowerCase();
+  }
+}
+
+/** Stable across bearer rotation; the bearer token is never hashed or written. */
+export function iterateClaimConnectionFingerprint({ url }) {
+  return createHash("sha256")
+    .update(canonicalIterateClaimServerIdentity(url))
+    .digest("hex")
+    .slice(0, 24);
+}
+
+function validRecord(value) {
+  return Boolean(
+    value &&
+      typeof value.runId === "string" &&
+      UUID_RE.test(value.runId) &&
+      Number.isSafeInteger(value.expectedIterateTurn) &&
+      value.expectedIterateTurn >= 0 &&
+      typeof value.claimId === "string" &&
+      UUID_RE.test(value.claimId),
+  );
+}
+
+function defaultProcessAlive(pid) {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code !== "ESRCH";
+  }
+}
+
+// A PID only identifies a process until the OS reuses it. Persist an identity
+// tied to the process incarnation as well, so an owner left by a crash or
+// reboot cannot be mistaken for the later process that inherited its PID.
+// Linux exposes a boot UUID plus the process start tick without spawning. The
+// supported desktop platforms expose an absolute process start time through
+// their standard process tools.
+export function defaultProcessInstanceIdentity(
+  pid,
+  {
+    readFileSync = nodeFs.readFileSync,
+    execute = execFileSync,
+    currentPid = process.pid,
+    selfIdentity = SELF_PROCESS_INSTANCE_ID,
+    platformName = platform(),
+  } = {},
+) {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return null;
+  try {
+    if (platformName === "linux") {
+      try {
+        const bootId = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
+        const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
+        const commandEnd = stat.lastIndexOf(")");
+        if (bootId && commandEnd >= 0) {
+          // After the command's closing parenthesis, index 0 is field 3
+          // (state), making index 19 field 22 (start tick since boot).
+          const startTick = stat.slice(commandEnd + 1).trim().split(/\s+/)[19];
+          if (startTick) return `linux:${bootId}:${startTick}`;
+        }
+      } catch {
+        // Sandboxed Linux environments may not mount procfs. Fall through to
+        // the same POSIX process-start query used by macOS/BSD.
+      }
+    }
+    if (platformName === "win32") {
+      const ticks = execute(
+        "powershell.exe",
+        [
+          "-NoProfile",
+          "-NonInteractive",
+          "-Command",
+          `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`,
+        ],
+        { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] },
+      ).trim();
+      return ticks ? `win32:${ticks}` : null;
+    }
+    const started = execute("ps", ["-o", "lstart=", "-p", String(pid)], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    }).trim();
+    if (!started) return null;
+    if (["darwin", "freebsd", "openbsd", "netbsd"].includes(platformName)) {
+      const boot = execute("sysctl", ["-n", "kern.boottime"], {
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      }).trim();
+      // BSD ps exposes lstart only to the second. Mark that precision in the
+      // durable identity: unequal values prove reuse, while equal values are
+      // intentionally treated as ambiguous/live rather than risking two
+      // daemons when a PID is recycled inside the same second.
+      return boot ? `${platformName}:coarse-second:${boot}:${started}` : null;
+    }
+    return `${platformName}:coarse-second:${started}`;
+  } catch {
+    // This remains available in restricted runtimes that deny procfs/ps
+    // access, and is shared by every store created in this process. A
+    // different process that cannot inspect it will conservatively fail
+    // closed rather than preempting a possibly-live owner.
+    return pid === currentPid ? selfIdentity : null;
+  }
+}
+
+function syncDirectory(fs, path) {
+  const fd = fs.openSync(path, "r");
+  try {
+    fs.fsyncSync(fd);
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
+/**
+ * One file per claim plus one exclusive agent/server scope owner.
+ * @param {{ agentId: string, url: string, token?: string, dir?: string,
+ *   log?: { error?: (message: string) => void }, fs?: any,
+ *   isProcessAlive?: (pid: number) => boolean,
+ *   getProcessInstanceIdentity?: (pid: number) => string | null,
+ *   pid?: number, ownerId?: string }} options
+ */
+export function createIterateClaimRecoveryStore({
+  agentId,
+  url,
+  // Accepted for call-site compatibility only. Deliberately unused.
+  token: _token,
+  dir = ITERATE_CLAIM_RECOVERY_DIR,
+  log,
+  fs = nodeFs,
+  isProcessAlive = defaultProcessAlive,
+  getProcessInstanceIdentity = defaultProcessInstanceIdentity,
+  pid = process.pid,
+  ownerId = randomUUID(),
+} = {}) {
+  const scope = `${safeSegment(agentId)}-${iterateClaimConnectionFingerprint({ url })}`;
+  const scopeDir = join(dir, scope);
+  const ownerKey = safeSegment(ownerId);
+  const lockName = `.owner.${ownerKey}.lock`;
+  const lockPath = join(scopeDir, lockName);
+  const electionPath = join(scopeDir, ".owner-election.lock");
+  const recordPath = (claimId) => join(scopeDir, `${safeSegment(claimId)}.json`);
+  const processInstanceIdentity = getProcessInstanceIdentity(pid);
+  let lockHeld = false;
+  let heldElectionPath = null;
+
+  function ensureScope() {
+    fs.mkdirSync(scopeDir, { recursive: true, mode: 0o700 });
+    fs.chmodSync(scopeDir, 0o700);
+  }
+
+  function durableWrite(path, value, { exclusive = false } = {}) {
+    const fd = fs.openSync(path, exclusive ? "wx" : "w", 0o600);
+    try {
+      fs.writeFileSync(fd, value, { encoding: "utf8" });
+      fs.chmodSync(path, 0o600);
+      fs.fsyncSync(fd);
+    } finally {
+      fs.closeSync(fd);
+    }
+  }
+
+  function readLock(path = lockPath) {
+    try {
+      return JSON.parse(fs.readFileSync(path, "utf8"));
+    } catch {
+      return null;
+    }
+  }
+
+  function matchesOwner(value) {
+    return Boolean(
+      value?.ownerId === ownerId &&
+        value?.pid === pid &&
+        value?.processInstanceIdentity === processInstanceIdentity,
+    );
+  }
+
+  function validLock(value, name) {
+    return Boolean(
+      value &&
+        Number.isSafeInteger(value.pid) &&
+        value.pid > 0 &&
+        typeof value.ownerId === "string" &&
+        safeSegment(value.ownerId) &&
+        (value.processInstanceIdentity === undefined ||
+          (typeof value.processInstanceIdentity === "string" &&
+            value.processInstanceIdentity.length > 0)) &&
+        name === `.owner.${safeSegment(value.ownerId)}.lock`,
+    );
+  }
+
+  function ownerIsLive(owner) {
+    if (!isProcessAlive(owner.pid)) return false;
+    // Locks written by the previous daemon version have no incarnation
+    // identity. During a rolling upgrade, numeric liveness is the only safe
+    // evidence available: never preempt a live legacy owner, but reclaim it
+    // once that PID is gone.
+    if (owner.processInstanceIdentity === undefined) return true;
+    // This marker is written only when the owner could not inspect even its
+    // own OS process identity. It is deliberately not cross-process
+    // comparable: a later contender may have broader procfs/ps permissions,
+    // but its different real identity does not prove that the live fallback
+    // owner was replaced.
+    if (owner.processInstanceIdentity.startsWith(INTRINSIC_SELF_ID_PREFIX)) return true;
+    let currentIdentity;
+    try {
+      currentIdentity = getProcessInstanceIdentity(owner.pid);
+    } catch {
+      // A live-looking process whose identity cannot be inspected is not safe
+      // to preempt. A later poll can recover it once liveness is knowable.
+      return true;
+    }
+    if (!currentIdentity) return true;
+    // The reverse transition is equally non-comparable: this process may have
+    // published a durable OS identity while inspection worked, then fallen
+    // back later (sandbox/profile change, transient procfs denial). Numeric
+    // liveness plus an intrinsic current-process marker is ambiguous, not
+    // evidence that the published owner was replaced.
+    if (currentIdentity.startsWith(INTRINSIC_SELF_ID_PREFIX)) return true;
+    return currentIdentity === owner.processInstanceIdentity;
+  }
+
+  function ownsPublishedLock() {
+    if (!lockHeld) return false;
+    const owner = readLock();
+    const elected = heldElectionPath ? readLock(heldElectionPath) : null;
+    if (matchesOwner(owner) && matchesOwner(elected)) {
+      return true;
+    }
+    lockHeld = false;
+    return false;
+  }
+
+  function removeOwnPublishedLock() {
+    const owner = readLock();
+    const elected = heldElectionPath ? readLock(heldElectionPath) : null;
+    if (!matchesOwner(owner) || !matchesOwner(elected)) return false;
+    try {
+      fs.unlinkSync(heldElectionPath);
+      fs.unlinkSync(lockPath);
+      syncDirectory(fs, scopeDir);
+      lockHeld = false;
+      heldElectionPath = null;
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  function removeOwnCandidate() {
+    if (!matchesOwner(readLock())) return false;
+    try {
+      fs.unlinkSync(lockPath);
+      syncDirectory(fs, scopeDir);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  function acquireElection() {
+    let candidatePath = electionPath;
+    let traversalBudget;
+    try {
+      // A valid chain with N existing anchors reaches either a live owner or
+      // its first missing successor in at most N + 1 hops. Deriving the bound
+      // from durable state avoids a lifetime crash-generation ceiling while
+      // still bounding work on corrupt or adversarial directories.
+      traversalBudget =
+        fs
+          .readdirSync(scopeDir)
+          .filter(
+            (name) =>
+              name === ".owner-election.lock" ||
+              (name.startsWith(".owner-election-after-") && name.endsWith(".lock")),
+          ).length + 1;
+    } catch {
+      return false;
+    }
+    const visited = new Set();
+    for (let attempt = 0; attempt < traversalBudget; attempt += 1) {
+      if (visited.has(candidatePath)) return false;
+      visited.add(candidatePath);
+      try {
+        fs.linkSync(lockPath, candidatePath);
+        syncDirectory(fs, scopeDir);
+        heldElectionPath = candidatePath;
+        return true;
+      } catch (error) {
+        if (error?.code !== "EEXIST") return false;
+      }
+      const elected = readLock(candidatePath);
+      if (matchesOwner(elected)) {
+        heldElectionPath = candidatePath;
+        return true;
+      }
+      if (elected && validLock(elected, `.owner.${safeSegment(elected.ownerId)}.lock`) && ownerIsLive(elected)) {
+        return false;
+      }
+      // Never replace or unlink a stale anchor: path-based cleanup cannot be
+      // conditional on the inode inspected above, so a delayed contender
+      // could otherwise remove a newer winner. Every contender instead walks
+      // the same immutable successor chain and atomically races to publish the
+      // first missing link.
+      try {
+        const stat = fs.statSync(candidatePath);
+        const staleKey = createHash("sha256")
+          .update(`${stat.dev}:${stat.ino}:${stat.size}:${safeSegment(elected?.ownerId)}`)
+          .digest("hex")
+          .slice(0, 24);
+        candidatePath = join(scopeDir, `.owner-election-after-${staleKey}.lock`);
+      } catch (error) {
+        if (error?.code === "ENOENT") {
+          candidatePath = electionPath;
+          continue;
+        }
+        return false;
+      }
+    }
+    return false;
+  }
+
+  return {
+    acquireLock() {
+      if (lockHeld) return true;
+      if (!ownerKey || !processInstanceIdentity) return false;
+      try {
+        ensureScope();
+      } catch (error) {
+        log?.error?.(`iterate claim scope unavailable: ${error?.message || error}`);
+        return false;
+      }
+      // Publish through an fsynced private file and an atomic hard link. Readers
+      // can therefore see either no contender or a complete contender, never a
+      // partially-written lock. Every process has an immutable, unique path;
+      // stale cleanup can never unlink a later contender at another path.
+      let temp = join(scopeDir, `.owner-pending.${ownerKey}.${pid}.${randomUUID()}.tmp`);
+      try {
+        durableWrite(temp, JSON.stringify({
+          pid,
+          ownerId,
+          processInstanceIdentity,
+          lockProtocol: 2,
+          state: "contender",
+        }), { exclusive: true });
+        fs.linkSync(temp, lockPath);
+        fs.unlinkSync(temp);
+        temp = null;
+        syncDirectory(fs, scopeDir);
+      } catch (error) {
+        if (temp) {
+          try {
+            fs.unlinkSync(temp);
+          } catch {
+            /* best-effort cleanup of an unpublished contender */
+          }
+        }
+        removeOwnCandidate();
+        log?.error?.(`iterate claim scope not locked: ${error?.message || error}`);
+        return false;
+      }
+
+      if (!acquireElection()) {
+        removeOwnCandidate();
+        return false;
+      }
+
+      let names;
+      try {
+        names = fs
+          .readdirSync(scopeDir)
+          .filter((name) => name.startsWith(".owner.") && name.endsWith(".lock"));
+      } catch (error) {
+        removeOwnPublishedLock();
+        log?.error?.(`iterate claim scope not inspected: ${error?.message || error}`);
+        return false;
+      }
+
+      let ownSeen = false;
+      let liveContender = false;
+      const stalePaths = [];
+      for (const name of names) {
+        const path = join(scopeDir, name);
+        const owner = readLock(path);
+        if (
+          name === lockName &&
+          owner?.ownerId === ownerId &&
+          owner?.pid === pid &&
+          owner?.processInstanceIdentity === processInstanceIdentity
+        ) {
+          ownSeen = true;
+          continue;
+        }
+        if (owner?.lockProtocol === 2 && owner?.state === "contender") {
+          if (!ownerIsLive(owner)) stalePaths.push(path);
+          continue;
+        }
+        if (!validLock(owner, name) || !ownerIsLive(owner)) {
+          stalePaths.push(path);
+          continue;
+        }
+        liveContender = true;
+      }
+
+      if (!ownSeen || liveContender) {
+        removeOwnPublishedLock();
+        return false;
+      }
+
+      // These paths are immutable per owner UUID. A newly-published contender
+      // necessarily has a different path and is left untouched; if it appears
+      // after our scan it will observe this live owner and withdraw itself.
+      for (const path of stalePaths) {
+        try {
+          fs.unlinkSync(path);
+          syncDirectory(fs, scopeDir);
+        } catch (error) {
+          if (error?.code !== "ENOENT") {
+            log?.error?.(`stale iterate claim lock not cleared: ${error?.message || error}`);
+          }
+        }
+      }
+      lockHeld = true;
+      return true;
+    },
+
+    releaseLock() {
+      if (!ownsPublishedLock()) return false;
+      if (!removeOwnPublishedLock()) {
+        const error = new Error("published owner changed or could not be removed");
+        log?.error?.(`iterate claim scope lock not released: ${error?.message || error}`);
+        return false;
+      }
+      return true;
+    },
+
+    holdsLock() {
+      return ownsPublishedLock();
+    },
+
+    persist(record) {
+      if (!ownsPublishedLock() || !validRecord(record) || !safeSegment(record.claimId)) return false;
+      let temp = null;
+      try {
+        ensureScope();
+        temp = join(scopeDir, `.${safeSegment(record.claimId)}.${pid}.${randomUUID()}.tmp`);
+        durableWrite(temp, JSON.stringify(record), { exclusive: true });
+        fs.renameSync(temp, recordPath(record.claimId));
+        fs.chmodSync(recordPath(record.claimId), 0o600);
+        syncDirectory(fs, scopeDir);
+        return true;
+      } catch (error) {
+        if (temp) {
+          try {
+            fs.unlinkSync(temp);
+          } catch {
+            /* best-effort cleanup of an incomplete temp file */
+          }
+        }
+        log?.error?.(`iterate claim recovery not persisted: ${error?.message || error}`);
+        return false;
+      }
+    },
+
+    remove(claimId) {
+      if (!ownsPublishedLock()) return false;
+      try {
+        fs.unlinkSync(recordPath(claimId));
+        syncDirectory(fs, scopeDir);
+        return true;
+      } catch (error) {
+        if (error?.code === "ENOENT" && ownsPublishedLock()) return true;
+        log?.error?.(`iterate claim recovery not cleared: ${error?.message || error}`);
+        return false;
+      }
+    },
+
+    list() {
+      if (!ownsPublishedLock()) return { records: [], unresolved: 1 };
+      let names;
+      try {
+        names = fs.readdirSync(scopeDir).filter((name) => name.endsWith(".json"));
+      } catch (error) {
+        log?.error?.(`iterate claim recovery not read: ${error?.message || error}`);
+        return { records: [], unresolved: 1 };
+      }
+      const records = [];
+      let unresolved = 0;
+      for (const name of names) {
+        try {
+          const value = JSON.parse(fs.readFileSync(join(scopeDir, name), "utf8"));
+          if (validRecord(value)) records.push(value);
+          else unresolved += 1;
+        } catch (error) {
+          unresolved += 1;
+          log?.error?.(`iterate claim recovery record unreadable (${name}): ${error?.message || error}`);
+        }
+      }
+      return { records, unresolved };
+    },
+  };
+}
+
+export const TERMINAL_ITERATE_RELEASE_KINDS = new Set([
+  "released",
+  "already-released",
+  "not-owned",
+]);
+
+export async function releaseDaemonIterateClaim({ tool, recoveryStore, record }) {
+  const result = await tool("update_run", {
+    runId: record.runId,
+    releaseIterateClaim: true,
+    expectedIterateTurn: record.expectedIterateTurn,
+    claimId: record.claimId,
+  });
+  if (!TERMINAL_ITERATE_RELEASE_KINDS.has(result?.releaseKind)) return result;
+  if (!recoveryStore?.remove(record.claimId)) {
+    throw new Error("the Iterate claim recovery record could not be cleared");
+  }
+  return result;
+}
+
+export async function reconcileDaemonIterateClaims({ tool, recoveryStore, log = console }) {
+  if (!recoveryStore) return { unresolved: 0 };
+  const listed = recoveryStore.list();
+  let unresolved = listed.unresolved;
+  for (const record of listed.records) {
+    try {
+      const result = await releaseDaemonIterateClaim({ tool, recoveryStore, record });
+      if (!TERMINAL_ITERATE_RELEASE_KINDS.has(result?.releaseKind)) unresolved += 1;
+    } catch (error) {
+      unresolved += 1;
+      log?.error?.(`iterate claim recovery deferred: ${error?.message || error}`);
+    }
+  }
+  return { unresolved };
+}