@legionworks/facet 1.9.0

package/src/service/lifecycle/orphan-cleanup.ts

@@ -0,0 +1,128 @@
+/**
+ * Orphan cleanup — remove artifacts from a previous crashed run before
+ * we acquire the lock for this process.
+ *
+ * Two categories of orphan:
+ *   1. Stale lock files: a lock whose pid is dead. We never touch a
+ *      lock whose pid is alive — that would be a destructive kill of
+ *      a healthy sibling service.
+ *   2. Stray WAL/SHM sidecars with no matching database file: a
+ *      crash after SQLite created sidecars but before it durably
+ *      renamed them leaves those files behind. The next start deletes
+ *      them so the new SQLite open sees a clean WAL mode.
+ *
+ * Anything tied to a LIVE pid is left alone — orphan cleanup is for
+ * after a crash, not against a running peer.
+ */
+
+import { existsSync, rmSync, unlinkSync } from "node:fs";
+
+import {
+  isPidAlive,
+  readLockMetadata,
+  readPidStartTimeTicks as readPidStartTimeTicksReal,
+} from "./process-lock";
+
+export interface OrphanProfile {
+  readonly path: string;
+  readonly pid: number;
+  readonly startTime: number;
+}
+
+export interface OrphanProcess {
+  readonly pid: number;
+  readonly startTime: number;
+}
+
+export interface OrphanCleanupInput {
+  readonly lockPath: string;
+  readonly databasePath: string;
+  readonly profiles?: readonly OrphanProfile[];
+  readonly processes?: readonly OrphanProcess[];
+  /** Test seam: override start-time reads (e.g. to simulate unreadable /proc). Defaults to the real reader. */
+  readonly readPidStartTimeTicks?: (pid: number) => number | null;
+}
+
+export interface OrphanCleanupResult {
+  readonly removed: {
+    readonly lock: boolean;
+    readonly walSidecars: readonly string[];
+    readonly profiles: readonly string[];
+  };
+  readonly killedPids: readonly number[];
+}
+
+export function runOrphanCleanup(input: OrphanCleanupInput): OrphanCleanupResult {
+  const readPidStartTimeTicks = input.readPidStartTimeTicks ?? readPidStartTimeTicksReal;
+  let removedLock = false;
+  const removedSidecars: string[] = [];
+  const removedProfiles: string[] = [];
+  const killedPids: number[] = [];
+
+  const lockMetadata = readLockMetadata(input.lockPath);
+  if (lockMetadata !== null && (!isPidAlive(lockMetadata.pid) || isLockOwnerStale(lockMetadata))) {
+    try {
+      unlinkSync(input.lockPath);
+      removedLock = true;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+
+  for (const profile of input.profiles ?? []) {
+    if (isPidAlive(profile.pid)) {
+      const startTime = readPidStartTimeTicks(profile.pid);
+      // Fail closed: a live pid whose start time we could not read is
+      // NOT evidence of a mismatch — `null !== profile.startTime` would
+      // otherwise fall through to deletion below and remove a live
+      // process's profile directory on transient/unreadable /proc
+      // metadata. Only a confirmed dead pid or a confirmed start-time
+      // mismatch is grounds for cleanup.
+      if (startTime === null || startTime === profile.startTime) continue;
+    }
+    if (!existsSync(profile.path)) continue;
+    rmSync(profile.path, { recursive: true, force: true });
+    removedProfiles.push(profile.path);
+  }
+  for (const process of input.processes ?? []) {
+    if (!isPidAlive(process.pid) || readPidStartTimeTicks(process.pid) !== process.startTime)
+      continue;
+    try {
+      processKill(process.pid);
+      killedPids.push(process.pid);
+    } catch {
+      // The process may have exited between the identity check and the signal.
+    }
+  }
+
+  // Sidecars are only orphan if the database itself is gone.
+  if (!existsSync(input.databasePath)) {
+    for (const suffix of ["-wal", "-shm"]) {
+      const sidecar = `${input.databasePath}${suffix}`;
+      if (!existsSync(sidecar)) continue;
+      try {
+        unlinkSync(sidecar);
+        removedSidecars.push(suffix);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      }
+    }
+  }
+
+  return {
+    removed: { lock: removedLock, walSidecars: removedSidecars, profiles: removedProfiles },
+    killedPids,
+  };
+}
+
+function isLockOwnerStale(metadata: { pid: number; startTimeTicks?: number | null }): boolean {
+  return (
+    metadata.startTimeTicks !== undefined &&
+    metadata.startTimeTicks !== null &&
+    readPidStartTimeTicksReal(metadata.pid) !== metadata.startTimeTicks
+  );
+}
+
+function processKill(pid: number): void {
+  process.kill(pid, "SIGTERM");
+}

package/src/service/lifecycle/process-lock.ts

@@ -0,0 +1,235 @@
+/**
+ * Process lock with metadata.
+ *
+ * The lock file holds a JSON record describing the live service
+ * instance: pid, startTime, port, contractVersion. Acquisition is
+ * atomic — a missing/empty lock file is claimed via O_EXCL, a present
+ * lock is reclaimed only when the recorded pid is dead OR the
+ * startTime disagrees with the live process. Live contention surfaces
+ * as a typed constraint error so a second service start fails loud.
+ *
+ * The contract-version field is recorded at acquisition so an upgrade
+ * that changes `FACET_SCHEMA_VERSION` cannot silently bind to a stale
+ * lock from the previous build.
+ */
+
+import { constants as fsConstants } from "node:fs";
+import {
+  closeSync,
+  existsSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+import { FacetError } from "../../shared/errors/facet-error";
+import { FACET_SCHEMA_VERSION } from "../../shared/contracts/envelope";
+import { isPidAlive, readPidStartTimeTicks } from "../../shared/util/process";
+import { ensureOwnerOnlyDirectory } from "../../shared/util/dir-permissions";
+
+export { isPidAlive, readPidStartTimeTicks };
+
+export interface LockMetadata {
+  readonly pid: number;
+  readonly startTime: number;
+  readonly startTimeTicks?: number | null;
+  readonly port: number;
+  readonly contractVersion: string;
+}
+
+export type AcquireResult = { ok: true; metadata: LockMetadata } | { ok: false; error: FacetError };
+
+const LOCK_RETRY_ATTEMPTS = 5;
+
+const STALE_LOCK_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
+export function isLockStale(metadata: LockMetadata): boolean {
+  if (metadata.contractVersion !== FACET_SCHEMA_VERSION) {
+    // Cross-version lock: only stale when the recorded pid is dead.
+    // A live foreign cross-version pid is still a legitimate owner
+    // of its lock — we never reclaim a live peer.
+    return !isPidAlive(metadata.pid);
+  }
+  // Strong signal: the OS-recorded start time for the recorded pid
+  // does not match the startTime we have on file. If the pid is dead
+  // (or unreadable) and the record is older than the staleness budget,
+  // the lock is stale.
+  const pidAlive = isPidAlive(metadata.pid);
+  if (pidAlive) {
+    const osStart = readPidStartTimeTicks(metadata.pid);
+    if (osStart === null) {
+      // Cannot verify the OS start time; be conservative and treat the
+      // lock as LIVE so we never unlink a live foreign pid.
+      return false;
+    }
+    if (metadata.startTimeTicks !== undefined && metadata.startTimeTicks !== null) {
+      return osStart !== metadata.startTimeTicks;
+    }
+    // The recorded startTime is wall-clock ms; compare the age in
+    // ms against a coarse staleness budget. If a live pid's recorded
+    // startTime is far in the past, this lock is from a previous
+    // incarnation of that pid slot and is stale.
+    const ageMs = Date.now() - metadata.startTime;
+    return ageMs > STALE_LOCK_MAX_AGE_MS;
+  }
+  // Pid is dead → lock is stale regardless of recorded startTime.
+  return true;
+}
+
+/**
+ * Read the metadata record from `lockPath`. Returns null when the file
+ * is missing, unreadable, or not valid JSON — a corrupt lock is treated
+ * as "no live owner", so the next acquireLock() call reclaims it.
+ */
+export function readLockMetadata(lockPath: string): LockMetadata | null {
+  if (!existsSync(lockPath)) return null;
+  try {
+    const text = readFileSync(lockPath, "utf8");
+    const parsed = JSON.parse(text) as Partial<LockMetadata>;
+    if (
+      typeof parsed.pid !== "number" ||
+      typeof parsed.startTime !== "number" ||
+      typeof parsed.port !== "number" ||
+      typeof parsed.contractVersion !== "string"
+    ) {
+      return null;
+    }
+    const startTimeTicks = typeof parsed.startTimeTicks === "number" ? parsed.startTimeTicks : null;
+    return {
+      pid: parsed.pid,
+      startTime: parsed.startTime,
+      ...(startTimeTicks === null ? {} : { startTimeTicks }),
+      port: parsed.port,
+      contractVersion: parsed.contractVersion,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Atomically write the metadata record. Uses a tmpfile+rename so a
+ * crashed write never leaves a half-formed JSON on disk.
+ */
+export function writeLockMetadata(lockPath: string, metadata: LockMetadata): void {
+  const lockDir = dirname(lockPath);
+  ensureOwnerOnlyDirectory(lockDir);
+  const tmpPath = join(lockDir, `.facet-lock-${crypto.randomUUID()}.tmp`);
+  try {
+    writeFileSync(tmpPath, JSON.stringify(metadata), { mode: 0o600 });
+    renameSync(tmpPath, lockPath);
+  } catch (error) {
+    try {
+      unlinkSync(tmpPath);
+    } catch {
+      // Preserve the original write or rename failure.
+    }
+    throw error;
+  }
+}
+
+/**
+ * Acquire the process lock. If the lock file is missing, claim it
+ * atomically via O_EXCL. If it exists and is stale (pid dead or
+ * contract-version mismatch or empty metadata), reclaim. Otherwise
+ * return a typed constraint error.
+ */
+export function acquireLock(lockPath: string, metadata: LockMetadata): AcquireResult {
+  ensureOwnerOnlyDirectory(dirname(lockPath));
+
+  for (let attempt = 0; attempt < LOCK_RETRY_ATTEMPTS; attempt += 1) {
+    let fd: number;
+    try {
+      fd = openSync(
+        lockPath,
+        fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL,
+        0o600,
+      );
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      fd = -1;
+    }
+    if (fd >= 0) {
+      closeSync(fd);
+      writeLockMetadata(lockPath, metadata);
+      return { ok: true, metadata };
+    }
+
+    const existing = readLockMetadata(lockPath);
+    if (existing === null) {
+      // Lock exists but unparseable: there is no live owner we can
+      // verify, so reclaim is safe (no foreign pid to clobber).
+      try {
+        unlinkSync(lockPath);
+      } catch {
+        // Someone else raced us; loop will retry.
+      }
+      continue;
+    }
+
+    // Cross-version lock: only reclaim if the recorded owner is dead.
+    // A live foreign-pid owner running a different build is still a
+    // legitimate owner of its lock — we must not unlink it. Per D3,
+    // reclaim requires independent proof of death (kill(pid,0) ESRCH)
+    // OR an OS startTime mismatch, never a bare pid inequality.
+    if (existing.contractVersion !== metadata.contractVersion) {
+      if (!isPidAlive(existing.pid) || isLockStale(existing)) {
+        try {
+          unlinkSync(lockPath);
+        } catch {
+          continue;
+        }
+        continue;
+      }
+      return {
+        ok: false,
+        error: new FacetError("constraint", "Another facet service holds the lock", {
+          retryable: false,
+          details: { pid: existing.pid, port: existing.port, lockPath },
+        }),
+      };
+    }
+
+    // Same-version lock: reclaim ONLY when the recorded pid is dead
+    // OR the recorded startTime disagrees with what we can verify for
+    // that pid. A pid-mismatch alone is never sufficient — that would
+    // let any process clobber a live sibling's lock.
+    if (!isPidAlive(existing.pid) || isLockStale(existing)) {
+      try {
+        unlinkSync(lockPath);
+      } catch {
+        continue;
+      }
+      continue;
+    }
+
+    return {
+      ok: false,
+      error: new FacetError("constraint", "Another facet service holds the lock", {
+        retryable: false,
+        details: { pid: existing.pid, port: existing.port, lockPath },
+      }),
+    };
+  }
+
+  // We've retried the race window; give up with a typed error rather
+  // than blocking forever.
+  return {
+    ok: false,
+    error: new FacetError("constraint", "Could not acquire process lock after retries", {
+      retryable: false,
+      details: { lockPath, attempts: LOCK_RETRY_ATTEMPTS },
+    }),
+  };
+}
+
+export function releaseLock(lockPath: string): void {
+  try {
+    unlinkSync(lockPath);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+  }
+}

package/src/service/main.ts

@@ -0,0 +1,136 @@
+/**
+ * Service entrypoint. Parses a tiny env/arg surface, starts the loopback
+ * service, writes a single structured ready line to stderr, and shuts
+ * down gracefully on SIGTERM/SIGINT.
+ *
+ * Recognized env/args:
+ *   FACET_INSECURE_AUTO=1       opt into boot-time isolation fallback
+ *   --db-path <path>            override default database path
+ *   --install-token-path <p>    override install token path
+ *   --promote-token-path <p>    override promote token path
+ *   --lock-path <p>             override lock path
+ *   --evidence-path <p>         override evidence root (parent/child agreement)
+ *   --idle-timeout-ms <n>       override idle window (default 30s)
+ *   --tier0-runner-path <p>     path to a module exporting `runTier0`;
+ *                                the service uses a dynamic import on
+ *                                this path so the static boundary
+ *                                check still flags any hardcoded
+ *                                `src/validation/**` import.
+ *   --tier1-runner-path <p>     path to a module exporting `createTier1Runner`;
+ *
+ * The ready line is a single JSON object on stderr so external CLI
+ * tooling can scrape it without parsing prose.
+ */
+
+import type {
+  Tier0IsolationProbe,
+  Tier0Runner,
+  Tier0RunnerFactory,
+  Tier1AvailabilityProbe,
+  Tier1RunnerFactory,
+} from "../shared/contracts/validation";
+import { FacetError } from "../shared/errors/facet-error";
+import {
+  runServiceProcess,
+  type LoadedTier0Runner,
+  type LoadedTier1Runner,
+  type ParsedServiceArgs,
+  type ServiceRunnerModules,
+} from "./process";
+
+/**
+ * Dynamic-import the Tier 0 runner from the caller-provided path.
+ * The path comes from CLI argv (NOT a static string literal), so the
+ * boundary checker's regex does not match — there is no static
+ * `import "../validation/..."` in this file. The runtime resolution
+ * is the canonical mechanism for cross-process dependency injection
+ * in the service child; the CLI (src/cli/) is the only place that
+ * knows the concrete path.
+ */
+async function loadTier0Runner(path: string): Promise<LoadedTier0Runner> {
+  // The path is an arbitrary string at runtime; TypeScript cannot
+  // type-check the import so we cast via unknown. The contract is
+  // enforced by `typeof mod.runTier0 === "function"`.
+  const dynamicPath = path;
+  const mod = (await import(dynamicPath)) as {
+    createTier0Runner?: unknown;
+    runTier0?: unknown;
+    probeTier0Isolation?: unknown;
+  };
+  if (typeof mod.createTier0Runner === "function") {
+    return {
+      factory: mod.createTier0Runner as Tier0RunnerFactory,
+      ...(typeof mod.probeTier0Isolation === "function"
+        ? { probe: mod.probeTier0Isolation as Tier0IsolationProbe }
+        : {}),
+    };
+  }
+  if (typeof mod.runTier0 !== "function") {
+    throw new FacetError(
+      "internal",
+      `Tier 0 runner module '${path}' did not export a runTier0 function`,
+      { retryable: false },
+    );
+  }
+  const runner = mod.runTier0 as Tier0Runner;
+  return { factory: () => runner };
+}
+
+async function loadTier1Runner(path: string): Promise<LoadedTier1Runner> {
+  const dynamicPath = path;
+  const mod = (await import(dynamicPath)) as {
+    createTier1Runner?: unknown;
+    probeTier1Availability?: unknown;
+  };
+  if (typeof mod.createTier1Runner !== "function") {
+    throw new FacetError(
+      "internal",
+      `Tier 1 runner module '${path}' did not export a createTier1Runner function`,
+      { retryable: false },
+    );
+  }
+  return {
+    factory: mod.createTier1Runner as Tier1RunnerFactory,
+    ...(typeof mod.probeTier1Availability === "function"
+      ? { probe: mod.probeTier1Availability as Tier1AvailabilityProbe }
+      : {}),
+  };
+}
+
+async function loadRequiredTier0Runner(path: string | undefined): Promise<LoadedTier0Runner> {
+  if (path === undefined) {
+    throw new FacetError(
+      "internal",
+      "Tier 0 runner path is required (pass --tier0-runner-path <module>)",
+      { retryable: false },
+    );
+  }
+  return loadTier0Runner(path);
+}
+
+function requireTier1RunnerPath(path: string | undefined): string {
+  if (path === undefined) {
+    throw new FacetError(
+      "internal",
+      "Tier 1 runner path is required (pass --tier1-runner-path <module>)",
+      { retryable: false },
+    );
+  }
+  return path;
+}
+
+async function loadModules(args: ParsedServiceArgs): Promise<ServiceRunnerModules> {
+  const tier1RunnerPath = requireTier1RunnerPath(args.tier1RunnerPath);
+  return {
+    tier0: await loadRequiredTier0Runner(args.tier0RunnerPath),
+    loadTier1: () => loadTier1Runner(tier1RunnerPath),
+  };
+}
+
+async function main(): Promise<void> {
+  process.exitCode = await runServiceProcess(process.argv.slice(2), process.env, loadModules);
+}
+
+if (import.meta.main) {
+  void main();
+}