@volter/twin-fly 0.1.0

package/src/fly-docker.ts

@@ -0,0 +1,539 @@
+// The REAL execution plane: `FlyContainerRuntime` implemented over the LOCAL Docker daemon via
+// the `docker` CLI. This is what makes the fly twin a REAL-PLANE twin (the supabase doctrine at
+// the compute layer): the control plane is twinned, but a created/started machine genuinely runs
+// `config.image` as a local container — config.env (plus Fly's documented FLY_* runtime
+// environment) becomes the container env, every service's internal_port is published on an
+// ephemeral LOOPBACK port, config.mounts become named docker volumes, stop is `docker stop`,
+// start RECREATES the container from the image (Fly resets a stopped Machine's rootfs — only
+// mounted volumes survive), destroy is `docker rm -f`, exec is `docker exec`.
+//
+// This module makes NO vendor network calls (D4 is about the vendor; the Docker daemon is local
+// infrastructure, the same class as supabase's real Postgres) and is NEVER exercised by
+// capability verifies or the gate: the handler defaults to the pure-ledger VIRTUAL runtime, and
+// this file's only proof is fly-docker.integration.test.ts, which self-skips LOUDLY when
+// `docker info` fails. `dockerAvailable()` probes the daemon (not just the binary), the
+// cookbook/saas-ai-supabase/run.ts pattern.
+//
+// Naming: every artifact is namespaced `fly-twin-…` so a crashed run can be swept with
+// `docker ps -aq --filter label=dev.volter.fly-twin | xargs docker rm -f`.
+import { spawn, spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, statfsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { withFileLock } from '@volter/world-core';
+import type { FlyContainerRuntime, FlyContainerSpec, FlyExecResult } from './fly-machines.ts';
+
+export const FLY_DOCKER_LABEL = 'dev.volter.fly-twin';
+const FLY_WORLD_OWNER_LABEL = 'dev.volter.world-owner';
+const FLY_MEMORY_LABEL = 'dev.volter.memory-mib';
+const FLY_CPU_LABEL = 'dev.volter.cpus';
+const MIB = 1024 * 1024;
+
+/** Writable storage a machine needs for itself before its own memory-sized footprint counts: an
+ *  image pull plus a writable layer. A property of the substrate, not a knob. */
+const IMAGE_FOOTPRINT_MIB = 2048;
+
+/** MiB of writable storage admission keeps FREE beyond the machine's own need — the DEFAULT of a
+ *  configured mechanism value, not a constant. A World whose host disk is smaller than the
+ *  reserve configures it down (`world-fly serve --storage-reserve-mib N`,
+ *  `FlyDockerRuntimeOptions.storageReserveMiB`); the runtime never lowers its own floor. */
+export const DEFAULT_STORAGE_RESERVE_MIB = 2048;
+
+/** Options of the REAL execution plane. Configuration reaches this runtime the way the plane
+ *  itself does — resolved at the HOST BOUNDARY (cli.ts via fly-runtime-choice.ts) and handed in,
+ *  never read from the process here. */
+export interface FlyDockerRuntimeOptions {
+  /** MiB kept free beyond a machine's own writable-storage need (default 2048). */
+  storageReserveMiB?: number;
+}
+
+function docker(args: string[], opts: { timeoutMs?: number } = {}): { status: number; stdout: string; stderr: string } {
+  const r = spawnSync('docker', args, { encoding: 'utf8', timeout: opts.timeoutMs ?? 120_000 });
+  if (r.error) throw new Error(`docker ${args[0]}: ${r.error.message}`);
+  return { status: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+}
+
+/**
+ * The writable-storage floor for ONE machine: its own need (never below an image pull plus a
+ * writable layer) plus the reserve the host keeps free. Pure arithmetic, so the admission rule is
+ * checkable without a daemon.
+ */
+export function writableStorageFloorMiB(
+  memoryMiB: number,
+  reserveMiB: number = DEFAULT_STORAGE_RESERVE_MIB,
+): { requiredMiB: number; reserveMiB: number; floorMiB: number } {
+  const requiredMiB = Math.max(IMAGE_FOOTPRINT_MIB, memoryMiB);
+  return { requiredMiB, reserveMiB, floorMiB: requiredMiB + reserveMiB };
+}
+
+/** What the writable-storage probe measured, and the SUBSTRATE it measured it on (named in the
+ *  capacity log so a refusal is diagnosable; the caller's error stays in World vocabulary). */
+export type WritableStorage = { availableMiB: number; substrate: string };
+
+/**
+ * The host facts the writable-storage probe reads. Injected — the same seam doctrine the
+ * execution plane itself uses — so every branch (host-visible docker root, Colima, OrbStack,
+ * Docker Desktop) is provable OFFLINE against fakes, with no daemon anywhere.
+ */
+export interface StorageProbeHost {
+  readonly platform: string;
+  readonly home: string;
+  /** Does this HOST path exist? */
+  exists(path: string): boolean;
+  /** Free MiB on the host volume holding `path`; undefined when it cannot be read. */
+  freeMiB(path: string): number | undefined;
+  /** Entries of a host directory; [] when unreadable. */
+  entries(path: string): string[];
+  docker(args: string[]): { status: number; stdout: string; stderr: string };
+  run(command: string, args: string[], timeoutMs: number): { status: number | null; stdout: string; stderr: string };
+  warn(line: string): void;
+}
+
+const HOST_STORAGE_PROBE: StorageProbeHost = {
+  platform: process.platform,
+  home: homedir(),
+  exists: (path) => existsSync(path),
+  freeMiB(path) {
+    try {
+      const fs = statfsSync(path);
+      return Math.floor((Number(fs.bavail) * Number(fs.bsize)) / MIB);
+    } catch {
+      return undefined;
+    }
+  },
+  entries(path) {
+    try {
+      return readdirSync(path);
+    } catch {
+      return [];
+    }
+  },
+  docker: (args) => docker(args),
+  run(command, args, timeoutMs) {
+    const r = spawnSync(command, args, { encoding: 'utf8', timeout: timeoutMs });
+    return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+  },
+  warn: (line) => { process.stderr.write(line); },
+};
+
+/**
+ * How much writable storage the local execution substrate can still take, measured INSIDE that
+ * substrate when the docker root is not host-visible. Honest by construction: every branch either
+ * MEASURES a real filesystem or returns undefined (admission then refuses) — capacity is never
+ * assumed, defaulted or fabricated.
+ */
+export function probeWritableStorage(rootDir: string, host: StorageProbeHost = HOST_STORAGE_PROBE): WritableStorage | undefined {
+  if (rootDir && host.exists(rootDir)) {
+    const availableMiB = host.freeMiB(rootDir);
+    if (availableMiB === undefined) {
+      host.warn(`[capacity] writable-storage verification failed: docker root "${rootDir}" could not be measured\n`);
+      return undefined;
+    }
+    return { availableMiB, substrate: `docker root "${rootDir}"` };
+  }
+  // The daemon keeps its data in a hidden Linux machine. Detect WHICH one from the active
+  // endpoint (and, for the substrates that also install the classic socket path, from the daemon's
+  // own identity), then measure that machine's storage; this detail never crosses the service log.
+  const context = host.docker(['context', 'inspect', '--format', '{{json .Endpoints.docker.Host}}']);
+  let endpoint = '';
+  try { endpoint = context.status === 0 ? JSON.parse(context.stdout.trim() || '""') as string : ''; } catch {
+    host.warn('[capacity] writable-storage verification failed: the active docker context endpoint could not be read\n');
+    return undefined;
+  }
+  // `~/.colima/<profile>/docker.sock` is colima's own documented DOCKER_HOST; a colima install
+  // that predates the dotted directory uses `.../colima/<profile>/docker.sock`. Both are the same
+  // substrate, so the leading dot is optional here — anchored to a path segment so no unrelated
+  // directory ending in "colima" can claim the branch.
+  const colima = /(?:^|\/)\.?colima\/([^/]+)\/docker\.sock$/u.exec(endpoint);
+  if (colima) return colimaStorage(colima[1]!, rootDir, host);
+  // OrbStack is asked about BEFORE Docker Desktop: both keep a growable image under the user's
+  // Library and both may be installed at once, so the ACTIVE daemon — not whichever directory
+  // happens to exist — decides which image is the one being written to.
+  if (host.platform === 'darwin' && isOrbStack(endpoint, host)) return orbstackStorage(host);
+  // Docker Desktop keeps its Linux machine's disk as a growable image under the user's
+  // Library container; free space on the volume HOLDING that image is the binding
+  // constraint on how much more the daemon can write, and it is host-visible — no
+  // container run, no network. (The VM's internal ceiling is typically far larger.)
+  const desktopData = join(host.home, 'Library', 'Containers', 'com.docker.docker', 'Data');
+  if (host.platform === 'darwin' && host.exists(desktopData)) {
+    const availableMiB = host.freeMiB(desktopData);
+    if (availableMiB === undefined) {
+      host.warn(`[capacity] writable-storage verification failed: the Docker Desktop data volume "${desktopData}" could not be measured\n`);
+      return undefined;
+    }
+    return { availableMiB, substrate: `Docker Desktop data volume "${desktopData}"` };
+  }
+  host.warn(`[capacity] writable-storage verification failed: docker root "${rootDir}" is not host-visible and endpoint "${endpoint}" is neither a Colima profile, OrbStack, nor Docker Desktop on macOS\n`);
+  return undefined;
+}
+
+/** Is the ACTIVE daemon OrbStack? Its own socket path is the cheap tell; OrbStack also takes over
+ *  the classic `/var/run/docker.sock`, so an endpoint that says nothing is settled by asking the
+ *  daemon what it is — `docker info` reports the operating system literally as "OrbStack". */
+function isOrbStack(endpoint: string, host: StorageProbeHost): boolean {
+  if (/\/\.orbstack\/run\/docker\.sock$/u.test(endpoint)) return true;
+  const info = host.docker(['info', '--format', '{{.OperatingSystem}}']);
+  return info.status === 0 && /^orbstack$/iu.test(info.stdout.trim());
+}
+
+/** OrbStack keeps the WHOLE Linux machine — images, containers, volumes — in one SPARSE
+ *  `data.img.raw` inside its group container ("it only takes as much space as you use, and
+ *  automatically shrinks when you delete data", per the file's own README), with no disk ceiling
+ *  configured by default. So free space on the host volume HOLDING that image is the binding
+ *  constraint on how much more the daemon can write, and it is host-visible — no container run,
+ *  no ssh, no network. Same shape as the Docker Desktop branch. The group container carries
+ *  Apple's team prefix, so it is FOUND rather than hard-coded; when it cannot be found the probe
+ *  refuses instead of guessing a volume. */
+function orbstackStorage(host: StorageProbeHost): WritableStorage | undefined {
+  const groupContainers = join(host.home, 'Library', 'Group Containers');
+  const container = host.entries(groupContainers).filter((entry) => entry.endsWith('.dev.orbstack')).sort()[0];
+  const dataDir = container === undefined ? undefined : join(groupContainers, container, 'data');
+  if (dataDir === undefined || !host.exists(dataDir)) {
+    host.warn(`[capacity] writable-storage verification failed: the active daemon is OrbStack but its data image was not found under "${groupContainers}"\n`);
+    return undefined;
+  }
+  const availableMiB = host.freeMiB(dataDir);
+  if (availableMiB === undefined) {
+    host.warn(`[capacity] writable-storage verification failed: the OrbStack data volume "${dataDir}" could not be measured\n`);
+    return undefined;
+  }
+  return { availableMiB, substrate: `OrbStack data volume "${dataDir}"` };
+}
+
+/** Colima keeps the daemon in a hidden Linux machine reachable over its own ssh config; query
+ *  that filesystem internally. */
+function colimaStorage(profile: string, rootDir: string, host: StorageProbeHost): WritableStorage | undefined {
+  const config = host.run('colima', ['-p', profile, 'ssh-config'], 5_000);
+  if (config.status !== 0) {
+    host.warn(`[capacity] writable-storage verification failed: colima ssh-config for profile "${profile}" exited ${String(config.status)}\n`);
+    return undefined;
+  }
+  const field = (name: string): string | undefined => {
+    const value = new RegExp(`^\\s*${name}\\s+(.+?)\\s*$`, 'mu').exec(config.stdout)?.[1];
+    return value?.replace(/^"|"$/gu, '');
+  };
+  const identity = field('IdentityFile');
+  const user = field('User');
+  const hostname = field('Hostname');
+  const port = field('Port');
+  const controlPath = field('ControlPath');
+  if (!identity || !user || !hostname || !port) {
+    host.warn(`[capacity] writable-storage verification failed: colima ssh-config for profile "${profile}" is missing connection fields\n`);
+    return undefined;
+  }
+  const status = host.run('ssh', [
+    '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'BatchMode=yes',
+    '-o', 'IdentitiesOnly=yes', ...(controlPath ? ['-o', 'ControlMaster=auto', '-o', `ControlPath=${controlPath}`, '-o', 'ControlPersist=yes'] : []),
+    '-i', identity, '-p', port, `${user}@${hostname}`, 'df', '-Pk', rootDir || '/',
+  ], 5_000);
+  if (status.status !== 0) {
+    host.warn(`[capacity] internal writable-storage inspection failed: ${(status.stderr || '').trim()}\n`);
+    return undefined;
+  }
+  const line = status.stdout.trim().split(/\r?\n/u).at(-1) ?? '';
+  const fields = line.trim().split(/\s+/u);
+  const availableKiB = Number(fields[3]);
+  if (!Number.isFinite(availableKiB)) {
+    host.warn(`[capacity] writable-storage verification failed: colima profile "${profile}" reported an unreadable df line "${line}"\n`);
+    return undefined;
+  }
+  return { availableMiB: Math.floor(availableKiB / 1024), substrate: `colima profile "${profile}"` };
+}
+
+/** Is a Docker DAEMON actually reachable (not just the binary present)? */
+export function dockerAvailable(): boolean {
+  try {
+    const r = spawnSync('docker', ['info'], { stdio: 'ignore', timeout: 15_000 });
+    return r.status === 0;
+  } catch {
+    return false;
+  }
+}
+
+export function flyContainerName(machineId: string): string {
+  return `fly-twin-${machineId}`;
+}
+export function flyVolumeName(volumeId: string): string {
+  return `fly-twin-${volumeId}`;
+}
+
+export class FlyDockerRuntime implements FlyContainerRuntime {
+  readonly kind = 'docker';
+  readonly #ownerId: string;
+  readonly #logFollowers = new Map<string, ReturnType<typeof spawn>>();
+  readonly #machineIds = new Map<string, string>();
+  readonly #capacityLock: string;
+  readonly #storageReserveMiB: number;
+
+  constructor(root = process.cwd(), options: FlyDockerRuntimeOptions = {}) {
+    // A malformed reserve FAILS here rather than silently reverting to the default: a World that
+    // asked for a different admission floor and got the old one would be told nothing.
+    const reserve = options.storageReserveMiB;
+    if (reserve !== undefined && (!Number.isInteger(reserve) || reserve < 0)) {
+      throw new Error(`machine provider: storage reserve must be a non-negative whole number of MiB (got ${JSON.stringify(reserve)})`);
+    }
+    this.#storageReserveMiB = reserve ?? DEFAULT_STORAGE_RESERVE_MIB;
+    // The service data-root is stable for the lifetime of one named World and
+    // opaque outside this implementation. Hashing it gives every backing
+    // resource an exact lifecycle owner without exposing host paths.
+    this.#ownerId = createHash('sha256').update(resolve(root)).digest('hex').slice(0, 32);
+    // Subordinate compute from Worlds rooted in different repositories still shares one machine,
+    // so admission uses one user-machine lock rather than a per-checkout lock.
+    this.#capacityLock = join(homedir(), '.volter', 'world-resource-claims', 'subordinate-compute.lock');
+    mkdirSync(join(homedir(), '.volter', 'world-resource-claims'), { recursive: true, mode: 0o700 });
+  }
+
+  async run(spec: FlyContainerSpec): Promise<{ containerRef: string; publishedPorts?: Array<{ internal: number; host: number }> }> {
+    const containerRef = withFileLock(this.#capacityLock, () => {
+      this.#admit(spec);
+      const name = flyContainerName(spec.machineId);
+      const args = [
+        'run', '-d', '--name', name,
+        '--memory', `${spec.memoryMiB}m`,
+        '--memory-swap', `${spec.memoryMiB}m`,
+        '--cpus', String(spec.cpus),
+        '--label', `${FLY_DOCKER_LABEL}=${spec.appName}`,
+        '--label', `${FLY_WORLD_OWNER_LABEL}=${this.#ownerId}`,
+        '--label', `${FLY_MEMORY_LABEL}=${spec.memoryMiB}`,
+        '--label', `${FLY_CPU_LABEL}=${spec.cpus}`,
+      ];
+      for (const [k, v] of Object.entries(spec.env)) args.push('-e', `${k}=${v}`);
+      // Publish every internal port on an EPHEMERAL loopback port (fixed host ports would collide
+      // across machines); the assigned ports are read back below and ledgered as twin_local_ports.
+      for (const p of spec.ports) args.push('-p', `127.0.0.1::${p.internal}`);
+      for (const m of spec.mounts) args.push('-v', `${flyVolumeName(m.volumeId)}:${m.path}`);
+      if (spec.entrypoint && spec.entrypoint.length > 0) args.push('--entrypoint', spec.entrypoint[0]!);
+      args.push(spec.image);
+      if (spec.entrypoint && spec.entrypoint.length > 1) args.push(...spec.entrypoint.slice(1));
+      if (spec.cmd && spec.cmd.length > 0) args.push(...spec.cmd);
+      const r = docker(args);
+      if (r.status !== 0) {
+        const detail = r.stderr.trim() || `launch command exited ${r.status}`;
+        process.stderr.write(`[machine ${spec.machineId}] internal launch failure: ${detail}\n`);
+        const reason = /no space left on device|enospc/iu.test(detail) ? 'insufficient writable storage'
+          : /out of memory|cannot allocate memory|killed/iu.test(detail) ? 'insufficient memory'
+            : /cannot connect|not running|daemon/iu.test(detail) ? 'local execution capacity is unavailable'
+              : 'local execution failed';
+        throw new Error(`${this.#context(spec)}: ${reason}. Log: ${this.#logPath()}`);
+      }
+      return r.stdout.trim();
+    });
+    this.#machineIds.set(containerRef, spec.machineId);
+    this.#followMachineLogs(containerRef, spec.machineId);
+    const publishedPorts: Array<{ internal: number; host: number }> = [];
+    for (const p of spec.ports) {
+      const port = docker(['port', containerRef, String(p.internal)]);
+      // "127.0.0.1:55007" (possibly several lines for v4/v6) — take the first v4 mapping.
+      const match = port.stdout.split('\n').map((l) => l.trim()).find((l) => l.startsWith('127.0.0.1:'));
+      if (match) publishedPorts.push({ internal: p.internal, host: Number(match.split(':').pop()) });
+    }
+    return { containerRef, publishedPorts };
+  }
+
+  /** Admit and reserve by the same limits that `run` enforces. The lock around this method and
+   * resource creation prevents two concurrent machine requests from both seeing the same final
+   * capacity. Storage is measured inside the local execution substrate when it is discoverable;
+   * callers still receive only the declared World/service/resource vocabulary. */
+  #admit(spec: FlyContainerSpec): void {
+    const info = docker(['info', '--format', '{{.MemTotal}} {{.NCPU}} {{.DockerRootDir}}']);
+    if (info.status !== 0) {
+      process.stderr.write(`[machine ${spec.machineId}] capacity inspection failed: ${info.stderr.trim()}\n`);
+      throw new Error(`${this.#context(spec)}: local execution capacity is unavailable. Log: ${this.#logPath()}`);
+    }
+    const [memoryRaw, cpuRaw, rootDir = ''] = info.stdout.trim().split(/\s+/u);
+    const totalMemoryMiB = Math.floor(Number(memoryRaw) / MIB);
+    const totalCpus = Number(cpuRaw);
+    if (!Number.isFinite(totalMemoryMiB) || totalMemoryMiB <= 0 || !Number.isFinite(totalCpus) || totalCpus <= 0) {
+      process.stderr.write(`[machine ${spec.machineId}] invalid capacity report: ${info.stdout.trim()}\n`);
+      throw new Error(`${this.#context(spec)}: local execution capacity could not be verified. Log: ${this.#logPath()}`);
+    }
+    const refs = docker(['ps', '-aq', '--filter', `label=${FLY_DOCKER_LABEL}`]);
+    if (refs.status !== 0) {
+      process.stderr.write(`[machine ${spec.machineId}] reservation inspection failed: ${refs.stderr.trim()}\n`);
+      throw new Error(`${this.#context(spec)}: local execution capacity could not be verified. Log: ${this.#logPath()}`);
+    }
+    let reservedMemoryMiB = 0;
+    let reservedCpus = 0;
+    for (const ref of refs.stdout.split(/\s+/u).filter(Boolean)) {
+      const reservation = docker(['inspect', '--format',
+        `{{index .Config.Labels "${FLY_MEMORY_LABEL}"}}|{{index .Config.Labels "${FLY_CPU_LABEL}"}}|{{.HostConfig.Memory}}|{{.HostConfig.NanoCpus}}`, ref]);
+      if (reservation.status !== 0) {
+        process.stderr.write(`[machine ${spec.machineId}] existing reservation inspection failed for ${ref.slice(0, 12)}: ${reservation.stderr.trim()}\n`);
+        throw new Error(`${this.#context(spec)}: resource admission refused because existing local execution capacity could not be verified. Log: ${this.#logPath()}`);
+      }
+      const [memoryLabel, cpuLabel, enforcedMemoryBytes, enforcedNanoCpus] = reservation.stdout.trim().split('|');
+      const memoryMiB = Number(memoryLabel) || Math.ceil(Number(enforcedMemoryBytes) / MIB);
+      const cpus = Number(cpuLabel) || (Number(enforcedNanoCpus) / 1_000_000_000);
+      if (!Number.isFinite(memoryMiB) || memoryMiB <= 0 || !Number.isFinite(cpus) || cpus <= 0) {
+        process.stderr.write(`[machine ${spec.machineId}] existing execution resource ${ref.slice(0, 12)} has no enforceable memory/CPU reservation\n`);
+        throw new Error(`${this.#context(spec)}: resource admission refused because an existing local execution resource has undeclared capacity. Stop its owning World, then retry. Log: ${this.#logPath()}`);
+      }
+      reservedMemoryMiB += memoryMiB;
+      reservedCpus += cpus;
+    }
+    const memorySafetyMiB = Math.max(256, Math.min(1024, Math.floor(totalMemoryMiB * 0.1)));
+    const availableMemoryMiB = Math.max(0, totalMemoryMiB - memorySafetyMiB - reservedMemoryMiB);
+    const availableCpus = Math.max(0, totalCpus - reservedCpus);
+    if (spec.memoryMiB > availableMemoryMiB || spec.cpus > availableCpus) {
+      const details = [
+        ...(spec.memoryMiB > availableMemoryMiB ? [`memory requires ${spec.memoryMiB} MiB, ${availableMemoryMiB} MiB available`] : []),
+        ...(spec.cpus > availableCpus ? [`CPU requires ${spec.cpus}, ${availableCpus} available`] : []),
+      ];
+      throw new Error(`${this.#context(spec)}: resource admission refused (${details.join('; ')}). Log: ${this.#logPath()}`);
+    }
+
+    const storage = probeWritableStorage(rootDir);
+    const { requiredMiB, reserveMiB, floorMiB } = writableStorageFloorMiB(spec.memoryMiB, this.#storageReserveMiB);
+    if (storage === undefined) {
+      throw new Error(`${this.#context(spec)}: resource admission refused because writable-storage capacity could not be verified. Log: ${this.#logPath()}`);
+    }
+    if (floorMiB > storage.availableMiB) {
+      // The substrate and the measured number belong in the capacity log (a refusal has to be
+      // diagnosable); the caller's error stays in World/service/resource vocabulary.
+      process.stderr.write(`[capacity] writable-storage refused: ${storage.substrate} reports ${storage.availableMiB} MiB available, ${floorMiB} MiB needed (${requiredMiB} MiB machine + ${reserveMiB} MiB reserve)\n`);
+      throw new Error(`${this.#context(spec)}: resource admission refused (writable storage requires ${requiredMiB} MiB plus ${reserveMiB} MiB safety, ${storage.availableMiB} MiB available). Log: ${this.#logPath()}`);
+    }
+  }
+
+  #context(spec: FlyContainerSpec): string {
+    const world = process.env.VOLTER_WORLD_NAME ?? 'standalone';
+    const service = process.env.VOLTER_WORLD_SERVICE_ID ?? 'machine-provider';
+    return `World "${world}": service "${service}": machine "${spec.machineId}"`;
+  }
+
+  #logPath(): string {
+    return process.env.VOLTER_WORLD_SERVICE_LOG ?? '(service stderr)';
+  }
+
+  async stop(containerRef: string, opts: { signal?: string; timeoutSeconds?: number } = {}): Promise<void> {
+    const args = ['stop'];
+    if (opts.signal) args.push('--signal', opts.signal);
+    if (opts.timeoutSeconds !== undefined) args.push('--time', String(opts.timeoutSeconds));
+    args.push(containerRef);
+    const r = docker(args);
+    if (r.status !== 0) throw new Error(r.stderr.trim() || `docker stop exited ${r.status}`);
+  }
+
+  /**
+   * Start a STOPPED machine — by RECREATING the container, never `docker start`.
+   *
+   * Real Fly resets the root filesystem on stop→start: "Stopped Machines that are restarted are
+   * completely reset to their original state so that they start clean on the next run"
+   * (fly.io/docs/machines/api/machines-resource). `docker start` resumes the old writable layer,
+   * which would make this twin MORE FORGIVING than the vendor at exactly the point where the
+   * difference bites: an app that writes state outside a volume would pass local rehearsal and
+   * lose that state on the first real stop/start cycle.
+   *
+   * So the old container is removed and a fresh one is run from `spec.image` with the same
+   * env/ports/mounts wiring. `docker rm -v` removes only ANONYMOUS volumes, so the named
+   * `fly-twin-<volume id>` volumes (this twin's mapping of Fly volumes) carry over — mounted
+   * data survives, everything else starts clean, exactly like the vendor. The new container gets
+   * a new handle and new ephemeral loopback ports; both are returned and re-ledgered.
+   */
+  async start(containerRef: string, spec: FlyContainerSpec): Promise<{ containerRef: string; publishedPorts?: Array<{ internal: number; host: number }> }> {
+    // A ref that is already gone is not a failure: the fresh run below IS the reset.
+    try { await this.remove(containerRef); } catch { /* already reaped */ }
+    return this.run(spec);
+  }
+
+  async remove(containerRef: string): Promise<void> {
+    const r = docker(['rm', '-f', '-v', containerRef]);
+    if (r.status !== 0) throw new Error(r.stderr.trim() || `docker rm exited ${r.status}`);
+    this.#logFollowers.get(containerRef)?.kill('SIGTERM');
+    this.#logFollowers.delete(containerRef);
+    this.#machineIds.delete(containerRef);
+  }
+
+  async pause(containerRef: string): Promise<void> {
+    const r = docker(['pause', containerRef]);
+    if (r.status !== 0) throw new Error(r.stderr.trim() || `docker pause exited ${r.status}`);
+  }
+
+  async unpause(containerRef: string): Promise<void> {
+    const r = docker(['unpause', containerRef]);
+    if (r.status !== 0) throw new Error(r.stderr.trim() || `docker unpause exited ${r.status}`);
+  }
+
+  async signal(containerRef: string, signal: string): Promise<void> {
+    const r = docker(['kill', '--signal', signal, containerRef]);
+    if (r.status !== 0) throw new Error(r.stderr.trim() || `docker kill exited ${r.status}`);
+  }
+
+  async exec(containerRef: string, cmd: string[], opts: { timeoutSeconds?: number } = {}): Promise<FlyExecResult> {
+    const r = docker(['exec', containerRef, ...cmd], { timeoutMs: (opts.timeoutSeconds ?? 60) * 1000 });
+    return { exitCode: r.status, stdout: r.stdout, stderr: r.stderr };
+  }
+
+  async inspect(containerRef: string): Promise<{ running: boolean; exitCode?: number; oomKilled?: boolean }> {
+    const r = docker(['inspect', '--format', '{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}', containerRef]);
+    if (r.status !== 0) return { running: false }; // container gone entirely
+    const [running, exitCode, oomKilled] = r.stdout.trim().split(/\s+/);
+    return {
+      running: running === 'true',
+      ...(exitCode !== undefined ? { exitCode: Number(exitCode) } : {}),
+      ...(oomKilled !== undefined ? { oomKilled: oomKilled === 'true' } : {}),
+    };
+  }
+
+  async createVolume(volumeId: string): Promise<void> {
+    const r = docker([
+      'volume', 'create',
+      '--label', FLY_DOCKER_LABEL,
+      '--label', `${FLY_WORLD_OWNER_LABEL}=${this.#ownerId}`,
+      flyVolumeName(volumeId),
+    ]);
+    if (r.status !== 0) throw new Error(r.stderr.trim() || `docker volume create exited ${r.status}`);
+  }
+
+  async removeVolume(volumeId: string): Promise<void> {
+    const r = docker(['volume', 'rm', '-f', flyVolumeName(volumeId)]);
+    if (r.status !== 0) throw new Error(r.stderr.trim() || `docker volume rm exited ${r.status}`);
+  }
+
+  async cleanup(): Promise<void> {
+    const selector = `label=${FLY_WORLD_OWNER_LABEL}=${this.#ownerId}`;
+    const containers = docker(['ps', '-aq', '--filter', selector]);
+    if (containers.status !== 0) throw new Error(containers.stderr.trim() || 'could not enumerate owned compute resources');
+    const containerRefs = containers.stdout.split(/\s+/u).filter(Boolean);
+    if (containerRefs.length > 0) {
+      const removed = docker(['rm', '-f', '-v', ...containerRefs]);
+      if (removed.status !== 0) throw new Error(removed.stderr.trim() || 'could not reclaim owned compute resources');
+    }
+
+    const volumes = docker(['volume', 'ls', '-q', '--filter', selector]);
+    if (volumes.status !== 0) throw new Error(volumes.stderr.trim() || 'could not enumerate owned persistent resources');
+    const volumeRefs = volumes.stdout.split(/\s+/u).filter(Boolean);
+    if (volumeRefs.length > 0) {
+      const removed = docker(['volume', 'rm', '-f', ...volumeRefs]);
+      if (removed.status !== 0) throw new Error(removed.stderr.trim() || 'could not reclaim owned persistent resources');
+    }
+    for (const follower of this.#logFollowers.values()) follower.kill('SIGTERM');
+    this.#logFollowers.clear();
+    this.#machineIds.clear();
+  }
+
+  /** Forward each owned Machine's output into the provider service's stdout/stderr. The World
+   * already retains that declared service log, so diagnostics remain at the World boundary and
+   * never require callers to identify or inspect this runtime's backing substrate. */
+  #followMachineLogs(containerRef: string, machineId: string): void {
+    if (this.#logFollowers.has(containerRef)) return;
+    const follower = spawn('docker', ['logs', '--follow', '--timestamps', containerRef], {
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    this.#logFollowers.set(containerRef, follower);
+    const write = (stream: NodeJS.WriteStream, chunk: Buffer): void => {
+      const prefix = `[machine ${machineId}] `;
+      const text = chunk.toString('utf8').replace(/\n(?=.)/gu, `\n${prefix}`);
+      stream.write(`${prefix}${text}`);
+    };
+    follower.stdout?.on('data', (chunk: Buffer) => write(process.stdout, chunk));
+    follower.stderr?.on('data', (chunk: Buffer) => write(process.stderr, chunk));
+    follower.once('error', (error) => {
+      process.stderr.write(`[machine ${machineId}] log stream unavailable: ${error.message}\n`);
+    });
+    follower.once('close', () => {
+      if (this.#logFollowers.get(containerRef) === follower) this.#logFollowers.delete(containerRef);
+    });
+  }
+}