@volter/twin-fly 0.1.0

package/dist/src/fly-docker.js

@@ -0,0 +1,509 @@
+// The REAL execution plane: `FlyContainerRuntime` implemented over the LOCAL Docker daemon via
+// the `docker` CLI. This is what makes the fly twin a REAL-PLANE twin (the supabase doctrine at
+// the compute layer): the control plane is twinned, but a created/started machine genuinely runs
+// `config.image` as a local container — config.env (plus Fly's documented FLY_* runtime
+// environment) becomes the container env, every service's internal_port is published on an
+// ephemeral LOOPBACK port, config.mounts become named docker volumes, stop is `docker stop`,
+// start RECREATES the container from the image (Fly resets a stopped Machine's rootfs — only
+// mounted volumes survive), destroy is `docker rm -f`, exec is `docker exec`.
+//
+// This module makes NO vendor network calls (D4 is about the vendor; the Docker daemon is local
+// infrastructure, the same class as supabase's real Postgres) and is NEVER exercised by
+// capability verifies or the gate: the handler defaults to the pure-ledger VIRTUAL runtime, and
+// this file's only proof is fly-docker.integration.test.ts, which self-skips LOUDLY when
+// `docker info` fails. `dockerAvailable()` probes the daemon (not just the binary), the
+// cookbook/saas-ai-supabase/run.ts pattern.
+//
+// Naming: every artifact is namespaced `fly-twin-…` so a crashed run can be swept with
+// `docker ps -aq --filter label=dev.volter.fly-twin | xargs docker rm -f`.
+import { spawn, spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, statfsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { withFileLock } from '@volter/world-core';
+export const FLY_DOCKER_LABEL = 'dev.volter.fly-twin';
+const FLY_WORLD_OWNER_LABEL = 'dev.volter.world-owner';
+const FLY_MEMORY_LABEL = 'dev.volter.memory-mib';
+const FLY_CPU_LABEL = 'dev.volter.cpus';
+const MIB = 1024 * 1024;
+/** Writable storage a machine needs for itself before its own memory-sized footprint counts: an
+ *  image pull plus a writable layer. A property of the substrate, not a knob. */
+const IMAGE_FOOTPRINT_MIB = 2048;
+/** MiB of writable storage admission keeps FREE beyond the machine's own need — the DEFAULT of a
+ *  configured mechanism value, not a constant. A World whose host disk is smaller than the
+ *  reserve configures it down (`world-fly serve --storage-reserve-mib N`,
+ *  `FlyDockerRuntimeOptions.storageReserveMiB`); the runtime never lowers its own floor. */
+export const DEFAULT_STORAGE_RESERVE_MIB = 2048;
+function docker(args, opts = {}) {
+    const r = spawnSync('docker', args, { encoding: 'utf8', timeout: opts.timeoutMs ?? 120_000 });
+    if (r.error)
+        throw new Error(`docker ${args[0]}: ${r.error.message}`);
+    return { status: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+}
+/**
+ * The writable-storage floor for ONE machine: its own need (never below an image pull plus a
+ * writable layer) plus the reserve the host keeps free. Pure arithmetic, so the admission rule is
+ * checkable without a daemon.
+ */
+export function writableStorageFloorMiB(memoryMiB, reserveMiB = DEFAULT_STORAGE_RESERVE_MIB) {
+    const requiredMiB = Math.max(IMAGE_FOOTPRINT_MIB, memoryMiB);
+    return { requiredMiB, reserveMiB, floorMiB: requiredMiB + reserveMiB };
+}
+const HOST_STORAGE_PROBE = {
+    platform: process.platform,
+    home: homedir(),
+    exists: (path) => existsSync(path),
+    freeMiB(path) {
+        try {
+            const fs = statfsSync(path);
+            return Math.floor((Number(fs.bavail) * Number(fs.bsize)) / MIB);
+        }
+        catch {
+            return undefined;
+        }
+    },
+    entries(path) {
+        try {
+            return readdirSync(path);
+        }
+        catch {
+            return [];
+        }
+    },
+    docker: (args) => docker(args),
+    run(command, args, timeoutMs) {
+        const r = spawnSync(command, args, { encoding: 'utf8', timeout: timeoutMs });
+        return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+    },
+    warn: (line) => { process.stderr.write(line); },
+};
+/**
+ * How much writable storage the local execution substrate can still take, measured INSIDE that
+ * substrate when the docker root is not host-visible. Honest by construction: every branch either
+ * MEASURES a real filesystem or returns undefined (admission then refuses) — capacity is never
+ * assumed, defaulted or fabricated.
+ */
+export function probeWritableStorage(rootDir, host = HOST_STORAGE_PROBE) {
+    if (rootDir && host.exists(rootDir)) {
+        const availableMiB = host.freeMiB(rootDir);
+        if (availableMiB === undefined) {
+            host.warn(`[capacity] writable-storage verification failed: docker root "${rootDir}" could not be measured\n`);
+            return undefined;
+        }
+        return { availableMiB, substrate: `docker root "${rootDir}"` };
+    }
+    // The daemon keeps its data in a hidden Linux machine. Detect WHICH one from the active
+    // endpoint (and, for the substrates that also install the classic socket path, from the daemon's
+    // own identity), then measure that machine's storage; this detail never crosses the service log.
+    const context = host.docker(['context', 'inspect', '--format', '{{json .Endpoints.docker.Host}}']);
+    let endpoint = '';
+    try {
+        endpoint = context.status === 0 ? JSON.parse(context.stdout.trim() || '""') : '';
+    }
+    catch {
+        host.warn('[capacity] writable-storage verification failed: the active docker context endpoint could not be read\n');
+        return undefined;
+    }
+    // `~/.colima/<profile>/docker.sock` is colima's own documented DOCKER_HOST; a colima install
+    // that predates the dotted directory uses `.../colima/<profile>/docker.sock`. Both are the same
+    // substrate, so the leading dot is optional here — anchored to a path segment so no unrelated
+    // directory ending in "colima" can claim the branch.
+    const colima = /(?:^|\/)\.?colima\/([^/]+)\/docker\.sock$/u.exec(endpoint);
+    if (colima)
+        return colimaStorage(colima[1], rootDir, host);
+    // OrbStack is asked about BEFORE Docker Desktop: both keep a growable image under the user's
+    // Library and both may be installed at once, so the ACTIVE daemon — not whichever directory
+    // happens to exist — decides which image is the one being written to.
+    if (host.platform === 'darwin' && isOrbStack(endpoint, host))
+        return orbstackStorage(host);
+    // Docker Desktop keeps its Linux machine's disk as a growable image under the user's
+    // Library container; free space on the volume HOLDING that image is the binding
+    // constraint on how much more the daemon can write, and it is host-visible — no
+    // container run, no network. (The VM's internal ceiling is typically far larger.)
+    const desktopData = join(host.home, 'Library', 'Containers', 'com.docker.docker', 'Data');
+    if (host.platform === 'darwin' && host.exists(desktopData)) {
+        const availableMiB = host.freeMiB(desktopData);
+        if (availableMiB === undefined) {
+            host.warn(`[capacity] writable-storage verification failed: the Docker Desktop data volume "${desktopData}" could not be measured\n`);
+            return undefined;
+        }
+        return { availableMiB, substrate: `Docker Desktop data volume "${desktopData}"` };
+    }
+    host.warn(`[capacity] writable-storage verification failed: docker root "${rootDir}" is not host-visible and endpoint "${endpoint}" is neither a Colima profile, OrbStack, nor Docker Desktop on macOS\n`);
+    return undefined;
+}
+/** Is the ACTIVE daemon OrbStack? Its own socket path is the cheap tell; OrbStack also takes over
+ *  the classic `/var/run/docker.sock`, so an endpoint that says nothing is settled by asking the
+ *  daemon what it is — `docker info` reports the operating system literally as "OrbStack". */
+function isOrbStack(endpoint, host) {
+    if (/\/\.orbstack\/run\/docker\.sock$/u.test(endpoint))
+        return true;
+    const info = host.docker(['info', '--format', '{{.OperatingSystem}}']);
+    return info.status === 0 && /^orbstack$/iu.test(info.stdout.trim());
+}
+/** OrbStack keeps the WHOLE Linux machine — images, containers, volumes — in one SPARSE
+ *  `data.img.raw` inside its group container ("it only takes as much space as you use, and
+ *  automatically shrinks when you delete data", per the file's own README), with no disk ceiling
+ *  configured by default. So free space on the host volume HOLDING that image is the binding
+ *  constraint on how much more the daemon can write, and it is host-visible — no container run,
+ *  no ssh, no network. Same shape as the Docker Desktop branch. The group container carries
+ *  Apple's team prefix, so it is FOUND rather than hard-coded; when it cannot be found the probe
+ *  refuses instead of guessing a volume. */
+function orbstackStorage(host) {
+    const groupContainers = join(host.home, 'Library', 'Group Containers');
+    const container = host.entries(groupContainers).filter((entry) => entry.endsWith('.dev.orbstack')).sort()[0];
+    const dataDir = container === undefined ? undefined : join(groupContainers, container, 'data');
+    if (dataDir === undefined || !host.exists(dataDir)) {
+        host.warn(`[capacity] writable-storage verification failed: the active daemon is OrbStack but its data image was not found under "${groupContainers}"\n`);
+        return undefined;
+    }
+    const availableMiB = host.freeMiB(dataDir);
+    if (availableMiB === undefined) {
+        host.warn(`[capacity] writable-storage verification failed: the OrbStack data volume "${dataDir}" could not be measured\n`);
+        return undefined;
+    }
+    return { availableMiB, substrate: `OrbStack data volume "${dataDir}"` };
+}
+/** Colima keeps the daemon in a hidden Linux machine reachable over its own ssh config; query
+ *  that filesystem internally. */
+function colimaStorage(profile, rootDir, host) {
+    const config = host.run('colima', ['-p', profile, 'ssh-config'], 5_000);
+    if (config.status !== 0) {
+        host.warn(`[capacity] writable-storage verification failed: colima ssh-config for profile "${profile}" exited ${String(config.status)}\n`);
+        return undefined;
+    }
+    const field = (name) => {
+        const value = new RegExp(`^\\s*${name}\\s+(.+?)\\s*$`, 'mu').exec(config.stdout)?.[1];
+        return value?.replace(/^"|"$/gu, '');
+    };
+    const identity = field('IdentityFile');
+    const user = field('User');
+    const hostname = field('Hostname');
+    const port = field('Port');
+    const controlPath = field('ControlPath');
+    if (!identity || !user || !hostname || !port) {
+        host.warn(`[capacity] writable-storage verification failed: colima ssh-config for profile "${profile}" is missing connection fields\n`);
+        return undefined;
+    }
+    const status = host.run('ssh', [
+        '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'BatchMode=yes',
+        '-o', 'IdentitiesOnly=yes', ...(controlPath ? ['-o', 'ControlMaster=auto', '-o', `ControlPath=${controlPath}`, '-o', 'ControlPersist=yes'] : []),
+        '-i', identity, '-p', port, `${user}@${hostname}`, 'df', '-Pk', rootDir || '/',
+    ], 5_000);
+    if (status.status !== 0) {
+        host.warn(`[capacity] internal writable-storage inspection failed: ${(status.stderr || '').trim()}\n`);
+        return undefined;
+    }
+    const line = status.stdout.trim().split(/\r?\n/u).at(-1) ?? '';
+    const fields = line.trim().split(/\s+/u);
+    const availableKiB = Number(fields[3]);
+    if (!Number.isFinite(availableKiB)) {
+        host.warn(`[capacity] writable-storage verification failed: colima profile "${profile}" reported an unreadable df line "${line}"\n`);
+        return undefined;
+    }
+    return { availableMiB: Math.floor(availableKiB / 1024), substrate: `colima profile "${profile}"` };
+}
+/** Is a Docker DAEMON actually reachable (not just the binary present)? */
+export function dockerAvailable() {
+    try {
+        const r = spawnSync('docker', ['info'], { stdio: 'ignore', timeout: 15_000 });
+        return r.status === 0;
+    }
+    catch {
+        return false;
+    }
+}
+export function flyContainerName(machineId) {
+    return `fly-twin-${machineId}`;
+}
+export function flyVolumeName(volumeId) {
+    return `fly-twin-${volumeId}`;
+}
+export class FlyDockerRuntime {
+    kind = 'docker';
+    #ownerId;
+    #logFollowers = new Map();
+    #machineIds = new Map();
+    #capacityLock;
+    #storageReserveMiB;
+    constructor(root = process.cwd(), options = {}) {
+        // A malformed reserve FAILS here rather than silently reverting to the default: a World that
+        // asked for a different admission floor and got the old one would be told nothing.
+        const reserve = options.storageReserveMiB;
+        if (reserve !== undefined && (!Number.isInteger(reserve) || reserve < 0)) {
+            throw new Error(`machine provider: storage reserve must be a non-negative whole number of MiB (got ${JSON.stringify(reserve)})`);
+        }
+        this.#storageReserveMiB = reserve ?? DEFAULT_STORAGE_RESERVE_MIB;
+        // The service data-root is stable for the lifetime of one named World and
+        // opaque outside this implementation. Hashing it gives every backing
+        // resource an exact lifecycle owner without exposing host paths.
+        this.#ownerId = createHash('sha256').update(resolve(root)).digest('hex').slice(0, 32);
+        // Subordinate compute from Worlds rooted in different repositories still shares one machine,
+        // so admission uses one user-machine lock rather than a per-checkout lock.
+        this.#capacityLock = join(homedir(), '.volter', 'world-resource-claims', 'subordinate-compute.lock');
+        mkdirSync(join(homedir(), '.volter', 'world-resource-claims'), { recursive: true, mode: 0o700 });
+    }
+    async run(spec) {
+        const containerRef = withFileLock(this.#capacityLock, () => {
+            this.#admit(spec);
+            const name = flyContainerName(spec.machineId);
+            const args = [
+                'run', '-d', '--name', name,
+                '--memory', `${spec.memoryMiB}m`,
+                '--memory-swap', `${spec.memoryMiB}m`,
+                '--cpus', String(spec.cpus),
+                '--label', `${FLY_DOCKER_LABEL}=${spec.appName}`,
+                '--label', `${FLY_WORLD_OWNER_LABEL}=${this.#ownerId}`,
+                '--label', `${FLY_MEMORY_LABEL}=${spec.memoryMiB}`,
+                '--label', `${FLY_CPU_LABEL}=${spec.cpus}`,
+            ];
+            for (const [k, v] of Object.entries(spec.env))
+                args.push('-e', `${k}=${v}`);
+            // Publish every internal port on an EPHEMERAL loopback port (fixed host ports would collide
+            // across machines); the assigned ports are read back below and ledgered as twin_local_ports.
+            for (const p of spec.ports)
+                args.push('-p', `127.0.0.1::${p.internal}`);
+            for (const m of spec.mounts)
+                args.push('-v', `${flyVolumeName(m.volumeId)}:${m.path}`);
+            if (spec.entrypoint && spec.entrypoint.length > 0)
+                args.push('--entrypoint', spec.entrypoint[0]);
+            args.push(spec.image);
+            if (spec.entrypoint && spec.entrypoint.length > 1)
+                args.push(...spec.entrypoint.slice(1));
+            if (spec.cmd && spec.cmd.length > 0)
+                args.push(...spec.cmd);
+            const r = docker(args);
+            if (r.status !== 0) {
+                const detail = r.stderr.trim() || `launch command exited ${r.status}`;
+                process.stderr.write(`[machine ${spec.machineId}] internal launch failure: ${detail}\n`);
+                const reason = /no space left on device|enospc/iu.test(detail) ? 'insufficient writable storage'
+                    : /out of memory|cannot allocate memory|killed/iu.test(detail) ? 'insufficient memory'
+                        : /cannot connect|not running|daemon/iu.test(detail) ? 'local execution capacity is unavailable'
+                            : 'local execution failed';
+                throw new Error(`${this.#context(spec)}: ${reason}. Log: ${this.#logPath()}`);
+            }
+            return r.stdout.trim();
+        });
+        this.#machineIds.set(containerRef, spec.machineId);
+        this.#followMachineLogs(containerRef, spec.machineId);
+        const publishedPorts = [];
+        for (const p of spec.ports) {
+            const port = docker(['port', containerRef, String(p.internal)]);
+            // "127.0.0.1:55007" (possibly several lines for v4/v6) — take the first v4 mapping.
+            const match = port.stdout.split('\n').map((l) => l.trim()).find((l) => l.startsWith('127.0.0.1:'));
+            if (match)
+                publishedPorts.push({ internal: p.internal, host: Number(match.split(':').pop()) });
+        }
+        return { containerRef, publishedPorts };
+    }
+    /** Admit and reserve by the same limits that `run` enforces. The lock around this method and
+     * resource creation prevents two concurrent machine requests from both seeing the same final
+     * capacity. Storage is measured inside the local execution substrate when it is discoverable;
+     * callers still receive only the declared World/service/resource vocabulary. */
+    #admit(spec) {
+        const info = docker(['info', '--format', '{{.MemTotal}} {{.NCPU}} {{.DockerRootDir}}']);
+        if (info.status !== 0) {
+            process.stderr.write(`[machine ${spec.machineId}] capacity inspection failed: ${info.stderr.trim()}\n`);
+            throw new Error(`${this.#context(spec)}: local execution capacity is unavailable. Log: ${this.#logPath()}`);
+        }
+        const [memoryRaw, cpuRaw, rootDir = ''] = info.stdout.trim().split(/\s+/u);
+        const totalMemoryMiB = Math.floor(Number(memoryRaw) / MIB);
+        const totalCpus = Number(cpuRaw);
+        if (!Number.isFinite(totalMemoryMiB) || totalMemoryMiB <= 0 || !Number.isFinite(totalCpus) || totalCpus <= 0) {
+            process.stderr.write(`[machine ${spec.machineId}] invalid capacity report: ${info.stdout.trim()}\n`);
+            throw new Error(`${this.#context(spec)}: local execution capacity could not be verified. Log: ${this.#logPath()}`);
+        }
+        const refs = docker(['ps', '-aq', '--filter', `label=${FLY_DOCKER_LABEL}`]);
+        if (refs.status !== 0) {
+            process.stderr.write(`[machine ${spec.machineId}] reservation inspection failed: ${refs.stderr.trim()}\n`);
+            throw new Error(`${this.#context(spec)}: local execution capacity could not be verified. Log: ${this.#logPath()}`);
+        }
+        let reservedMemoryMiB = 0;
+        let reservedCpus = 0;
+        for (const ref of refs.stdout.split(/\s+/u).filter(Boolean)) {
+            const reservation = docker(['inspect', '--format',
+                `{{index .Config.Labels "${FLY_MEMORY_LABEL}"}}|{{index .Config.Labels "${FLY_CPU_LABEL}"}}|{{.HostConfig.Memory}}|{{.HostConfig.NanoCpus}}`, ref]);
+            if (reservation.status !== 0) {
+                process.stderr.write(`[machine ${spec.machineId}] existing reservation inspection failed for ${ref.slice(0, 12)}: ${reservation.stderr.trim()}\n`);
+                throw new Error(`${this.#context(spec)}: resource admission refused because existing local execution capacity could not be verified. Log: ${this.#logPath()}`);
+            }
+            const [memoryLabel, cpuLabel, enforcedMemoryBytes, enforcedNanoCpus] = reservation.stdout.trim().split('|');
+            const memoryMiB = Number(memoryLabel) || Math.ceil(Number(enforcedMemoryBytes) / MIB);
+            const cpus = Number(cpuLabel) || (Number(enforcedNanoCpus) / 1_000_000_000);
+            if (!Number.isFinite(memoryMiB) || memoryMiB <= 0 || !Number.isFinite(cpus) || cpus <= 0) {
+                process.stderr.write(`[machine ${spec.machineId}] existing execution resource ${ref.slice(0, 12)} has no enforceable memory/CPU reservation\n`);
+                throw new Error(`${this.#context(spec)}: resource admission refused because an existing local execution resource has undeclared capacity. Stop its owning World, then retry. Log: ${this.#logPath()}`);
+            }
+            reservedMemoryMiB += memoryMiB;
+            reservedCpus += cpus;
+        }
+        const memorySafetyMiB = Math.max(256, Math.min(1024, Math.floor(totalMemoryMiB * 0.1)));
+        const availableMemoryMiB = Math.max(0, totalMemoryMiB - memorySafetyMiB - reservedMemoryMiB);
+        const availableCpus = Math.max(0, totalCpus - reservedCpus);
+        if (spec.memoryMiB > availableMemoryMiB || spec.cpus > availableCpus) {
+            const details = [
+                ...(spec.memoryMiB > availableMemoryMiB ? [`memory requires ${spec.memoryMiB} MiB, ${availableMemoryMiB} MiB available`] : []),
+                ...(spec.cpus > availableCpus ? [`CPU requires ${spec.cpus}, ${availableCpus} available`] : []),
+            ];
+            throw new Error(`${this.#context(spec)}: resource admission refused (${details.join('; ')}). Log: ${this.#logPath()}`);
+        }
+        const storage = probeWritableStorage(rootDir);
+        const { requiredMiB, reserveMiB, floorMiB } = writableStorageFloorMiB(spec.memoryMiB, this.#storageReserveMiB);
+        if (storage === undefined) {
+            throw new Error(`${this.#context(spec)}: resource admission refused because writable-storage capacity could not be verified. Log: ${this.#logPath()}`);
+        }
+        if (floorMiB > storage.availableMiB) {
+            // The substrate and the measured number belong in the capacity log (a refusal has to be
+            // diagnosable); the caller's error stays in World/service/resource vocabulary.
+            process.stderr.write(`[capacity] writable-storage refused: ${storage.substrate} reports ${storage.availableMiB} MiB available, ${floorMiB} MiB needed (${requiredMiB} MiB machine + ${reserveMiB} MiB reserve)\n`);
+            throw new Error(`${this.#context(spec)}: resource admission refused (writable storage requires ${requiredMiB} MiB plus ${reserveMiB} MiB safety, ${storage.availableMiB} MiB available). Log: ${this.#logPath()}`);
+        }
+    }
+    #context(spec) {
+        const world = process.env.VOLTER_WORLD_NAME ?? 'standalone';
+        const service = process.env.VOLTER_WORLD_SERVICE_ID ?? 'machine-provider';
+        return `World "${world}": service "${service}": machine "${spec.machineId}"`;
+    }
+    #logPath() {
+        return process.env.VOLTER_WORLD_SERVICE_LOG ?? '(service stderr)';
+    }
+    async stop(containerRef, opts = {}) {
+        const args = ['stop'];
+        if (opts.signal)
+            args.push('--signal', opts.signal);
+        if (opts.timeoutSeconds !== undefined)
+            args.push('--time', String(opts.timeoutSeconds));
+        args.push(containerRef);
+        const r = docker(args);
+        if (r.status !== 0)
+            throw new Error(r.stderr.trim() || `docker stop exited ${r.status}`);
+    }
+    /**
+     * Start a STOPPED machine — by RECREATING the container, never `docker start`.
+     *
+     * Real Fly resets the root filesystem on stop→start: "Stopped Machines that are restarted are
+     * completely reset to their original state so that they start clean on the next run"
+     * (fly.io/docs/machines/api/machines-resource). `docker start` resumes the old writable layer,
+     * which would make this twin MORE FORGIVING than the vendor at exactly the point where the
+     * difference bites: an app that writes state outside a volume would pass local rehearsal and
+     * lose that state on the first real stop/start cycle.
+     *
+     * So the old container is removed and a fresh one is run from `spec.image` with the same
+     * env/ports/mounts wiring. `docker rm -v` removes only ANONYMOUS volumes, so the named
+     * `fly-twin-<volume id>` volumes (this twin's mapping of Fly volumes) carry over — mounted
+     * data survives, everything else starts clean, exactly like the vendor. The new container gets
+     * a new handle and new ephemeral loopback ports; both are returned and re-ledgered.
+     */
+    async start(containerRef, spec) {
+        // A ref that is already gone is not a failure: the fresh run below IS the reset.
+        try {
+            await this.remove(containerRef);
+        }
+        catch { /* already reaped */ }
+        return this.run(spec);
+    }
+    async remove(containerRef) {
+        const r = docker(['rm', '-f', '-v', containerRef]);
+        if (r.status !== 0)
+            throw new Error(r.stderr.trim() || `docker rm exited ${r.status}`);
+        this.#logFollowers.get(containerRef)?.kill('SIGTERM');
+        this.#logFollowers.delete(containerRef);
+        this.#machineIds.delete(containerRef);
+    }
+    async pause(containerRef) {
+        const r = docker(['pause', containerRef]);
+        if (r.status !== 0)
+            throw new Error(r.stderr.trim() || `docker pause exited ${r.status}`);
+    }
+    async unpause(containerRef) {
+        const r = docker(['unpause', containerRef]);
+        if (r.status !== 0)
+            throw new Error(r.stderr.trim() || `docker unpause exited ${r.status}`);
+    }
+    async signal(containerRef, signal) {
+        const r = docker(['kill', '--signal', signal, containerRef]);
+        if (r.status !== 0)
+            throw new Error(r.stderr.trim() || `docker kill exited ${r.status}`);
+    }
+    async exec(containerRef, cmd, opts = {}) {
+        const r = docker(['exec', containerRef, ...cmd], { timeoutMs: (opts.timeoutSeconds ?? 60) * 1000 });
+        return { exitCode: r.status, stdout: r.stdout, stderr: r.stderr };
+    }
+    async inspect(containerRef) {
+        const r = docker(['inspect', '--format', '{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}', containerRef]);
+        if (r.status !== 0)
+            return { running: false }; // container gone entirely
+        const [running, exitCode, oomKilled] = r.stdout.trim().split(/\s+/);
+        return {
+            running: running === 'true',
+            ...(exitCode !== undefined ? { exitCode: Number(exitCode) } : {}),
+            ...(oomKilled !== undefined ? { oomKilled: oomKilled === 'true' } : {}),
+        };
+    }
+    async createVolume(volumeId) {
+        const r = docker([
+            'volume', 'create',
+            '--label', FLY_DOCKER_LABEL,
+            '--label', `${FLY_WORLD_OWNER_LABEL}=${this.#ownerId}`,
+            flyVolumeName(volumeId),
+        ]);
+        if (r.status !== 0)
+            throw new Error(r.stderr.trim() || `docker volume create exited ${r.status}`);
+    }
+    async removeVolume(volumeId) {
+        const r = docker(['volume', 'rm', '-f', flyVolumeName(volumeId)]);
+        if (r.status !== 0)
+            throw new Error(r.stderr.trim() || `docker volume rm exited ${r.status}`);
+    }
+    async cleanup() {
+        const selector = `label=${FLY_WORLD_OWNER_LABEL}=${this.#ownerId}`;
+        const containers = docker(['ps', '-aq', '--filter', selector]);
+        if (containers.status !== 0)
+            throw new Error(containers.stderr.trim() || 'could not enumerate owned compute resources');
+        const containerRefs = containers.stdout.split(/\s+/u).filter(Boolean);
+        if (containerRefs.length > 0) {
+            const removed = docker(['rm', '-f', '-v', ...containerRefs]);
+            if (removed.status !== 0)
+                throw new Error(removed.stderr.trim() || 'could not reclaim owned compute resources');
+        }
+        const volumes = docker(['volume', 'ls', '-q', '--filter', selector]);
+        if (volumes.status !== 0)
+            throw new Error(volumes.stderr.trim() || 'could not enumerate owned persistent resources');
+        const volumeRefs = volumes.stdout.split(/\s+/u).filter(Boolean);
+        if (volumeRefs.length > 0) {
+            const removed = docker(['volume', 'rm', '-f', ...volumeRefs]);
+            if (removed.status !== 0)
+                throw new Error(removed.stderr.trim() || 'could not reclaim owned persistent resources');
+        }
+        for (const follower of this.#logFollowers.values())
+            follower.kill('SIGTERM');
+        this.#logFollowers.clear();
+        this.#machineIds.clear();
+    }
+    /** Forward each owned Machine's output into the provider service's stdout/stderr. The World
+     * already retains that declared service log, so diagnostics remain at the World boundary and
+     * never require callers to identify or inspect this runtime's backing substrate. */
+    #followMachineLogs(containerRef, machineId) {
+        if (this.#logFollowers.has(containerRef))
+            return;
+        const follower = spawn('docker', ['logs', '--follow', '--timestamps', containerRef], {
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        this.#logFollowers.set(containerRef, follower);
+        const write = (stream, chunk) => {
+            const prefix = `[machine ${machineId}] `;
+            const text = chunk.toString('utf8').replace(/\n(?=.)/gu, `\n${prefix}`);
+            stream.write(`${prefix}${text}`);
+        };
+        follower.stdout?.on('data', (chunk) => write(process.stdout, chunk));
+        follower.stderr?.on('data', (chunk) => write(process.stderr, chunk));
+        follower.once('error', (error) => {
+            process.stderr.write(`[machine ${machineId}] log stream unavailable: ${error.message}\n`);
+        });
+        follower.once('close', () => {
+            if (this.#logFollowers.get(containerRef) === follower)
+                this.#logFollowers.delete(containerRef);
+        });
+    }
+}

package/dist/src/fly-machines.d.ts

@@ -0,0 +1,152 @@
+export declare const SERVICE = "fly";
+/** One port the execution plane should expose on loopback. `internal` is the service's
+ *  internal_port from the machine config; the runtime picks (and reports) the host port. */
+export type FlyPortSpec = {
+    internal: number;
+};
+/** What the control plane asks the execution plane to run. Derived from fly.MachineConfig. */
+export type FlyContainerSpec = {
+    machineId: string;
+    appName: string;
+    machineName: string;
+    /** The docker image to run (fly.MachineConfig.image). */
+    image: string;
+    /** Guest limits from fly.MachineConfig.guest. A real execution runtime MUST enforce them. */
+    memoryMiB: number;
+    cpus: number;
+    /** config.env merged UNDER Fly's documented FLY_* runtime environment (flyRuntimeEnv). */
+    env: Record<string, string>;
+    /** init.cmd / init.entrypoint (fly.MachineInit). */
+    cmd?: string[];
+    entrypoint?: string[];
+    /** Every service's internal_port, deduped. */
+    ports: FlyPortSpec[];
+    /** config.mounts resolved to volume ids: mount a named local volume at `path`. */
+    mounts: Array<{
+        volumeId: string;
+        path: string;
+    }>;
+};
+export type FlyExecResult = {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+};
+/**
+ * The execution plane, injected. `fly-docker.ts` is the real one (docker CLI over the local
+ * daemon); `VIRTUAL_FLY_RUNTIME` below is the offline default (pure ledger, nothing runs);
+ * tests inject recording/failing fakes. Optional members are capabilities a runtime may lack —
+ * the handler degrades honestly (exec without a runtime that can exec → Fly-shaped 400, never a
+ * fabricated success).
+ */
+export interface FlyContainerRuntime {
+    /** 'virtual' | 'docker' | a test fake's own tag. Recorded on the machine row (twin-only). */
+    readonly kind: string;
+    /** Launch the container. Returns the runtime's handle + any published loopback ports. */
+    run(spec: FlyContainerSpec): Promise<{
+        containerRef: string;
+        publishedPorts?: Array<{
+            internal: number;
+            host: number;
+        }>;
+    }>;
+    /** Stop the container (SIGTERM by default; `signal`/`timeoutSeconds` from Fly's StopRequest). */
+    stop(containerRef: string, opts?: {
+        signal?: string;
+        timeoutSeconds?: number;
+    }): Promise<void>;
+    /** Start a STOPPED container. Fly RESETS a stopped Machine's root filesystem on start —
+     *  "Stopped Machines that are restarted are completely reset to their original state so that
+     *  they start clean on the next run" (fly.io/docs/machines/api/machines-resource) — so a
+     *  faithful execution plane RECREATES the container from `spec` instead of resuming the old
+     *  writable layer; only mounted volumes survive. It therefore returns a NEW handle and NEW
+     *  published ports, exactly like `run`. Absent → the handler removes and re-runs the spec,
+     *  which is the same semantics by another route. */
+    start?(containerRef: string, spec: FlyContainerSpec): Promise<{
+        containerRef: string;
+        publishedPorts?: Array<{
+            internal: number;
+            host: number;
+        }>;
+    }>;
+    /** Remove the container (docker rm -f). */
+    remove(containerRef: string): Promise<void>;
+    /** Suspend/resume (docker pause/unpause). Optional; absent → suspend degrades to stop. */
+    pause?(containerRef: string): Promise<void>;
+    unpause?(containerRef: string): Promise<void>;
+    /** Deliver a signal without a state transition (docker kill --signal). */
+    signal?(containerRef: string, signal: string): Promise<void>;
+    /** Run a command inside the machine (docker exec). Absent → Fly-shaped 400 (see fly-twin.ts). */
+    exec?(containerRef: string, cmd: string[], opts?: {
+        timeoutSeconds?: number;
+    }): Promise<FlyExecResult>;
+    /** Is the container still running? Absent → the control-plane state is taken as-is. */
+    inspect?(containerRef: string): Promise<{
+        running: boolean;
+        exitCode?: number;
+        oomKilled?: boolean;
+    }>;
+    /** Materialize/destroy a named local volume (docker volume create/rm). Optional. */
+    createVolume?(volumeId: string): Promise<void>;
+    removeVolume?(volumeId: string): Promise<void>;
+    /** Reclaim every backing resource owned by this runtime instance. */
+    cleanup?(): Promise<void>;
+}
+/**
+ * The offline default: a pure-ledger execution plane. Nothing runs, nothing listens; every
+ * lifecycle call succeeds and `inspect` reports the container as still running (so control-plane
+ * state is authoritative). This is what makes the ENTIRE control plane — states, events, wait,
+ * leases — verifiable with no Docker daemon anywhere near the gate.
+ */
+export declare const VIRTUAL_FLY_RUNTIME: FlyContainerRuntime;
+export declare function flyRuntimeEnv(m: {
+    id: string;
+    app_name: string;
+    region: string;
+    instance_id: string;
+    image: string;
+    memory_mb: number;
+    private_ip: string;
+}): Record<string, string>;
+export declare const FLY_MACHINE_STATES: readonly ["created", "starting", "started", "stopping", "stopped", "suspending", "suspended", "destroying", "destroyed", "replacing", "failed"];
+export type FlyMachineState = typeof FLY_MACHINE_STATES[number];
+export declare function nowIso(occurredAt?: string): string;
+export declare function nowEpochMs(occurredAt?: string): number;
+/** The live rows of a type (protocol 2: a subject's id is the vendor's — an app's name, a machine's id, a volume's id —
+ *  with no type prefix; a delete is the kernel's tombstone `deleted: true`, cleared by the subject's next write). */
+export declare function rows(type: string, root?: string): Array<Record<string, unknown>>;
+/** Every row of a type the tree has ever held, tombstoned ones included — the ordinal an id seed takes. */
+export declare function countRows(type: string, root?: string): number;
+export declare function getRow(type: string, id: string, root?: string): Record<string, unknown> | undefined;
+/**
+ * The one kernel write choke point. Every write folds a per-subject `_rev` ordinal into `fields`
+ * (read-modify-write off the current projection): the kernel dedupes actions by content +
+ * millisecond timestamp, so without `rev` a machine returning to a PREVIOUS value under a pinned
+ * clock (start→stop→start in one verify) would silently land as `replayed` — reply and stored
+ * state disagreeing (the upstash lesson, ADDING_A_TWIN §5).
+ */
+export declare function write(type: string, id: string, fields: Record<string, unknown>, op: string, root: string | undefined, occurredAt: string | undefined): Promise<Record<string, unknown>>;
+export declare function newMachineId(seed: string): string;
+export declare function newInstanceId(occurredAt: string | undefined, seed: string): string;
+export declare function newVolumeId(seed: string): string;
+export declare function newLeaseNonce(seed: string): string;
+/** An app's internal numeric id — the same seeded source, read as a 24-bit integer. */
+export declare function newAppNumericId(seed: string): number;
+export declare function newMachineName(seed: string): string;
+/** 6PN address: deterministic per machine id (fdaa:… shape, fly-go's "internal 6PN address").
+ *  Ledgered only — nothing routes it locally; published loopback ports are the local
+ *  reachability story. */
+export declare function machinePrivateIp(machineId: string): string;
+export type FlyMachineEvent = {
+    id: string;
+    type: string;
+    status: string;
+    source: 'flyd' | 'user';
+    timestamp: number;
+    /** fly-go MachineEvent.request — used for the exit payload ({exit_event:{exit_code}}), which is
+     *  what flyctl reads to say WHY a machine died. Populated by the runtime-exit fold. */
+    request?: Record<string, unknown>;
+};
+export declare function machineEvent(type: string, status: string, source: 'flyd' | 'user', occurredAt: string | undefined, seq: number, request?: Record<string, unknown>): FlyMachineEvent;
+export declare function imageRefOf(image: string): Record<string, unknown>;
+export declare function containerSpecOf(m: Record<string, unknown>): FlyContainerSpec;