@atomic-chat/core 0.5.1 → 0.13.2

package/dist/runtime/managed-text/lifecycle.js

@@ -0,0 +1,859 @@
+/**
+ * The engine-neutral load lifecycle of a managed text model (openspec change `add-tensorrt-llm-linux`,
+ * task 2.12; spec `tensorrt-llm-runtime`, design D8/D11): everything between "load this model in a
+ * container" and a `SessionInfo` a caller can talk OpenAI HTTP to, and back again.
+ *
+ * One load, in order:
+ *  1. resolve the adapter by the installation's `adapter_id` (`MANAGED_ADAPTER_UNAVAILABLE` if this
+ *     core has none) and let it validate the settings — before anything touches Docker;
+ *  2. `stopping-previous`: the caller's own callback (GPU residency, task 2.15) frees the card;
+ *  3. `starting-container`: engine cache dir, watchdog script and this generation's heartbeat dir on
+ *     disk; the heartbeat ticker running and its first write landed; `docker create` (journalled at
+ *     once), `docker start` — with a fresh host port if the one core picked was taken meanwhile;
+ *  4. `initializing-engine` (entered when an adapter log marker shows up, or at once if the adapter
+ *     has none): poll `docker inspect` and the readiness probe, re-emitting progress with the elapsed
+ *     time on every poll. A container that exits fails the load right there, classified by the
+ *     adapter from its log tail — never by waiting out the timeout;
+ *  5. `ready`: a session gateway with a fresh key in front of the container, and the `SessionInfo`
+ *     (`pid: null`, `execution: 'container'`, a new `generation`) carrying the gateway's port and key.
+ *
+ * Unload, cancel and every failure end the same way: gateway closed, heartbeat stopped, container
+ * stopped with Docker's confirmation, then removed, then its journal record dropped. A stop Docker
+ * would not confirm keeps the model in `stop-unconfirmed` — no session, but still holding its GPU for
+ * task 2.15 to see — and answers `MANAGED_STOP_UNCONFIRMED`; unloading again retries the stop.
+ *
+ * While that stop is in flight the model is `stopping`: one teardown per entry, shared by every
+ * caller (a second unload, a crash noticed meanwhile); a load of the same model waits for it and
+ * starts fresh only after a confirmed stop. Every removal from the session table and every event is
+ * guarded by the entry's identity, so a finished teardown can never touch a newer load's session.
+ *
+ * There is no `engine_id` branch here: the engine id only travels into the journal record and the
+ * container's discovery labels as data.
+ */
+import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+import { AtomicCoreError } from '../../contracts/index.js';
+import { CONTAINER_ENGINE_CACHE_PATH, CONTAINER_HEARTBEAT_PATH, CONTAINER_MODEL_PATH, containerLogs, createContainer, inspectContainer, removeContainer, startContainer, startHeartbeatTicker, stopContainer, watchdogEnv, writeWatchdogScript, } from '../container/index.js';
+import { loadCancelledError, raceLoadCancel } from '../shared/index.js';
+import { projectSessionPort } from './backend-target.js';
+import { ensureEngineCacheDir, removeEngineCaches } from './engine-cache.js';
+import { generateGatewayKey, startManagedGateway } from './gateway.js';
+import { advanceStage, exitErrorCode, exitFailureDetails, lastLogLines, isPortBindConflict, readContainerState, resolveReadinessTimeoutMs, stripDockerTimestamps, } from './load-policy.js';
+import { probeReadiness } from './readiness.js';
+/** The heartbeat file's name inside its generation's directory (and inside the container). */
+export const HEARTBEAT_FILE = 'heartbeat';
+export const DEFAULT_MANAGED_LIFECYCLE_TIMINGS = {
+    pollIntervalMs: 1_000,
+    probeTimeoutMs: 2_000,
+    heartbeatReadyTimeoutMs: 10_000,
+    stopTimeoutSecs: 10,
+    logTailLines: 200,
+    portAttempts: 3,
+    monitorIntervalMs: 5_000,
+    forwardingConfirmPolls: 3,
+    guestProbeEveryPolls: 5,
+    republishAttempts: 1,
+};
+/** "The engine answers inside the guest, Windows cannot reach it": what a WSL load does about it. */
+class ForwardingFailure extends Error {
+    kind;
+    constructor(kind) {
+        super(`the engine answers in the WSL distribution but not on Windows (${kind})`);
+        this.kind = kind;
+    }
+}
+/** A session (or a load) the WSL VM took with it. */
+const wslStoppedError = () => new AtomicCoreError('MODEL_LOAD_FAILED', 'The Atomic Chat WSL distribution stopped (wsl --shutdown, or the WSL VM ended), and the model with it. Load it again to start it.', 'wsl-stopped');
+/** The login name a container user without a passwd entry is given (`USER`/`LOGNAME`). */
+export const CONTAINER_USER_NAME = 'atomic';
+/** Inside the engine cache: the container user's `HOME` and `XDG_CACHE_HOME`, created by core before start. */
+const CONTAINER_HOME_DIR = 'home';
+const CONTAINER_XDG_CACHE_DIR = 'xdg-cache';
+/**
+ * What a non-root user with no passwd entry in the image needs to run a Python engine: `getpass`
+ * (which torch's inductor calls) reads `LOGNAME`/`USER` before the passwd database and raises for an
+ * unknown uid without them, and `HOME`/`XDG_CACHE_HOME` must point somewhere that user can write —
+ * the read-write engine cache, so whatever lands there persists with the cache and goes with it.
+ */
+function containerUserEnv() {
+    return {
+        HOME: `${CONTAINER_ENGINE_CACHE_PATH}/${CONTAINER_HOME_DIR}`,
+        XDG_CACHE_HOME: `${CONTAINER_ENGINE_CACHE_PATH}/${CONTAINER_XDG_CACHE_DIR}`,
+        USER: CONTAINER_USER_NAME,
+        LOGNAME: CONTAINER_USER_NAME,
+    };
+}
+/** A load that ended because the engine exited, or never got ready: carries the classification and log tail. */
+export class ManagedLoadError extends AtomicCoreError {
+    classification;
+    exitCode;
+    constructor(code, message, logTail, classification, exitCode = null) {
+        super(code, message, logTail);
+        this.name = 'ManagedLoadError';
+        this.classification = classification;
+        this.exitCode = exitCode;
+    }
+}
+function defaultSleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted) {
+            reject(loadCancelledError());
+            return;
+        }
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(loadCancelledError());
+        };
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener('abort', onAbort, { once: true });
+    });
+}
+/** A bare file name: no separator, not `.`/`..`, nothing a shell or a path join could read as more. */
+const LAUNCH_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+/**
+ * Writes an adapter's `ManagedEngineLaunch.files` into this generation's directory, which the
+ * container mounts read-only (final review I-2: the engine's option file). A name that is not a bare
+ * file name, or that is the watchdog's own heartbeat file, fails the load before any container exists.
+ */
+async function writeLaunchFiles(dir, files) {
+    for (const [name, content] of Object.entries(files ?? {})) {
+        if (!LAUNCH_FILE_NAME.test(name) || name === HEARTBEAT_FILE) {
+            throw new AtomicCoreError('INVALID_ARGUMENT', 'An engine launch file must have a bare file name other than the heartbeat file.', name);
+        }
+        await writeFile(join(dir, name), content, { mode: 0o644 });
+    }
+}
+/** Rejects after `ms` of real time unless `settle` is called first; never keeps the process alive. */
+function deadline(ms, error) {
+    let timer;
+    const promise = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(error()), ms);
+        timer.unref?.();
+    });
+    promise.catch(() => { });
+    return { promise, settle: () => clearTimeout(timer) };
+}
+/**
+ * Everything a running container was started with that a later load could ask differently for: the
+ * adapter's restart-relevant settings (`restartKey`, all of them by default), the pinned descriptor
+ * and image, the card and the model directory. A load of a ready model with the same key joins its
+ * session; any difference reloads it, since none of these can change inside a running container
+ * (spec "изменение настроек, требующих перезапуска, MUST применяться только при следующей загрузке").
+ * A setting outside the restart key — one the gateway enforces per request, or one only a load reads
+ * — never costs a restart (findings-2.14-r1.md item 3).
+ */
+function loadKeyOf(request, adapter, settings) {
+    const { installation } = request;
+    return JSON.stringify([
+        adapter.restartKey === undefined ? settings : adapter.restartKey(settings),
+        installation.descriptor_id,
+        installation.adapter_id,
+        installation.adapter_contract_version,
+        installation.image.repository,
+        installation.image.digest,
+        request.gpuUuid,
+        request.modelPath,
+    ]);
+}
+export class ManagedTextLifecycle {
+    deps;
+    entries = new Map();
+    /** Set by `shutdown()`: from then on no load may start a container. */
+    closed = false;
+    lastAttempts = new Map();
+    timings;
+    log;
+    fetchFn;
+    now;
+    sleep;
+    newGeneration;
+    startHeartbeat;
+    startGateway;
+    constructor(deps) {
+        this.deps = deps;
+        this.timings = { ...DEFAULT_MANAGED_LIFECYCLE_TIMINGS, ...deps.timings };
+        this.log = deps.log ?? (() => { });
+        this.fetchFn = deps.fetch ?? fetch;
+        this.now = deps.now ?? Date.now;
+        this.sleep = deps.sleep ?? defaultSleep;
+        this.newGeneration = deps.newGeneration ?? randomUUID;
+        this.startHeartbeat = deps.startHeartbeat ?? startHeartbeatTicker;
+        this.startGateway = deps.startGateway ?? startManagedGateway;
+        deps.keeper?.onStopped(() => this.onDistributionStopped());
+    }
+    /** Ready sessions only: a loading or stop-unconfirmed model has none to offer. */
+    list() {
+        return [...this.entries.values()].flatMap((e) => (e.state === 'ready' && e.info ? [e.info] : []));
+    }
+    findSession(modelId) {
+        const entry = this.entries.get(modelId);
+        return entry?.state === 'ready' ? entry.info : undefined;
+    }
+    isLoading(modelId) {
+        return this.entries.get(modelId)?.state === 'loading';
+    }
+    /** Every model holding its GPU, including one whose stop Docker would not confirm. */
+    reservations() {
+        return [...this.entries.values()].map((e) => ({
+            model_id: e.modelId,
+            generation: e.generation,
+            gpu_uuid: e.gpuUuid,
+            container_id: e.containerId ?? null,
+            state: e.state,
+        }));
+    }
+    lastAttempt(modelId) {
+        return this.lastAttempts.get(modelId);
+    }
+    /** The loaded model's live log tail, or its last attempt's kept tail, or undefined. */
+    async logs(modelId) {
+        const entry = this.entries.get(modelId);
+        if (entry?.state === 'ready' && entry.containerId !== undefined) {
+            return this.tail(entry.containerId);
+        }
+        return this.lastAttempts.get(modelId)?.log_tail;
+    }
+    /** Removes engine caches, refusing one a container of this core still mounts. */
+    async removeEngineCaches(selector) {
+        const busy = [...this.entries.values()].find((e) => (selector.modelId === undefined || selector.modelId === e.modelId) &&
+            (selector.descriptorId === undefined || selector.descriptorId === e.descriptorId));
+        if (busy) {
+            throw new AtomicCoreError('MANAGED_RESOURCE_IN_USE', 'That engine cache is mounted into a container; unload the model first.', busy.modelId);
+        }
+        return removeEngineCaches(this.deps.paths, selector);
+    }
+    async load(request) {
+        this.assertOpen();
+        let existing = this.entries.get(request.modelId);
+        if (existing?.state === 'stopping') {
+            // Wait for the teardown in flight; only a confirmed stop frees the model for a fresh load. A
+            // cancel of this load ends the wait at once (final review M-7), and the teardown goes on.
+            await raceLoadCancel(existing.ending ?? Promise.resolve(null), request.signal);
+            // A shutdown that began while this load waited must not be followed by a new container.
+            this.assertOpen();
+            existing = this.entries.get(request.modelId);
+        }
+        if (existing?.state === 'stopping') {
+            throw new AtomicCoreError('MANAGED_OPERATION_CONFLICT', 'This model is being stopped.', request.modelId);
+        }
+        if (existing?.state === 'loading') {
+            throw new AtomicCoreError('MANAGED_OPERATION_CONFLICT', 'This model is already loading.', request.modelId);
+        }
+        if (existing?.state === 'stop-unconfirmed') {
+            throw new AtomicCoreError('MANAGED_STOP_UNCONFIRMED', "This model's previous container has not been confirmed stopped; unload it again first.", existing.containerId);
+        }
+        // Validated before a ready session is compared against, or replaced: settings the adapter
+        // rejects must never cost the caller the session that is already running.
+        const { installation } = request;
+        const adapter = this.deps.adapters.resolve(installation.adapter_id, installation.adapter_contract_version);
+        const settings = adapter.validateSettings(request.settings);
+        const timeoutMs = resolveReadinessTimeoutMs(adapter.readinessTimeoutMs(request.weightBytes, settings), request.timeoutMs);
+        const loadKey = loadKeyOf(request, adapter, settings);
+        if (existing?.state === 'ready' && existing.info) {
+            if (existing.loadKey === loadKey) {
+                existing.settings = settings;
+                return existing.info;
+            }
+            await this.unloadEntry(existing);
+            this.assertOpen();
+            if (this.entries.has(request.modelId)) {
+                throw new AtomicCoreError('MANAGED_OPERATION_CONFLICT', 'Another load of this model started while its previous session was stopping.', request.modelId);
+            }
+        }
+        if (request.signal?.aborted)
+            throw loadCancelledError();
+        const generation = this.newGeneration();
+        const loadAbort = new AbortController();
+        const forwardAbort = () => loadAbort.abort();
+        request.signal?.addEventListener('abort', forwardAbort, { once: true });
+        const entry = {
+            modelId: request.modelId,
+            generation,
+            gpuUuid: request.gpuUuid,
+            descriptorId: installation.descriptor_id,
+            modelPath: request.modelPath,
+            loadKey,
+            settings,
+            state: 'loading',
+            adapter,
+            loadAbort,
+            heartbeatDir: this.deps.paths.heartbeatDir(generation),
+            lease: this.deps.keeper?.acquire(`model ${request.modelId}`),
+        };
+        this.entries.set(request.modelId, entry);
+        this.lastAttempts.delete(request.modelId);
+        const run = this.runLoad(entry, request, settings, timeoutMs);
+        entry.settled = run.catch(() => { });
+        try {
+            return await run;
+        }
+        finally {
+            request.signal?.removeEventListener('abort', forwardAbort);
+        }
+    }
+    /**
+     * Ends the model's session with a Docker-confirmed stop. Unloading a model still loading cancels
+     * that load. An unconfirmed stop throws `MANAGED_STOP_UNCONFIRMED` and leaves the model reserved.
+     */
+    async unload(modelId) {
+        const entry = this.entries.get(modelId);
+        if (!entry)
+            return;
+        if (entry.state === 'loading') {
+            entry.loadAbort.abort();
+            await entry.settled;
+            const after = this.entries.get(modelId);
+            if (after === entry && after.state === 'stop-unconfirmed') {
+                throw this.stopUnconfirmed(after, 'the cancelled load');
+            }
+            return;
+        }
+        await this.unloadEntry(entry);
+    }
+    /** Unloads everything; failures are logged, never thrown, so one stuck container cannot block shutdown. */
+    async shutdown() {
+        this.closed = true;
+        await Promise.all([...this.entries.keys()].map((modelId) => this.unload(modelId).catch((error) => this.log('warn', `managed-text: unloading ${modelId} at shutdown failed: ${String(error)}`))));
+    }
+    assertOpen() {
+        if (this.closed) {
+            throw new AtomicCoreError('CORE_NOT_RUNNING', 'The managed runtime is shutting down; nothing new loads.');
+        }
+    }
+    /**
+     * Ends a session that is not loading with a Docker-confirmed stop. A teardown already in flight (a
+     * second unload, a crash, a failed load) is joined, not repeated, and only the first caller reports
+     * `session:unloaded`.
+     */
+    async unloadEntry(entry) {
+        const first = entry.ending === undefined;
+        const outcome = await this.end(entry);
+        if (outcome && !outcome.confirmed)
+            throw this.stopUnconfirmed(entry, outcome.reason);
+        if (first) {
+            this.deps.emit('session:unloaded', { provider: this.deps.provider, model_id: entry.modelId, pid: null });
+        }
+    }
+    // ── load ──────────────────────────────────────────────────────────────────────────────────────
+    async runLoad(entry, request, settings, timeoutMs) {
+        const startedAt = this.now();
+        const signal = entry.loadAbort.signal;
+        const progress = (stage) => this.deps.emit('session:load-progress', {
+            provider: this.deps.provider,
+            model_id: entry.modelId,
+            generation: entry.generation,
+            stage,
+            elapsed_ms: this.now() - startedAt,
+            ...(request.gpuSubstituted === undefined ? {} : { gpu_substituted: request.gpuSubstituted }),
+        });
+        try {
+            if (request.stopPrevious) {
+                progress('stopping-previous');
+                await raceLoadCancel(request.stopPrevious(signal, entry.generation), signal);
+            }
+            // Once, after stopPrevious and before anything of the container exists — never repeated on a
+            // port-bind retry, since it is about whether the card has room, not about the host port.
+            let plan = undefined;
+            if (request.beforeCreate) {
+                plan = await raceLoadCancel(request.beforeCreate(), signal);
+                this.checkAborted(signal);
+            }
+            progress('starting-container');
+            const cacheDir = await ensureEngineCacheDir(this.deps.paths, request.installation.descriptor_id, entry.modelId);
+            if (this.deps.containerUser !== undefined) {
+                await mkdir(join(cacheDir, CONTAINER_HOME_DIR), { recursive: true });
+                await mkdir(join(cacheDir, CONTAINER_XDG_CACHE_DIR), { recursive: true });
+            }
+            const launch = entry.adapter.buildLaunch({
+                modelId: entry.modelId,
+                settings,
+                modelPath: CONTAINER_MODEL_PATH,
+                engineCachePath: CONTAINER_ENGINE_CACHE_PATH,
+                generationFilesPath: CONTAINER_HEARTBEAT_PATH,
+                weightBytes: request.weightBytes,
+                family: request.family,
+                unifiedMemory: request.unifiedMemory === true,
+                gpuTotalVramBytes: request.gpuTotalVramBytes ?? null,
+                gpuComputeCapability: request.gpuComputeCapability ?? null,
+                ...(plan === undefined ? {} : { plan }),
+            });
+            await writeWatchdogScript(this.deps.paths.watchdogScript);
+            await mkdir(entry.heartbeatDir, { recursive: true });
+            await writeLaunchFiles(entry.heartbeatDir, launch.files);
+            this.checkAborted(signal);
+            entry.ticker = this.startHeartbeat({
+                path: join(entry.heartbeatDir, HEARTBEAT_FILE),
+                onError: (error) => this.log('warn', `managed-text: heartbeat write failed for ${entry.modelId}: ${String(error)}`),
+            });
+            await this.firstHeartbeat(entry.ticker, signal);
+            let prepared = await this.createAndStart(entry, request, launch, cacheDir, signal);
+            for (let republished = 0;; republished++) {
+                try {
+                    await this.waitReady(entry, prepared.target, timeoutMs, progress, signal);
+                    break;
+                }
+                catch (error) {
+                    if (!(error instanceof ForwardingFailure))
+                        throw error;
+                    // Another program holds the port on Windows: publish once more, on a port Docker picks anew.
+                    if (error.kind !== 'port-taken' || republished >= this.timings.republishAttempts) {
+                        throw (await this.deps.deployment.forwardingError?.()) ?? error;
+                    }
+                    this.log('warn', `managed-text: the Windows port of ${entry.modelId} is held by another program; publishing again`);
+                    await this.discardContainer(entry);
+                    prepared = await this.createAndStart(entry, request, launch, cacheDir, signal);
+                }
+            }
+            const refusal = await this.readyRefusal(entry, settings);
+            if (refusal !== null)
+                throw refusal;
+            const apiKey = generateGatewayKey();
+            const adapter = entry.adapter;
+            // What this session can do, for the rewriter to refuse what it cannot (findings-2.14-r1.md
+            // item 1): read off the same family the launch was built from, never guessed.
+            const capabilities = adapter.capabilities({ settings, family: request.family });
+            // Bound here, once, so the gateway itself never needs to know an adapter's settings shape
+            // (ManagedTextAdapter.rewriteRequestBody, ADR
+            // 2026-09-28-tensorrt-llm-output-cap-enforced-by-the-session-gateway). The settings are read off
+            // the entry on every request: a later load that joins this session with new per-request
+            // settings (outside the adapter's restart key) takes effect at once. Called through
+            // `adapter.rewriteRequestBody(...)`, not a detached local reference to the function, so an
+            // implementation that relies on `this` (an object method, not just a plain function like
+            // tensorrt-llm's own) still sees `this === adapter` when the gateway invokes it
+            // (findings-2.13-r2.md item 4); the same holds for `mapErrorResponse`.
+            entry.gateway = await this.startGateway({
+                upstream: { host: '127.0.0.1', port: projectSessionPort(prepared.target) },
+                apiKey,
+                allowedHosts: this.deps.allowedHosts,
+                routes: adapter.routes,
+                ...(adapter.rewritableRoutes === undefined ? {} : { rewritableRoutes: adapter.rewritableRoutes }),
+                ...(adapter.rewriteRequestBody === undefined
+                    ? {}
+                    : {
+                        rewriteRequestBody: (route, body) => adapter.rewriteRequestBody(route, body, entry.settings, capabilities),
+                    }),
+                ...(adapter.rewriteResponseFor === undefined
+                    ? {}
+                    : {
+                        rewriteResponseFor: (route, body) => adapter.rewriteResponseFor(route, body, request.family),
+                    }),
+                ...(adapter.mapErrorResponse === undefined
+                    ? {}
+                    : {
+                        mapErrorResponse: (route, status, body) => adapter.mapErrorResponse(route, status, body),
+                    }),
+                ...(adapter.describeStreamFailure === undefined
+                    ? {}
+                    : {
+                        streamFailure: async () => entry.containerId === undefined
+                            ? null
+                            : adapter.describeStreamFailure(await this.tail(entry.containerId)),
+                    }),
+            });
+            this.checkAborted(signal);
+            entry.info = {
+                pid: null,
+                port: entry.gateway.port,
+                model_id: entry.modelId,
+                model_path: entry.modelPath,
+                is_embedding: false,
+                api_key: apiKey,
+                execution: 'container',
+                generation: entry.generation,
+            };
+            entry.state = 'ready';
+            progress('ready');
+            this.startMonitor(entry);
+            return entry.info;
+        }
+        catch (error) {
+            const failure = entry.distributionStopped === true ? wslStoppedError() : signal.aborted ? loadCancelledError() : error;
+            await this.failLoad(entry, failure);
+            throw failure;
+        }
+    }
+    checkAborted(signal) {
+        if (signal.aborted)
+            throw loadCancelledError();
+    }
+    /** The watchdog must see a live heartbeat from its first poll; `ready` never settles on its own if writes keep failing. */
+    async firstHeartbeat(ticker, signal) {
+        const limit = deadline(this.timings.heartbeatReadyTimeoutMs, () => new AtomicCoreError('IO_ERROR', 'The container heartbeat file could not be written, so the watchdog would stop the engine.', `no successful write within ${this.timings.heartbeatReadyTimeoutMs} ms`));
+        try {
+            await raceLoadCancel(Promise.race([ticker.ready, limit.promise]), signal);
+        }
+        finally {
+            limit.settle();
+        }
+    }
+    createSpec(request, launch, cacheDir, prepared) {
+        return {
+            image: request.installation.image,
+            gpuUuid: request.gpuUuid,
+            selinux: request.selinux,
+            ...(request.selinux ? { selinuxDataRoot: this.deps.selinuxDataRoot } : {}),
+            mounts: {
+                // One resolver for all four mounts: the deployment's, which also resolved `heartbeat`.
+                model: { source: this.deps.deployment.mountSource(request.modelPath) },
+                engineCache: { source: this.deps.deployment.mountSource(cacheDir) },
+                entrypoint: { source: this.deps.deployment.mountSource(this.deps.paths.watchdogScript) },
+                heartbeat: { source: prepared.heartbeat.mount_source },
+            },
+            publication: prepared.publication,
+            ...(this.deps.containerUser === undefined ? {} : { user: this.deps.containerUser }),
+            labels: {
+                engine_id: request.installation.engine_id,
+                scope: this.deps.scope,
+                instance_id: this.deps.instanceId,
+            },
+            // The container user's identity after the adapter's env, so no adapter can point `HOME` at a
+            // place that user cannot write; the watchdog's own settings come last: an adapter can never
+            // switch it off or retime it.
+            env: {
+                ...launch.env,
+                ...(this.deps.containerUser === undefined ? {} : containerUserEnv()),
+                ...watchdogEnv({ heartbeatFile: `${CONTAINER_HEARTBEAT_PATH}/${HEARTBEAT_FILE}` }),
+            },
+            command: ['--', ...launch.argv],
+            ...(launch.shm_size === undefined ? {} : { shmSize: launch.shm_size }),
+        };
+    }
+    /**
+     * `docker create` + journal + `docker start`. Core picked the host port, so another process can
+     * bind it before Docker does; that one failure is retried with a new port (bounded), everything
+     * else fails the load. (`beforeCreate` ran once before the launch was built.)
+     */
+    async createAndStart(entry, request, launch, cacheDir, signal) {
+        for (let attempt = 1;; attempt++) {
+            const prepared = await this.deps.deployment.prepareLaunch(launch.engine, entry.heartbeatDir);
+            this.checkAborted(signal);
+            const spec = this.createSpec(request, launch, cacheDir, prepared);
+            const { containerId } = await createContainer(this.deps.exec, spec, this.deps.createContainerDeps);
+            entry.containerId = containerId;
+            try {
+                await this.deps.journal.add({
+                    container_id: containerId,
+                    engine_id: request.installation.engine_id,
+                    image_digest: request.installation.image.digest,
+                    scope: this.deps.scope,
+                    instance_id: this.deps.instanceId,
+                    created_at: new Date(this.now()).toISOString(),
+                });
+            }
+            catch (error) {
+                // Never started, and not journalled: remove it now, or nothing would ever find it again.
+                await removeContainer(this.deps.exec, containerId).catch((rmError) => this.log('error', `managed-text: container ${containerId} is neither journalled nor removed; remove it by hand ` +
+                    `(journal: ${String(error)}; rm: ${String(rmError)})`));
+                entry.containerId = undefined;
+                throw error;
+            }
+            this.checkAborted(signal);
+            try {
+                await startContainer(this.deps.exec, containerId);
+                return this.deps.deployment.resolveTarget === undefined
+                    ? prepared
+                    : await this.deps.deployment.resolveTarget(containerId, prepared);
+            }
+            catch (error) {
+                if (!isPortBindConflict(error) || attempt >= this.timings.portAttempts)
+                    throw error;
+                this.log('warn', `managed-text: host port ${prepared.publication.host_port} was taken before docker could bind it; retrying on another`);
+                await removeContainer(this.deps.exec, containerId);
+                await this.deps.journal.remove(containerId);
+                entry.containerId = undefined;
+            }
+        }
+    }
+    async waitReady(entry, target, timeoutMs, progress, signal) {
+        const containerId = entry.containerId;
+        const { adapter } = entry;
+        const deadlineAt = this.now() + timeoutMs;
+        let stage = advanceStage('starting-container', adapter.stageMarkers, '');
+        let redirectWarned = false;
+        let insideStreak = 0;
+        let polls = 0;
+        for (;;) {
+            this.checkAborted(signal);
+            const state = await inspectContainer(this.deps.exec, containerId)
+                .then(readContainerState)
+                .catch((error) => {
+                // A docker hiccup is not an exit: keep waiting, the timeout still bounds the load.
+                this.log('warn', `managed-text: docker inspect failed while loading ${entry.modelId}: ${String(error)}`);
+                return { exited: false, exitCode: null };
+            });
+            if (state.exited)
+                throw await this.exitFailure(containerId, adapter, state.exitCode);
+            if (stage !== 'initializing-engine') {
+                stage = advanceStage(stage, adapter.stageMarkers, await this.tail(containerId).catch(() => ''));
+            }
+            progress(stage);
+            const outcome = await probeReadiness(this.fetchFn, target, adapter.readiness, this.timings.probeTimeoutMs);
+            if (outcome === 'ready')
+                return;
+            // WSL: answering inside the guest while Windows cannot reach it is not "still starting".
+            const inGuest = this.deps.deployment.probeInGuest;
+            polls += 1;
+            if (inGuest !== undefined && (insideStreak > 0 || polls % this.timings.guestProbeEveryPolls === 0)) {
+                insideStreak =
+                    (await inGuest(target, adapter.readiness).catch(() => 'not-ready')) === 'ready'
+                        ? insideStreak + 1
+                        : 0;
+                if (insideStreak >= this.timings.forwardingConfirmPolls) {
+                    throw new ForwardingFailure((await this.deps.deployment.diagnoseForwarding?.(target).catch(() => 'not-forwarded')) ??
+                        'not-forwarded');
+                }
+            }
+            if (outcome === 'redirect' && !redirectWarned) {
+                redirectWarned = true;
+                this.log('warn', `managed-text: ${entry.modelId}'s readiness probe answered a redirect; not following it`);
+            }
+            if (this.now() >= deadlineAt) {
+                const tail = await this.tail(containerId).catch(() => '');
+                throw new ManagedLoadError('MODEL_LOAD_TIMED_OUT', `The engine did not become ready within ${Math.round(timeoutMs / 1000)} s.`, tail);
+            }
+            await this.sleep(this.timings.pollIntervalMs, signal);
+        }
+    }
+    async exitFailure(containerId, adapter, exitCode) {
+        const read = await this.tail(containerId).catch(() => '');
+        // The adapter reads the whole log, not the tail: a container that never got ready logged only its
+        // own start, and the line that says why it died can sit above a traceback longer than the tail
+        // (2026-09-29 VM run: an OOM surfaced as MODEL_LOAD_FAILED). The details stay the tail, led by
+        // whatever the adapter quoted from above it; when only the tail read failed, the tail is the end
+        // of the whole log instead of nothing.
+        const whole = await this.wholeLog(containerId).catch(() => read);
+        const tail = read !== '' ? read : lastLogLines(whole, this.timings.logTailLines);
+        const classification = adapter.classifyExit(whole, exitCode);
+        return new ManagedLoadError(exitErrorCode(classification.kind), classification.message, exitFailureDetails(tail, classification.excerpt), classification, exitCode);
+    }
+    /** The adapter's `classifyReady` verdict on a container that just got ready, as the load's error; null when fit. */
+    async readyRefusal(entry, settings) {
+        const adapter = entry.adapter;
+        if (adapter.classifyReady === undefined || entry.containerId === undefined)
+            return null;
+        const whole = await this.wholeLog(entry.containerId).catch(() => '');
+        const classification = adapter.classifyReady(whole, settings);
+        if (classification === null)
+            return null;
+        return new ManagedLoadError(exitErrorCode(classification.kind), classification.message, exitFailureDetails(lastLogLines(whole, this.timings.logTailLines), classification.excerpt), classification);
+    }
+    async tail(containerId) {
+        return stripDockerTimestamps(await containerLogs(this.deps.exec, containerId, this.timings.logTailLines));
+    }
+    /** Every line the container logged, bounded only by the docker exec's per-stream output cap (a
+     *  longer log keeps its start and its end). */
+    async wholeLog(containerId) {
+        return stripDockerTimestamps(await containerLogs(this.deps.exec, containerId, 'all'));
+    }
+    async failLoad(entry, failure) {
+        let logTail = failure instanceof ManagedLoadError ? (failure.details ?? '') : '';
+        if (!(failure instanceof ManagedLoadError) && entry.containerId !== undefined) {
+            logTail = await this.tail(entry.containerId).catch(() => '');
+        }
+        const error = failure instanceof AtomicCoreError
+            ? { code: failure.code, message: failure.message }
+            : { code: 'MODEL_LOAD_FAILED', message: String(failure) };
+        this.lastAttempts.set(entry.modelId, {
+            model_id: entry.modelId,
+            generation: entry.generation,
+            log_tail: logTail,
+            error,
+            at: this.now(),
+        });
+        await this.end(entry);
+    }
+    // ── stop ──────────────────────────────────────────────────────────────────────────────────────
+    stopUnconfirmed(entry, reason) {
+        return new AtomicCoreError('MANAGED_STOP_UNCONFIRMED', 'Docker did not confirm the model container stopped; its GPU stays reserved. Unload again to retry.', `${entry.containerId ?? 'no container'}: ${reason}`);
+    }
+    /**
+     * The one way an entry is stopped: `stopping` while the teardown runs, then gone (confirmed, or no
+     * container at all) or `stop-unconfirmed`. Concurrent callers share the same promise. A teardown
+     * that throws counts as unconfirmed — nothing proved the container stopped. The entry leaves the
+     * table only if it is still the one there, never a newer load's.
+     *
+     * `beforeStop` runs once the session is withdrawn but before the container is stopped and removed:
+     * a crash reads its log tail there, while the entry is already `stopping`, so no load in the
+     * meantime can be handed the dead session.
+     */
+    end(entry, beforeStop) {
+        if (entry.ending)
+            return entry.ending;
+        entry.state = 'stopping';
+        entry.ending = this.teardown(entry, beforeStop)
+            .catch((error) => {
+            this.log('error', `managed-text: stopping ${entry.modelId} failed: ${String(error)}`);
+            return { confirmed: false, reason: String(error) };
+        })
+            .then((outcome) => {
+            if (outcome !== null && !outcome.confirmed)
+                entry.state = 'stop-unconfirmed';
+            else {
+                if (this.entries.get(entry.modelId) === entry)
+                    this.entries.delete(entry.modelId);
+                // Nothing of this session runs any more: the distribution need not be held for it.
+                entry.lease?.release();
+            }
+            entry.ending = undefined;
+            return outcome;
+        });
+        return entry.ending;
+    }
+    /**
+     * Gateway closed, heartbeat stopped, container stopped (confirmed) and removed, journal record
+     * dropped. `null` when there was no container to stop. An unconfirmed stop leaves the entry in
+     * `stop-unconfirmed` with its container id, for a later retry and for residency to see.
+     *
+     * The heartbeat stops either way: if Docker merely lost track of an engine that is still running,
+     * the watchdog inside then ends it within its stale limit.
+     */
+    async teardown(entry, beforeStop) {
+        entry.monitor?.abort();
+        entry.monitor = undefined;
+        entry.info = undefined;
+        if (entry.gateway) {
+            const gateway = entry.gateway;
+            entry.gateway = undefined;
+            await gateway.close().catch(() => { });
+        }
+        entry.ticker?.stop();
+        entry.ticker = undefined;
+        await beforeStop?.().catch((error) => this.log('warn', `managed-text: before stopping ${entry.modelId}: ${String(error)}`));
+        if (entry.distributionStopped === true) {
+            // The VM took the container with it. Any docker call, or a touch of the guest's files through
+            // \\wsl.localhost, would start the VM again; the journal keeps the record, and the next start's
+            // reconcile removes the stopped container.
+            const hadContainer = entry.containerId !== undefined;
+            entry.containerId = undefined;
+            return hadContainer ? { confirmed: true, status: 'exited' } : null;
+        }
+        if (entry.containerId === undefined) {
+            await rm(entry.heartbeatDir, { recursive: true, force: true }).catch(() => { });
+            return null;
+        }
+        const containerId = entry.containerId;
+        const outcome = await stopContainer(this.deps.exec, containerId, this.timings.stopTimeoutSecs);
+        if (!outcome.confirmed)
+            return outcome;
+        try {
+            await removeContainer(this.deps.exec, containerId);
+            await this.deps.journal.remove(containerId);
+        }
+        catch (error) {
+            // The engine has exited, so the session is over; the journal keeps the record and the next
+            // startup's reconcile removes whatever is left.
+            this.log('warn', `managed-text: container ${containerId} stopped but could not be removed: ${String(error)}`);
+        }
+        entry.containerId = undefined;
+        await rm(entry.heartbeatDir, { recursive: true, force: true }).catch(() => { });
+        return outcome;
+    }
+    /** A started container this load gives up on before it was ready (a republication): stopped, removed, unjournalled. */
+    async discardContainer(entry) {
+        const containerId = entry.containerId;
+        if (containerId === undefined)
+            return;
+        const stopped = await stopContainer(this.deps.exec, containerId, this.timings.stopTimeoutSecs);
+        if (!stopped.confirmed)
+            throw this.stopUnconfirmed(entry, stopped.reason);
+        await removeContainer(this.deps.exec, containerId);
+        await this.deps.journal.remove(containerId);
+        entry.containerId = undefined;
+    }
+    /**
+     * The WSL distribution stopped under this core (design D8): every session and load ends at once,
+     * without a docker call, and a ready session is reported dead with `wsl-stopped`.
+     */
+    onDistributionStopped() {
+        for (const entry of [...this.entries.values()]) {
+            if (entry.state !== 'ready' && entry.state !== 'loading')
+                continue;
+            entry.distributionStopped = true;
+            // Let go at once: nothing of this session runs any more, and nothing may hold the VM up for it.
+            entry.lease?.release();
+            if (entry.state === 'loading') {
+                entry.loadAbort.abort();
+                continue;
+            }
+            const error = wslStoppedError();
+            this.lastAttempts.set(entry.modelId, {
+                model_id: entry.modelId,
+                generation: entry.generation,
+                log_tail: '',
+                error: { code: error.code, message: error.message },
+                at: this.now(),
+            });
+            void this.end(entry).then(() => this.deps.emit('session:died', {
+                provider: this.deps.provider,
+                pid: null,
+                model_id: entry.modelId,
+                exit_code: null,
+                signal: null,
+                message: error.message,
+                reason: 'wsl-stopped',
+            }));
+        }
+    }
+    // ── crash after ready ─────────────────────────────────────────────────────────────────────────
+    startMonitor(entry) {
+        const monitor = new AbortController();
+        entry.monitor = monitor;
+        const containerId = entry.containerId;
+        void (async () => {
+            for (;;) {
+                try {
+                    await this.sleep(this.timings.monitorIntervalMs, monitor.signal);
+                }
+                catch {
+                    return;
+                }
+                if (monitor.signal.aborted || entry.state !== 'ready')
+                    return;
+                const state = await inspectContainer(this.deps.exec, containerId)
+                    .then(readContainerState)
+                    .catch(() => ({ exited: false, exitCode: null }));
+                if (monitor.signal.aborted || entry.state !== 'ready')
+                    return;
+                if (state.exited) {
+                    await this.onCrash(entry, state.exitCode);
+                    return;
+                }
+            }
+        })();
+    }
+    /**
+     * The entry turns `stopping` synchronously, before its log tail is read: a load arriving while the
+     * tail is still being fetched waits for this teardown instead of being handed the dead session.
+     */
+    async onCrash(entry, exitCode) {
+        if (entry.state !== 'ready' || this.entries.get(entry.modelId) !== entry)
+            return;
+        const containerId = entry.containerId;
+        let message = '';
+        await this.end(entry, async () => {
+            const tail = await this.tail(containerId).catch(() => '');
+            const classification = entry.adapter.classifyExit(tail, exitCode);
+            message = classification.message;
+            this.lastAttempts.set(entry.modelId, {
+                model_id: entry.modelId,
+                generation: entry.generation,
+                log_tail: tail,
+                error: { code: exitErrorCode(classification.kind), message: classification.message },
+                at: this.now(),
+            });
+        });
+        this.deps.emit('session:died', {
+            provider: this.deps.provider,
+            pid: null,
+            model_id: entry.modelId,
+            exit_code: exitCode,
+            signal: null,
+            message,
+        });
+    }
+}
+//# sourceMappingURL=lifecycle.js.map