@atomic-chat/core 0.5.1 → 0.13.2

package/dist/runtime/container/reconcile.js

@@ -0,0 +1,79 @@
+/**
+ * Startup reconciliation of the execution journal (task 2.10): stop and remove the model containers
+ * a previous core instance left running, before the first load of this instance can be served (spec
+ * `tensorrt-llm-runtime`, "Восстановление контейнеров после перезапуска core"). Mirrors
+ * `core/reap-orphans.ts`'s shape for the native-process journal, one level down in this module so
+ * `core/create.ts` only ever calls an exported function, never touches Docker argv itself.
+ *
+ * Only a container whose id is recorded in the execution journal is ever inspected, stopped or
+ * removed here. A container the host happens to be running — even one carrying this core's own
+ * discovery labels (`ModelContainerLabels`, `types.ts`) — is left alone if it has no journal record:
+ * labels are informational only, never an authority to act (spec: "Метки контейнера MUST NOT служить
+ * единственным основанием для остановки"). "Belongs to a previous instance" is a plain instance-id
+ * comparison, not a liveness probe of that other instance: `InstanceLock` (`src/lock/`) guarantees at
+ * most one core owns a data folder at a time, so if *this* instance now holds the lock and journal, a
+ * record stamped with any other instance id can only be a *former* owner's — there is no live
+ * instance to disprove liveness against, unlike the native-process journal's `scanOrphans`, which
+ * must disprove liveness of another *process* that might still be running unrelated to lock
+ * ownership. This is documented rather than a call to `verifyProcessIdentity` because there is no
+ * "other core process" identity to check here at all.
+ */
+import { inspectContainer, removeContainer, stopContainer } from './operations.js';
+/** `docker stop --time` for a reconciled orphan: generous, since nothing is waiting on this container. */
+export const RECONCILE_STOP_TIMEOUT_SECONDS = 10;
+/**
+ * Reconciles every record in `journal` that does not belong to `currentInstanceId` against the host
+ * through `exec`. Safe to call with an empty journal (a no-op) or before the journal's directory
+ * exists (`ExecutionJournal.open` already handles that).
+ */
+export async function reconcileExecutions(journal, currentInstanceId, exec, log, stopTimeoutSeconds = RECONCILE_STOP_TIMEOUT_SECONDS, 
+/** Checked before each record: once aborted, the rest are reported as `skipped` and left alone. */
+signal) {
+    const result = {
+        stopped: [],
+        absent: [],
+        unconfirmed: [],
+        failed: [],
+        skipped: [],
+    };
+    for (const record of journal.list()) {
+        if (record.instance_id === currentInstanceId)
+            continue; // this instance's own container, not an orphan
+        if (signal?.aborted) {
+            result.skipped.push(record);
+            log('warn', `execution journal: out of time before container ${record.container_id} (engine ${record.engine_id}, ` +
+                `scope ${record.scope}) — leaving it and its journal record for the next startup`);
+            continue;
+        }
+        try {
+            const inspected = await inspectContainer(exec, record.container_id);
+            if (!inspected.found) {
+                await journal.remove(record.container_id);
+                result.absent.push(record);
+                continue;
+            }
+            const outcome = await stopContainer(exec, record.container_id, stopTimeoutSeconds);
+            if (!outcome.confirmed) {
+                log('warn', `execution journal: could not confirm container ${record.container_id} (engine ${record.engine_id}, ` +
+                    `scope ${record.scope}) stopped — leaving it and its journal record alone (${outcome.reason})`);
+                result.unconfirmed.push(record);
+                continue;
+            }
+            await removeContainer(exec, record.container_id);
+            await journal.remove(record.container_id);
+            result.stopped.push(record);
+            log('info', `stopped and removed orphaned managed-runtime container ${record.container_id} ` +
+                `(engine ${record.engine_id}, scope ${record.scope}) left by a previous core instance`);
+        }
+        catch (error) {
+            // inspectContainer/stopContainer/removeContainer threw (an IO_ERROR neither "absent" nor
+            // "unconfirmed" already covers): one bad record must never stop reconcile from reaching the rest.
+            const detail = error instanceof Error ? error.message : String(error);
+            log('error', `execution journal: reconcile failed for container ${record.container_id} (engine ${record.engine_id}, ` +
+                `scope ${record.scope}) — leaving it and its journal record alone (${detail})`);
+            result.failed.push(record);
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=reconcile.js.map

package/dist/runtime/container/reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconcile.js","sourceRoot":"","sources":["../../../src/runtime/container/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAOlF,0GAA0G;AAC1G,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAA;AAuBhD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAyB,EACzB,iBAAyB,EACzB,IAAgB,EAChB,GAAoB,EACpB,kBAAkB,GAAG,8BAA8B;AACnD,mGAAmG;AACnG,MAAoB;IAEpB,MAAM,MAAM,GAA6B;QACvC,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE;QACV,WAAW,EAAE,EAAE;QACf,MAAM,EAAE,EAAE;QACV,OAAO,EAAE,EAAE;KACZ,CAAA;IACD,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,WAAW,KAAK,iBAAiB;YAAE,SAAQ,CAAC,+CAA+C;QACtG,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC3B,GAAG,CACD,MAAM,EACN,mDAAmD,MAAM,CAAC,YAAY,YAAY,MAAM,CAAC,SAAS,IAAI;gBACpG,SAAS,MAAM,CAAC,KAAK,4DAA4D,CACpF,CAAA;YACD,SAAQ;QACV,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACnE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;gBACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAC1B,SAAQ;YACV,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAA;YAClF,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;gBACvB,GAAG,CACD,MAAM,EACN,kDAAkD,MAAM,CAAC,YAAY,YAAY,MAAM,CAAC,SAAS,IAAI;oBACnG,SAAS,MAAM,CAAC,KAAK,wDAAwD,OAAO,CAAC,MAAM,GAAG,CACjG,CAAA;gBACD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAC/B,SAAQ;YACV,CAAC;YAED,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YAChD,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;YACzC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC3B,GAAG,CACD,MAAM,EACN,0DAA0D,MAAM,CAAC,YAAY,GAAG;gBAC9E,WAAW,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,KAAK,oCAAoC,CACzF,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yFAAyF;YACzF,kGAAkG;YAClG,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACrE,GAAG,CACD,OAAO,EACP,qDAAqD,MAAM,CAAC,YAAY,YAAY,MAAM,CAAC,SAAS,IAAI;gBACtG,SAAS,MAAM,CAAC,KAAK,gDAAgD,MAAM,GAAG,CACjF,CAAA;YACD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/runtime/container/types.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Types for the Docker executor (task 2.8, openspec change `add-tensorrt-llm-linux`): building the
+ * argv for one model container's lifecycle, running it with no shell, and reading back what Docker
+ * reports.
+ *
+ * `ImageRef` reuses the contracts' `PlatformImage` shape (`repository` + a pinned `sha256:` digest)
+ * rather than redeclaring it, since a descriptor's `image['linux/amd64']` (task 2.1,
+ * `src/contracts/environment.ts`) is exactly what this module pulls, inspects and creates from.
+ * `ModelContainerCreateSpec.publication` is `managed-text`'s `HostPublication` (task 2.7): the
+ * deployment seam already resolved which host port and container port to publish, so this module
+ * only turns that decision into argv, never makes it.
+ */
+import type { PlatformImage } from '../../contracts/index.js';
+import type { HostPublication } from '../managed-text/index.js';
+/** A platform's pinned image: a bare repository plus its `sha256:` digest, never a tag. */
+export type ImageRef = PlatformImage;
+/** Where a mount's source lives in the Docker daemon's own filesystem view (`MountSourceResolver`, task 2.7). */
+export interface DockerMountSource {
+    source: string;
+}
+/** The four directories `tensorrt-llm-runtime`'s isolation and SELinux requirements name by role. */
+export interface ModelContainerMounts {
+    /** Read-only: the model's weight files. */
+    model: DockerMountSource;
+    /** Read-write: the engine's build cache, keyed by descriptor + model, kept across loads. */
+    engineCache: DockerMountSource;
+    /** Read-only: the watchdog entrypoint script core embeds and writes to disk (task 2.9). */
+    entrypoint: DockerMountSource;
+    /** Read-only: the heartbeat file core refreshes from the host side (task 2.9/2.12). */
+    heartbeat: DockerMountSource;
+}
+/** Discovery-only labels (never an authority for "is this ours"; the execution journal, task 2.10, is). */
+export interface ModelContainerLabels {
+    engine_id: string;
+    scope: string;
+    instance_id: string;
+}
+/**
+ * The numeric user and group a model container runs as (`--user <uid>:<gid>`): the core's own, so
+ * everything the engine writes into the read-write engine cache stays owned by — and removable by —
+ * the user who owns the data folder (final review I-1). Numeric only: the image has no passwd entry
+ * for this user, which is why the lifecycle also sets `HOME`/`USER`/`LOGNAME` for it.
+ */
+export interface ContainerUser {
+    uid: number;
+    gid: number;
+}
+/** Everything `buildCreateModelContainerArgv` needs to build one model container's `docker create` argv. */
+export interface ModelContainerCreateSpec {
+    image: ImageRef;
+    /** The NVIDIA GPU UUID this container is pinned to (`--gpus device=<uuid>`); never more than one card. */
+    gpuUuid: string;
+    /** Whether this Docker installation runs with SELinux enforcing (`docker info` `SecurityOptions`, design D15). */
+    selinux: boolean;
+    /**
+     * The one directory this executor owns and may `:z`-relabel. Required when `selinux` is true: every
+     * mount source must be it, or under it (spec "mounts work under SELinux" — never relabel outside
+     * this executor's own data, review round 1 item 1). Ignored when `selinux` is false.
+     */
+    selinuxDataRoot?: string;
+    mounts: ModelContainerMounts;
+    publication: HostPublication;
+    labels: ModelContainerLabels;
+    /** Defaults to `MODEL_CONTAINER_SHM_SIZE`; overridable for tests and for the eventual live-measured value. */
+    shmSize?: string;
+    /** `--user <uid>:<gid>`; absent, the image's own user (root for the NGC TensorRT-LLM release image). */
+    user?: ContainerUser;
+    env?: Record<string, string>;
+    command?: string[];
+}
+/** A one-shot `docker run --rm` probe: no mounts, no port publication (task 2.x's GPU check). */
+export interface OneShotRunSpec {
+    image: ImageRef;
+    gpuUuid?: string;
+    env?: Record<string, string>;
+    command?: string[];
+}
+/**
+ * Resolves symlinks/`.`/`..` to a canonical absolute path (`node:fs/promises`'s `realpath` in
+ * production; a test injects a fake). `operations.ts`'s `createContainer` runs every mount source and
+ * `selinuxDataRoot` through one of these before building argv (review round 2, item 2 — controller
+ * ruling): `argv.ts`'s `assertMountSource`/`assertWithinDataRoot` are lexical string checks, so a
+ * symlink *inside* an allowed data root that points *outside* it would pass them unresolved, while
+ * Docker's `:z` (and the bind mount itself) act on the resolved target, not the symlink's own path.
+ */
+export type Realpath = (path: string) => Promise<string>;
+/** What one docker CLI invocation answered. `code: null` means it never got as far as an exit code. */
+export interface DockerCommandResult {
+    code: number | null;
+    stdout: string;
+    stderr: string;
+}
+/** Per-call overrides a caller may pass to one `DockerExec` invocation (review round 1, item 3). */
+export interface DockerExecCallOptions {
+    /** Overrides the exec's own default deadline for just this call — `stopContainer` extends it past
+     *  `--time <timeoutSeconds>` so a legitimately slow stop is not mistaken for an unconfirmed one. */
+    timeoutMs?: number;
+}
+/**
+ * Runs one already-built argv against the real `docker`, or a fake, and reports what happened. The
+ * second parameter is additive (review round 1, item 3): an existing fake typed
+ * `(args: string[]) => Promise<DockerCommandResult>` — ignoring a second argument — remains a valid
+ * `DockerExec` with no change.
+ */
+export type DockerExec = (args: string[], options?: DockerExecCallOptions) => Promise<DockerCommandResult>;
+/**
+ * Whether a stop is safe to treat as done. `confirmed: false` covers both "docker answered with an
+ * error we don't recognize" and "we never got an answer at all" (our own exec call's timeout is not
+ * evidence the container stopped) — later tasks map an unconfirmed stop to `MANAGED_STOP_UNCONFIRMED`
+ * and keep holding the GPU reservation.
+ */
+export type StopOutcome = {
+    confirmed: true;
+    status: 'exited' | 'absent';
+} | {
+    confirmed: false;
+    reason: string;
+};
+/** Bytes pulled so far, summed across every image layer that has reported a size. */
+export interface PullProgress {
+    current: number;
+    total: number;
+}
+export type PullProgressCallback = (progress: PullProgress) => void;

package/dist/runtime/container/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/runtime/container/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/runtime/container/types.ts"],"names":[],"mappings":""}

package/dist/runtime/container/watchdog.d.ts

@@ -0,0 +1,114 @@
+/**
+ * The watchdog entrypoint for a `tensorrt-llm` model container (openspec change
+ * `add-tensorrt-llm-linux`, task 2.9, spec `tensorrt-llm-runtime` ▸ "Watchdog гарантирует
+ * освобождение GPU"). Docker gives it PID 1 in place of `trtllm-serve`: it starts the engine, then
+ * polls a heartbeat file the core keeps touching (see `heartbeat.ts`) from outside the container.
+ * If that file stops changing, the core is presumed dead (crashed, killed, machine slept) and the
+ * engine is torn down so the GPU is freed even though nothing asked for that on purpose. The
+ * requirement is explicit that this cannot be turned off by a setting: there is no config path that
+ * skips the poll loop, only the timing knobs below.
+ *
+ * For the script to actually be able to do this, the executor (task 2.8/2.12) MUST run it as the
+ * container's PID 1 — directly, or under `--init` — and MUST NOT run the container with
+ * `--pid=host`: a watchdog that is not PID 1 does not own the process tree Docker tears down when
+ * the container exits, and `trtllm-serve`'s own worker processes would then survive the watchdog's
+ * own exit instead of dying with it.
+ *
+ * The script is embedded as a plain POSIX `/bin/sh` string rather than loaded from a file at
+ * runtime, because the core must stay runtime-agnostic (no `import.meta.url`-relative loading) and
+ * because the container never gets write access to it — `writeWatchdogScript` below is the only way
+ * it reaches disk, mode `0555`, for the executor to bind-mount read-only next to the model.
+ *
+ * Docker argv/mount wiring, the executor that invokes this, and the lifecycle that runs the
+ * heartbeat ticker against a real container are out of scope here (tasks 2.8, 2.11, 2.12); this
+ * module only provides the script text, the function that writes it to a given path, and
+ * `watchdogEnv`, the one place that turns the timing constants into the env vars the script reads —
+ * callers should use it rather than building that env object by hand, so a future change to a
+ * default only has to happen here.
+ */
+/** How often the core touches the heartbeat file. */
+export declare const DEFAULT_HEARTBEAT_INTERVAL_SECS = 5;
+/** How long the heartbeat file may go unchanged before the watchdog kills the engine. */
+export declare const DEFAULT_WATCHDOG_STALE_LIMIT_SECS = 30;
+/** How often the watchdog polls the heartbeat file. */
+export declare const DEFAULT_WATCHDOG_POLL_INTERVAL_SECS = 5;
+/** TERM→KILL grace the watchdog gives the engine once it has decided to stop it. */
+export declare const DEFAULT_WATCHDOG_KILL_GRACE_SECS = 10;
+export declare const WATCHDOG_ENV_VARS: {
+    readonly heartbeatFile: "ATOMIC_WATCHDOG_HEARTBEAT_FILE";
+    readonly staleLimitSecs: "ATOMIC_WATCHDOG_STALE_LIMIT_SECS";
+    readonly pollIntervalSecs: "ATOMIC_WATCHDOG_POLL_INTERVAL_SECS";
+    readonly killGraceSecs: "ATOMIC_WATCHDOG_KILL_GRACE_SECS";
+};
+export interface WatchdogEnvOptions {
+    /** Path to the heartbeat file inside the container; no default, since it has no sane one. */
+    heartbeatFile: string;
+    staleLimitSecs?: number;
+    pollIntervalSecs?: number;
+    killGraceSecs?: number;
+}
+/**
+ * Builds the `ATOMIC_WATCHDOG_*` env vars the script needs, filling in the timing constants above
+ * for anything the caller does not override. This is the single place that turns those constants
+ * into the wire form the script reads — the executor/lifecycle MUST build the container's env
+ * through this function rather than re-deriving `ATOMIC_WATCHDOG_POLL_INTERVAL_SECS` and friends
+ * itself, so the three timing knobs never drift out of sync with the ticker's own default interval.
+ */
+export declare function watchdogEnv(options: WatchdogEnvOptions): Record<string, string>;
+/** The script's own config was missing or malformed (missing env var, missing `--` separator, a
+ *  timing value that is not a positive integer). */
+export declare const WATCHDOG_EXIT_CODE_CONFIG_ERROR = 96;
+/** The heartbeat observation (an mtime, or "missing") held unchanged for the stale threshold. */
+export declare const WATCHDOG_EXIT_CODE_STALE_HEARTBEAT = 97;
+/** Suggested filename for `writeWatchdogScript`'s target; the executor decides the actual path. */
+export declare const WATCHDOG_SCRIPT_FILENAME = "atomic-watchdog-entrypoint.sh";
+/** Mode the script is written with: readable and executable, never writable, not even by owner. */
+export declare const WATCHDOG_SCRIPT_MODE = 365;
+/** The watchdog entrypoint script text, ready to write to disk verbatim. */
+export declare const WATCHDOG_SCRIPT: string;
+/** The slice of a `fs.Stats` the no-op check needs — real `lstat` results satisfy this structurally. */
+export interface WatchdogScriptLstat {
+    isFile(): boolean;
+    mode: number;
+}
+/** The slice of `node:fs/promises` `writeWatchdogScript` needs; tests pass an in-memory fake. */
+export interface WatchdogScriptFs {
+    mkdir(path: string, options: {
+        recursive: boolean;
+    }): Promise<string | undefined>;
+    writeFile(path: string, data: string, options?: {
+        flag?: string;
+        mode?: number;
+    }): Promise<void>;
+    chmod(path: string, mode: number): Promise<void>;
+    rename(from: string, to: string): Promise<void>;
+    /** Cleans up an abandoned temp file after a failed `chmod`/`rename`; failures are swallowed. */
+    rm(path: string): Promise<void>;
+    /**
+     * `lstat`, not `stat`: a symlink at `path` must never read as "already current" through the file
+     * it points at, or `writeWatchdogScript` would leave the symlink in place instead of replacing it
+     * with a real, independently-owned regular file.
+     */
+    lstat(path: string): Promise<WatchdogScriptLstat>;
+    /** Used only to compare content once `lstat` has already confirmed a regular file at `0555`. */
+    readFile(path: string, encoding: 'utf8'): Promise<string>;
+}
+/**
+ * Writes the watchdog entrypoint script to `path`, atomically and idempotently.
+ *
+ * A second call on a path that already holds the current script at the right mode (the common case:
+ * the executor calls this on every container start, not only the first) is a no-op — worth checking,
+ * since the file is `0555` and a second `writeFile` straight at that path would fail `EACCES`. When
+ * it is not already current, the script is written to a temp file in the same directory (created
+ * exclusively, so two writers never interleave into the same temp name), locked to `0555`, then
+ * `rename`d over the target: a rename replaces whatever is at the destination — including a symlink
+ * itself, not whatever it points at — in one atomic step, so a container that is mid-read of the old
+ * script never sees a half-written file, and nothing already running with the old copy open loses it
+ * out from under itself. If `chmod` or `rename` fails partway through, the abandoned temp file is
+ * removed rather than left behind for the next call (or a directory listing) to trip over.
+ *
+ * The executor (task 2.8/2.12) bind-mounts the result into the model container read-only and passes
+ * it as the container's entrypoint; this function has no opinion on where `path` lives — that is the
+ * core's per-scope data directory, chosen by the caller.
+ */
+export declare function writeWatchdogScript(path: string, fs?: WatchdogScriptFs, host?: NodeJS.Platform): Promise<string>;