@infersec/conduit 1.112.0 → 1.113.0

package/dist/cli.js

@@ -19972,6 +19972,32 @@ object$5({
     sizeBytes: number$1().int().nonnegative().nullable()
 });
 
+const EngineExecutionErrorTypeSchema = _enum$1([
+    "config",
+    "crash",
+    "oom",
+    "prompt",
+    "unknown"
+]);
+const EngineExecutionReportPayloadSchema = object$5({
+    avgTps: number$1().nonnegative().finite().default(0),
+    completionTokens: number$1().int().nonnegative().default(0),
+    durationMs: number$1().int().nonnegative().default(0),
+    engineType: LLMEngineSchema.nullable(),
+    engineVersion: string$2().max(64).nullable().default(null),
+    errorDetail: string$2().max(2048).nullable().default(null),
+    errorType: EngineExecutionErrorTypeSchema.nullable(),
+    extraArgs: array$1(tuple([string$2().min(1).max(128), string$2().max(512)]))
+        .max(256)
+        .default([]),
+    finishedAtISO: string$2().datetime({ offset: true }),
+    peakTps: number$1().nonnegative().finite().nullable().default(null),
+    promptTokens: number$1().int().nonnegative().default(0),
+    runAtISO: string$2().datetime({ offset: true }),
+    success: boolean$1(),
+    ttftMs: number$1().int().nonnegative().default(0),
+    totalTokens: number$1().int().nonnegative().default(0)
+});
 const InferenceAgentLLMMetricsPayloadSchema = object$5({
     bytes: number$1().int().nonnegative(),
     completionTokens: number$1().int().nonnegative(),
@@ -20310,6 +20336,23 @@ const API_SERVICE_CONDUIT_API_REFERENCE = {
             }
         }
     },
+    "/conduit/api/v1/source/:sourceID/engine/execution": {
+        POST: {
+            auth: {
+                type: "api-key"
+            },
+            body: EngineExecutionReportPayloadSchema,
+            parameters: {
+                sourceID: ULIDSchema
+            },
+            response: {
+                schema: object$5({
+                    acknowledged: literal(true)
+                }),
+                type: "rest"
+            }
+        }
+    },
     "/conduit/api/v1/source/:sourceID/requests/:requestID/chunk": {
         POST: {
             auth: {
@@ -113767,6 +113810,19 @@ function createAPIClient({ apiKey, apiURL, inferenceSourceID, logger }) {
                 route: "/conduit/api/v1/source/:sourceID/state"
             });
         },
+        reportEngineExecution: async (payload) => {
+            await fetchByReference({
+                baseURL: apiURL,
+                body: payload,
+                fetch: fetchWithAPIKey,
+                method: "POST",
+                parameters: {
+                    sourceID: inferenceSourceID
+                },
+                reference: API_SERVICE_CONDUIT_API_REFERENCE,
+                route: "/conduit/api/v1/source/:sourceID/engine/execution"
+            });
+        },
         reportPromptMetrics: async (payload) => {
             await fetchByReference({
                 baseURL: apiURL,
@@ -125627,6 +125683,9 @@ class ModelManager extends EventEmitter {
     lifecycleState = "stopped";
     downloadLockHandle = null;
     stopRequested = false;
+    lastEngineExitCode = null;
+    lastEngineExitSignal = null;
+    reachedRunningState = false;
     modelsDirectory;
     constructor({ contextLength, engineConfig, enginePort, engineType, logger, model, root }) {
         super();
@@ -125749,6 +125808,9 @@ class ModelManager extends EventEmitter {
         this.lifecycleState = "starting";
         this.lastEngineError = null;
         this.stopRequested = false;
+        this.lastEngineExitCode = null;
+        this.lastEngineExitSignal = null;
+        this.reachedRunningState = false;
         this.logger.info("Starting LLM engine", {
             agentEngineType: this.engine
         });
@@ -125779,6 +125841,7 @@ class ModelManager extends EventEmitter {
             throw err;
         }
         this.lifecycleState = "running";
+        this.reachedRunningState = true;
         this.emit("engineReady");
     }
     async stop() {
@@ -125803,6 +125866,7 @@ class ModelManager extends EventEmitter {
             this.lifecycleState = "stopped";
             return;
         }
+        this.reachedRunningState = false;
         this.lifecycleState = "stopping";
         this.stopRequested = true;
         await processManager.stop();
@@ -125815,9 +125879,18 @@ class ModelManager extends EventEmitter {
             this.lifecycleState === "starting" ||
             this.lifecycleState === "errored");
     }
+    get lastExitCode() {
+        return this.lastEngineExitCode;
+    }
+    get lastExitSignal() {
+        return this.lastEngineExitSignal;
+    }
     get state() {
         return this.lifecycleState;
     }
+    get wasRunning() {
+        return this.reachedRunningState;
+    }
     async checkEngineReadiness() {
         switch (this.engine) {
             case "llama.cpp": {
@@ -125931,6 +126004,7 @@ class ModelManager extends EventEmitter {
                 if (readiness === "ready") {
                     this.clearHealthPoll();
                     this.lifecycleState = "running";
+                    this.reachedRunningState = true;
                     this.emit("engineReady");
                 }
             })
@@ -125993,6 +126067,8 @@ class ModelManager extends EventEmitter {
             }));
         });
         processManager.on("stopped", (code, signal) => {
+            this.lastEngineExitCode = code;
+            this.lastEngineExitSignal = signal;
             if (hasTerminated) {
                 return;
             }
@@ -126060,6 +126136,69 @@ class ModelManager extends EventEmitter {
     }
 }
 
+// Ordered most-specific first: a message mentioning a prompt-size failure should classify as
+// "prompt" even if it also touches memory text; memory outranks config because OOM kills frequently
+// emit sparse stderr. These are heuristics, not exhaustively enumerated engine vocabularies.
+const OOM_PATTERNS = [
+    /CUDA out of memory/i,
+    /No available memory for the cache blocks/i,
+    /\bOOMKilled\b/,
+    /out of memory/i,
+    /MemoryError/i,
+    /Cannot allocate memory/i
+];
+const PROMPT_PATTERNS = [
+    /prompt is too long/i,
+    /maximum context length exceeded/i,
+    /too many tokens/i
+];
+const CONFIG_PATTERNS = [
+    /unrecognized argument/i,
+    /invalid argument/i,
+    /error while loading state_dict/i,
+    /Architecture not understood/i,
+    /No such file or directory/i
+];
+/**
+ * Coarse engine-failure classification used only to fill `engine_execution.error_type`. Exit code and
+ * terminating signal are consulted FIRST (SIGKILL/SIGSEGV are reliable OOM signals regardless of
+ * how much stderr the engine produced); the message text then refines the category. Anything
+ * unrecognized is "unknown".
+ */
+function classifyEngineFailure({ error, exitCode, signal }) {
+    // 137 = SIGKILL (kernel OOM-killer), 139 = SIGSEGV (illegal memory access). Treat both as OOM
+    // so that memory-pressure deaths do not degrade to "unknown" when stderr is truncated.
+    if (exitCode === 137 || exitCode === 139) {
+        return "oom";
+    }
+    // Direct OS kills (SIGKILL/SIGSEGV) leave no meaningful exit code; honor them like 137/139.
+    if (signal === "SIGKILL" || signal === "SIGSEGV") {
+        return "oom";
+    }
+    const text = `${error.message}`.slice(0, 4000);
+    for (const pattern of PROMPT_PATTERNS) {
+        if (pattern.test(text)) {
+            return "prompt";
+        }
+    }
+    for (const pattern of OOM_PATTERNS) {
+        if (pattern.test(text)) {
+            return "oom";
+        }
+    }
+    for (const pattern of CONFIG_PATTERNS) {
+        if (pattern.test(text)) {
+            return "config";
+        }
+    }
+    // A process that exited abnormally with no recognizable diagnostic: treat as a crash rather
+    // than "unknown" so the two buckets distinguish "we saw nothing" from "it died badly".
+    if (exitCode !== null && exitCode !== 0) {
+        return "crash";
+    }
+    return "unknown";
+}
+
 const EXCEPTION_LINE_PATTERN = /([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception)):\s*(.+)/;
 const FALLBACK_DETAIL_MAX_LENGTH = 300;
 const FALLBACK_RAW_MAX_LENGTH = 500;
@@ -137936,12 +138075,20 @@ async function detectVRAMViaSysfs(pciBusSuffix) {
             const deviceDir = path$1.join(DRM_PATH, entry, "device");
             const totalStr = await readSysfsFile(path$1.join(deviceDir, "mem_info_vram_total"));
             const usedStr = await readSysfsFile(path$1.join(deviceDir, "mem_info_vram_used"));
+            const gttTotalStr = await readSysfsFile(path$1.join(deviceDir, "mem_info_gtt_total"));
+            const gttUsedStr = await readSysfsFile(path$1.join(deviceDir, "mem_info_gtt_used"));
             const totalBytes = totalStr !== null ? parseInt(totalStr, 10) : null;
             const usedBytes = usedStr !== null ? parseInt(usedStr, 10) : null;
+            const gttTotalBytes = gttTotalStr !== null ? parseInt(gttTotalStr, 10) : null;
+            const gttUsedBytes = gttUsedStr !== null ? parseInt(gttUsedStr, 10) : null;
             const validTotal = totalBytes !== null && Number.isFinite(totalBytes) && totalBytes >= 0;
             const validUsed = usedBytes !== null && Number.isFinite(usedBytes) && usedBytes >= 0;
+            const validGttTotal = gttTotalBytes !== null && Number.isFinite(gttTotalBytes) && gttTotalBytes >= 0;
+            const validGttUsed = gttUsedBytes !== null && Number.isFinite(gttUsedBytes) && gttUsedBytes >= 0;
             if (validTotal) {
                 return {
+                    gttTotalBytes: validGttTotal ? gttTotalBytes : null,
+                    gttUsedBytes: validGttUsed ? gttUsedBytes : null,
                     memoryTotalBytes: totalBytes,
                     memoryUsedBytes: validUsed ? usedBytes : null
                 };
@@ -137951,36 +138098,74 @@ async function detectVRAMViaSysfs(pciBusSuffix) {
     catch {
         // sysfs not available
     }
-    return { memoryTotalBytes: null, memoryUsedBytes: null };
+    return {
+        gttTotalBytes: null,
+        gttUsedBytes: null,
+        memoryTotalBytes: null,
+        memoryUsedBytes: null
+    };
+}
+// rocm-smi accepts a single --showmeminfo type per invocation, so VRAM and GTT
+// arrive from separate calls; both share this per-card parser.
+function parseRocmSmiMemory({ keyPrefix, stdout }) {
+    const parsed = JSON.parse(stdout);
+    const results = [];
+    const cards = Object.entries(parsed).filter(([key]) => key.startsWith("card"));
+    for (const [, data] of cards) {
+        const bus = data["PCI Bus"] ?? null;
+        const totalStr = data[`${keyPrefix} Total Memory (B)`] ?? null;
+        const usedStr = data[`${keyPrefix} Total Used Memory (B)`] ?? null;
+        const totalBytes = totalStr !== null ? parseInt(totalStr, 10) : null;
+        const usedBytes = usedStr !== null ? parseInt(usedStr, 10) : null;
+        const validTotal = totalBytes !== null && Number.isFinite(totalBytes) && totalBytes >= 0;
+        const validUsed = usedBytes !== null && Number.isFinite(usedBytes) && usedBytes >= 0;
+        if (bus) {
+            results.push({
+                bus,
+                memoryTotalBytes: validTotal ? totalBytes : null,
+                memoryUsedBytes: validUsed ? usedBytes : null
+            });
+        }
+    }
+    return results;
 }
-async function detectVRAMViaRocmSmi() {
+const ROCM_SMI_TIMEOUT_MS = 10_000;
+async function detectVRAMViaRocmSmi({ logger }) {
     try {
-        const { stdout } = await execa("rocm-smi", [
-            "--showbus",
-            "--showmeminfo",
-            "vram",
-            "--json"
+        const [vramResult, gttResult] = await Promise.allSettled([
+            execa("rocm-smi", ["--showbus", "--showmeminfo", "vram", "--json"], {
+                timeout: ROCM_SMI_TIMEOUT_MS
+            }),
+            execa("rocm-smi", ["--showbus", "--showmeminfo", "gtt", "--json"], {
+                timeout: ROCM_SMI_TIMEOUT_MS
+            })
         ]);
-        const parsed = JSON.parse(stdout);
-        const results = [];
-        const cards = Object.entries(parsed).filter(([key]) => key.startsWith("card"));
-        for (const [, data] of cards) {
-            const bus = data["PCI Bus"] ?? null;
-            const totalStr = data["VRAM Total Memory (B)"] ?? null;
-            const usedStr = data["VRAM Total Used Memory (B)"] ?? null;
-            const totalBytes = totalStr !== null ? parseInt(totalStr, 10) : null;
-            const usedBytes = usedStr !== null ? parseInt(usedStr, 10) : null;
-            const validTotal = totalBytes !== null && Number.isFinite(totalBytes) && totalBytes >= 0;
-            const validUsed = usedBytes !== null && Number.isFinite(usedBytes) && usedBytes >= 0;
-            if (bus) {
-                results.push({
-                    bus,
-                    memoryTotalBytes: validTotal ? totalBytes : null,
-                    memoryUsedBytes: validUsed ? usedBytes : null
+        if (vramResult.status !== "fulfilled")
+            return [];
+        let gttEntries = [];
+        if (gttResult.status === "fulfilled") {
+            try {
+                gttEntries = parseRocmSmiMemory({
+                    keyPrefix: "GTT",
+                    stdout: gttResult.value.stdout
                 });
             }
+            catch (error) {
+                // Unusable GTT output (e.g. older rocm-smi) degrades to VRAM-only
+                logger.warn("rocm-smi GTT output parse failed", { error: asError(error) });
+            }
         }
-        return results;
+        const gttByBus = new Map(gttEntries.map(entry => [entry.bus, entry]));
+        return parseRocmSmiMemory({ keyPrefix: "VRAM", stdout: vramResult.value.stdout }).map(vram => {
+            const gtt = gttByBus.get(vram.bus);
+            return {
+                bus: vram.bus,
+                gttTotalBytes: gtt?.memoryTotalBytes ?? null,
+                gttUsedBytes: gtt?.memoryUsedBytes ?? null,
+                memoryTotalBytes: vram.memoryTotalBytes,
+                memoryUsedBytes: vram.memoryUsedBytes
+            };
+        });
     }
     catch {
         return [];
@@ -138045,8 +138230,9 @@ async function detectGPUsViaNvidiaSmi({ logger }) {
                 vendor: "NVIDIA"
             });
         }
-        // Unified-memory devices (e.g. NVIDIA GB10 on DGX Spark) report "[N/A]" for
-        // memory. Their compute pool is system RAM, so fall back to /proc/meminfo.
+        // Shared/unified-memory devices (e.g. NVIDIA GB10 on DGX Spark) report
+        // "[N/A]" for memory. Their compute pool is system RAM, so fall back to
+        // /proc/meminfo. (AMD iGPUs get the same treatment via GTT merging.)
         if (gpus.some(gpu => gpu.memoryTotalBytes === null)) {
             const systemMemory = await readSystemMemoryBytes({ logger });
             const total = systemMemory.totalBytes;
@@ -138070,7 +138256,7 @@ async function detectGPUsViaNvidiaSmi({ logger }) {
     }
 }
 function buildMergedGPUs(options) {
-    const { lspciGPUs, nvidiaGPUs, rocmVRAM, siGPUs, sysfsVRAMMap } = options;
+    const { lspciGPUs, nvidiaGPUs, rocmVRAM, siGPUs, systemTotalBytes, sysfsVRAMMap } = options;
     const rocmByBus = new Map();
     for (const entry of rocmVRAM) {
         const key = normalizeBusAddress(entry.bus);
@@ -138111,6 +138297,7 @@ function buildMergedGPUs(options) {
                     gpu: existing,
                     key,
                     rocmByBus,
+                    systemTotalBytes,
                     sysfsVRAMMap
                 });
             }
@@ -138130,30 +138317,73 @@ function buildMergedGPUs(options) {
             gpu,
             key,
             rocmByBus,
+            systemTotalBytes,
             sysfsVRAMMap
         });
         byBus.set(key, gpu);
     }
     return [...byBus.values()];
 }
-function applySysfsOrRocmVRAM({ gpu, key, rocmByBus, sysfsVRAMMap }) {
+// Shared-memory GPUs (AMD iGPUs) expose a small dedicated VRAM carve-out via
+// mem_info_vram_total while the real usable pool - GTT - is carved dynamically
+// from system RAM. When GTT exceeds VRAM the device is treated as integrated
+// and the two pools combine (capped by installed RAM). Note that GTT defaults
+// to half of system RAM even on discrete GPUs, so a dGPU with less VRAM than
+// that also merges - acceptable, since GTT remains real addressable memory
+// for amdgpu compute (albeit slower over PCIe).
+function mergeGTTMemory({ gttTotalBytes, gttUsedBytes, systemTotalBytes, vramTotalBytes, vramUsedBytes }) {
+    if (vramTotalBytes === null || !Number.isFinite(vramTotalBytes)) {
+        return { memoryTotalBytes: null, memoryUsedBytes: null };
+    }
+    const isIntegrated = gttTotalBytes !== null && gttTotalBytes > vramTotalBytes;
+    if (!isIntegrated) {
+        return { memoryTotalBytes: vramTotalBytes, memoryUsedBytes: vramUsedBytes };
+    }
+    const combined = vramTotalBytes + gttTotalBytes;
+    const capped = systemTotalBytes !== null && systemTotalBytes > 0
+        ? Math.min(combined, systemTotalBytes)
+        : combined;
+    const hasCompleteUsage = vramUsedBytes !== null &&
+        Number.isFinite(vramUsedBytes) &&
+        gttUsedBytes !== null &&
+        Number.isFinite(gttUsedBytes);
+    const memoryUsedBytes = hasCompleteUsage
+        ? Math.min(vramUsedBytes + gttUsedBytes, capped)
+        : null;
+    return { memoryTotalBytes: capped, memoryUsedBytes };
+}
+function applySysfsOrRocmVRAM({ gpu, key, rocmByBus, systemTotalBytes, sysfsVRAMMap }) {
     const sysfs = sysfsVRAMMap.get(key);
-    let totalBytes = sysfs?.memoryTotalBytes ?? null;
-    let usedBytes = sysfs?.memoryUsedBytes ?? null;
-    if (totalBytes === null) {
+    let vramTotalBytes = sysfs?.memoryTotalBytes ?? null;
+    let vramUsedBytes = sysfs?.memoryUsedBytes ?? null;
+    let gttTotalBytes = sysfs?.gttTotalBytes ?? null;
+    let gttUsedBytes = sysfs?.gttUsedBytes ?? null;
+    if (vramTotalBytes === null) {
         const rocm = rocmByBus.get(key);
         if (rocm) {
-            totalBytes = rocm.memoryTotalBytes;
-            usedBytes = rocm.memoryUsedBytes;
-        }
-    }
-    if (totalBytes === null || !Number.isFinite(totalBytes))
+            vramTotalBytes = rocm.memoryTotalBytes;
+            vramUsedBytes = rocm.memoryUsedBytes;
+            gttTotalBytes = rocm.gttTotalBytes;
+            gttUsedBytes = rocm.gttUsedBytes;
+        }
+    }
+    const merged = mergeGTTMemory({
+        gttTotalBytes,
+        gttUsedBytes,
+        systemTotalBytes,
+        vramTotalBytes,
+        vramUsedBytes
+    });
+    if (merged.memoryTotalBytes === null || !Number.isFinite(merged.memoryTotalBytes))
         return;
-    gpu.memoryTotalBytes = totalBytes;
-    gpu.memoryUsedBytes = usedBytes !== null && Number.isFinite(usedBytes) ? usedBytes : null;
+    gpu.memoryTotalBytes = merged.memoryTotalBytes;
+    gpu.memoryUsedBytes =
+        merged.memoryUsedBytes !== null && Number.isFinite(merged.memoryUsedBytes)
+            ? merged.memoryUsedBytes
+            : null;
     gpu.memoryFreeBytes =
-        usedBytes !== null && Number.isFinite(usedBytes)
-            ? Math.max(totalBytes - usedBytes, 0)
+        gpu.memoryUsedBytes !== null
+            ? Math.max(gpu.memoryTotalBytes - gpu.memoryUsedBytes, 0)
             : null;
 }
 async function collectMachineMetadata({ logger }) {
@@ -138164,7 +138394,7 @@ async function collectMachineMetadata({ logger }) {
         si.graphics(),
         detectGPUsViaLspci(),
         detectGPUsViaNvidiaSmi({ logger }),
-        detectVRAMViaRocmSmi()
+        detectVRAMViaRocmSmi({ logger })
     ]);
     const cpuInfo = cpuResult.status === "fulfilled" ? cpuResult.value : null;
     const memInfo = memResult.status === "fulfilled" ? memResult.value : null;
@@ -138219,6 +138449,7 @@ async function collectMachineMetadata({ logger }) {
         nvidiaGPUs: resolvedNvidiaGPUs,
         rocmVRAM: resolvedRocmVRAM,
         siGPUs,
+        systemTotalBytes: memInfo?.total ?? null,
         sysfsVRAMMap
     });
     const machineMetadata = {
@@ -138279,6 +138510,119 @@ async function detectDockerVersion() {
     }
 }
 
+/**
+ * Flattens flat CLI extra-arg tokens into [arg, value] pairs, sorted by ARG NAME (ascending, ties by
+ * value). `--flag=value` pairs split on the first `=`; a bare `--flag` consumes the following token
+ * as its value when that token does not start with "-" (classic CLI convention); anything else
+ * (flags, non-strings) is dropped.
+ */
+function pairExtraArgs(tokens) {
+    if (!Array.isArray(tokens)) {
+        return [];
+    }
+    const list = tokens;
+    const pairs = [];
+    let index = 0;
+    while (index < list.length) {
+        const token = list[index];
+        if (typeof token !== "string" || token.length === 0 || !token.startsWith("-")) {
+            index++;
+            continue;
+        }
+        const separator = token.indexOf("=");
+        if (separator > -1) {
+            const arg = token.slice(0, separator);
+            if (arg.length > 0) {
+                pairs.push([arg, token.slice(separator + 1)]);
+            }
+            index++;
+            continue;
+        }
+        const next = list[index + 1];
+        const consumesNext = typeof next === "string" && next.length > 0 && !next.startsWith("-");
+        if (consumesNext) {
+            pairs.push([token, next]);
+            index += 2;
+        }
+        else {
+            pairs.push([token, ""]);
+            index++;
+        }
+    }
+    return pairs.sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0);
+}
+/**
+ * Files AT MOST ONE engine_execution report per engine startup. `beginStartup(runAt)` re-arms the
+ * latch on every fresh model start (initial boot or cycle) and stamps the startup epoch used as the
+ * row's `run_at`. Whichever of the three report* paths fires first wins: startup failure, in-flight
+ * crash, or first successful full prompt completion.
+ */
+class EngineExecutionReporter {
+    options;
+    currentStartupAt = null;
+    reportedForCurrentStartup = false;
+    constructor(options) {
+        this.options = options;
+    }
+    /** Re-arms the latch; the stamp becomes the row's `run_at` (moment startup began). */
+    beginStartup(runAt) {
+        this.currentStartupAt = runAt;
+        this.reportedForCurrentStartup = false;
+    }
+    /** Reports a startup failure (rejected prepare/start, readiness timeout, pre-ready death). */
+    async reportStartupFailure(report) {
+        await this.file(report, false);
+    }
+    /** Reports a spontaneous crash of an engine that had reached the running state. */
+    async reportRuntimeCrash(report) {
+        await this.file(report, false);
+    }
+    /** Reports the first fully-responded, token-bearing prompt completion since startup. */
+    async reportSuccess(report) {
+        await this.file(report, true);
+    }
+    async file(report, success) {
+        if (this.reportedForCurrentStartup || this.currentStartupAt === null) {
+            return;
+        }
+        // Latch BEFORE the network call: a thrown POST cannot double-file for this startup.
+        this.reportedForCurrentStartup = true;
+        const context = this.options.buildContext();
+        const payload = {
+            avgTps: report.throughput.avgTps,
+            completionTokens: report.usage.completionTokens,
+            durationMs: report.durationMs,
+            engineType: context.engineType,
+            engineVersion: context.engineVersion,
+            errorDetail: success ? null : report.errorDetail,
+            errorType: success ? null : report.errorType,
+            extraArgs: context.extraArgsPairs,
+            finishedAtISO: new Date().toISOString(),
+            peakTps: report.throughput.peakTps,
+            promptTokens: report.usage.promptTokens,
+            runAtISO: this.currentStartupAt.toISOString(),
+            success,
+            ttftMs: report.ttftMs,
+            totalTokens: report.usage.totalTokens
+        };
+        try {
+            await this.options.report(payload);
+            this.options.logger.info("Engine execution outcome reported", {
+                inferenceSourceID: this.options.sourceLabel,
+                success
+            });
+        }
+        catch (error) {
+            // Losing one report is preferable to filing two; the latch stays latched.
+            this.options.logger.warn("Failed to report engine execution outcome", {
+                error: asError(error),
+                inferenceSourceID: this.options.sourceLabel,
+                success
+            });
+        }
+    }
+}
+
 async function createApplication({ abortController, apiClient, configuration, logger }) {
     ensureDockerValidEnv();
     logger.info("Fetching conduit configuration");
@@ -138310,6 +138654,87 @@ async function createApplication({ abortController, apiClient, configuration, lo
             error: asError(error)
         });
     }
+    const reporter = new EngineExecutionReporter({
+        buildContext: () => {
+            const engineType = (conduitConfiguration.engineConfig?.type ??
+                "llama.cpp");
+            const versions = {
+                exllamav3: machine?.exllamav3Version ?? null,
+                "llama.cpp": machine?.llamaCppVersion ?? null,
+                "mlx-lm": machine?.mlxlmVersion ?? null,
+                sglang: machine?.sglangVersion ?? null,
+                "tensorrt-llm": machine?.tensorrtLlmVersion ?? null,
+                vllm: machine?.vllmVersion ?? null
+            };
+            return {
+                engineType,
+                engineVersion: versions[engineType] ?? null,
+                extraArgsPairs: pairExtraArgs(conduitConfiguration.engineConfig?.extraArgs)
+            };
+        },
+        logger,
+        report: payload => apiClient.reportEngineExecution(payload),
+        sourceLabel: configuration.inferenceSourceID
+    });
+    // Intercept the prompt-metrics chokepoint so the first fully-responded, token-bearing prompt of
+    // each fresh startup files the one-shot engine_execution success report. Handlers close over the
+    // SAME `apiClient` object and read `reportPromptMetrics` at request-dispatch time (which always
+    // follows this point), so the wrapped method is what they invoke.
+    const rawReportPromptMetrics = apiClient.reportPromptMetrics;
+    apiClient.reportPromptMetrics = async (payload) => {
+        if (payload.successful && payload.completionTokens > 0 && payload.latencyMs > 0) {
+            // The one-shot report is kicked off and its settlement attached HERE (before any await): if the
+            // metrics path throws below, the report promise must still be able to log its own rejection.
+            const successReport = reporter
+                .reportSuccess({
+                durationMs: payload.latencyMs,
+                errorDetail: null,
+                errorType: null,
+                throughput: {
+                    avgTps: payload.tokensPerSecond,
+                    peakTps: null
+                },
+                ttftMs: payload.timeToFirstTokenMs ?? 0,
+                usage: {
+                    completionTokens: payload.completionTokens,
+                    promptTokens: payload.promptTokens,
+                    totalTokens: payload.totalTokens
+                }
+            })
+                .catch(error => {
+                logger.warn("Engine execution success report failed", {
+                    error: asError(error)
+                });
+            });
+            await rawReportPromptMetrics(payload);
+            await successReport;
+            return;
+        }
+        await rawReportPromptMetrics(payload);
+    };
+    const SECRET_ARG_MASK_PATTERN = /(-{1,2}[A-Za-z0-9_.]*(?:api[-_]?key|hf[-_]?token|token)(?:\s+|[=:]))\S+/gi;
+    // Assembles the payload shared by the startup-failure and runtime-crash report paths. Stderr
+    // may echo secrets, so mask `--api-key`/token-looking args before they reach the DB.
+    function buildCrashReport(error, exitCode, signal) {
+        const classification = classifyEngineFailure({ error, exitCode, signal });
+        const raw = normalizeEngineError(error.message);
+        const masked = raw.replace(SECRET_ARG_MASK_PATTERN, "$1***");
+        return {
+            durationMs: 0,
+            errorDetail: masked.slice(0, 2048),
+            errorType: classification,
+            ttftMs: 0,
+            throughput: {
+                avgTps: 0,
+                peakTps: null
+            },
+            usage: {
+                completionTokens: 0,
+                promptTokens: 0,
+                totalTokens: 0
+            }
+        };
+    }
     const conduitStateManager = new ConduitStateManager({
         initialState: {
             state: "initialising"
@@ -138361,6 +138786,17 @@ async function createApplication({ abortController, apiClient, configuration, lo
             });
             stopRequestedByControl = false;
             setErrorState({ error: normalizeEngineError(err.message) });
+            // Spontaneous death of a SERVING engine → crash report, suppressed by the latch if the
+            // startup's one-shot outcome was already filed. Startup-path failures report from
+            // `startEngine`'s catch; this listener is the RUNTIME-crash path only.
+            if (modelManager.wasRunning && !err.message.includes("interrupted by stop request")) {
+                const crashReport = buildCrashReport(err, modelManager.lastExitCode, modelManager.lastExitSignal);
+                reporter.reportRuntimeCrash(crashReport).catch(crashReportError => {
+                    logger.warn("Engine execution crash report failed", {
+                        error: asError(crashReportError)
+                    });
+                });
+            }
         });
         modelManager.on("engineReady", () => {
             setOnlineState();
@@ -138420,24 +138856,40 @@ async function createApplication({ abortController, apiClient, configuration, lo
     };
     async function startEngine() {
         logger.info("Engine start requested");
-        conduitStateManager.setState({
-            modelFileName,
-            modelName,
-            state: "downloadingModelFiles",
-            totalProgress: {
-                file: 0,
-                total: 0
+        reporter.beginStartup(new Date());
+        try {
+            conduitStateManager.setState({
+                modelFileName,
+                modelName,
+                state: "downloadingModelFiles",
+                totalProgress: {
+                    file: 0,
+                    total: 0
+                }
+            });
+            await conduitStateReportManager.reportNow();
+            await modelManager.prepare({
+                onDownloadProgress: reportDownloadProgress
+            });
+            conduitStateManager.setState({
+                state: "bootingEngine"
+            });
+            await conduitStateReportManager.reportNow();
+            await modelManager.start();
+        }
+        catch (error) {
+            const parsedError = asError(error);
+            // Operator-initiated aborts are not startup failures worth reporting.
+            if (!parsedError.message.includes("interrupted by stop request")) {
+                const startupReport = buildCrashReport(parsedError, modelManager.lastExitCode, modelManager.lastExitSignal);
+                reporter.reportStartupFailure(startupReport).catch(startupReportError => {
+                    logger.warn("Engine execution startup report failed", {
+                        error: asError(startupReportError)
+                    });
+                });
             }
-        });
-        await conduitStateReportManager.reportNow();
-        await modelManager.prepare({
-            onDownloadProgress: reportDownloadProgress
-        });
-        conduitStateManager.setState({
-            state: "bootingEngine"
-        });
-        await conduitStateReportManager.reportNow();
-        await modelManager.start();
+            throw error;
+        }
     }
     async function stopEngine({ reason }) {
         if (!modelManager.canStop) {
@@ -342892,12 +343344,13 @@ async function runModelFit(options) {
     console.log();
     const feasible = filterFeasibleModels({ detection, models: recommendedModels });
     if (feasible.length === 0) {
+        const smallestTierGB = Math.min(...recommendedModels.map(model => model.vramTierGB));
         console.error("No recommended models fit this hardware. " +
             `Budget: ${formatBytes$1(detection.gpus.some(gpu => gpu.memoryTotalBytes)
                 ? Math.max(...detection.gpus
                     .map(gpu => gpu.memoryTotalBytes ?? 0)
                     .filter(bytes => bytes > 0))
-                : detection.memory.totalBytes)}. Smallest tier starts at 2 GB.`);
+                : detection.memory.totalBytes)}. Smallest tier starts at ${smallestTierGB} GB.`);
         process.exitCode = 1;
         return;
     }