@infersec/conduit 1.112.1 → 1.113.0

package/dist/apiClient/index.d.ts

@@ -1,9 +1,10 @@
-import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, InferenceAgentMachineReportPayload, ULID, type ConduitState } from "@infersec/definitions";
+import { EngineExecutionReportPayload, InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, InferenceAgentMachineReportPayload, ULID, type ConduitState } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 export interface APIClient {
     cycleInferenceSourceEngine: () => Promise<void>;
     getConduitConfiguration: () => Promise<InferenceAgentConfiguration>;
     reportConduitState: (state: ConduitState) => Promise<void>;
+    reportEngineExecution: (payload: EngineExecutionReportPayload) => Promise<void>;
     reportMachineMetadata: (payload: InferenceAgentMachineReportPayload) => Promise<void>;
     reportPromptMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
     updateMachineStatus: (status: {

package/dist/cli.js

@@ -19972,6 +19972,32 @@ object$5({
     sizeBytes: number$1().int().nonnegative().nullable()
 });
 
+const EngineExecutionErrorTypeSchema = _enum$1([
+    "config",
+    "crash",
+    "oom",
+    "prompt",
+    "unknown"
+]);
+const EngineExecutionReportPayloadSchema = object$5({
+    avgTps: number$1().nonnegative().finite().default(0),
+    completionTokens: number$1().int().nonnegative().default(0),
+    durationMs: number$1().int().nonnegative().default(0),
+    engineType: LLMEngineSchema.nullable(),
+    engineVersion: string$2().max(64).nullable().default(null),
+    errorDetail: string$2().max(2048).nullable().default(null),
+    errorType: EngineExecutionErrorTypeSchema.nullable(),
+    extraArgs: array$1(tuple([string$2().min(1).max(128), string$2().max(512)]))
+        .max(256)
+        .default([]),
+    finishedAtISO: string$2().datetime({ offset: true }),
+    peakTps: number$1().nonnegative().finite().nullable().default(null),
+    promptTokens: number$1().int().nonnegative().default(0),
+    runAtISO: string$2().datetime({ offset: true }),
+    success: boolean$1(),
+    ttftMs: number$1().int().nonnegative().default(0),
+    totalTokens: number$1().int().nonnegative().default(0)
+});
 const InferenceAgentLLMMetricsPayloadSchema = object$5({
     bytes: number$1().int().nonnegative(),
     completionTokens: number$1().int().nonnegative(),
@@ -20310,6 +20336,23 @@ const API_SERVICE_CONDUIT_API_REFERENCE = {
             }
         }
     },
+    "/conduit/api/v1/source/:sourceID/engine/execution": {
+        POST: {
+            auth: {
+                type: "api-key"
+            },
+            body: EngineExecutionReportPayloadSchema,
+            parameters: {
+                sourceID: ULIDSchema
+            },
+            response: {
+                schema: object$5({
+                    acknowledged: literal(true)
+                }),
+                type: "rest"
+            }
+        }
+    },
     "/conduit/api/v1/source/:sourceID/requests/:requestID/chunk": {
         POST: {
             auth: {
@@ -113767,6 +113810,19 @@ function createAPIClient({ apiKey, apiURL, inferenceSourceID, logger }) {
                 route: "/conduit/api/v1/source/:sourceID/state"
             });
         },
+        reportEngineExecution: async (payload) => {
+            await fetchByReference({
+                baseURL: apiURL,
+                body: payload,
+                fetch: fetchWithAPIKey,
+                method: "POST",
+                parameters: {
+                    sourceID: inferenceSourceID
+                },
+                reference: API_SERVICE_CONDUIT_API_REFERENCE,
+                route: "/conduit/api/v1/source/:sourceID/engine/execution"
+            });
+        },
         reportPromptMetrics: async (payload) => {
             await fetchByReference({
                 baseURL: apiURL,
@@ -125627,6 +125683,9 @@ class ModelManager extends EventEmitter {
     lifecycleState = "stopped";
     downloadLockHandle = null;
     stopRequested = false;
+    lastEngineExitCode = null;
+    lastEngineExitSignal = null;
+    reachedRunningState = false;
     modelsDirectory;
     constructor({ contextLength, engineConfig, enginePort, engineType, logger, model, root }) {
         super();
@@ -125749,6 +125808,9 @@ class ModelManager extends EventEmitter {
         this.lifecycleState = "starting";
         this.lastEngineError = null;
         this.stopRequested = false;
+        this.lastEngineExitCode = null;
+        this.lastEngineExitSignal = null;
+        this.reachedRunningState = false;
         this.logger.info("Starting LLM engine", {
             agentEngineType: this.engine
         });
@@ -125779,6 +125841,7 @@ class ModelManager extends EventEmitter {
             throw err;
         }
         this.lifecycleState = "running";
+        this.reachedRunningState = true;
         this.emit("engineReady");
     }
     async stop() {
@@ -125803,6 +125866,7 @@ class ModelManager extends EventEmitter {
             this.lifecycleState = "stopped";
             return;
         }
+        this.reachedRunningState = false;
         this.lifecycleState = "stopping";
         this.stopRequested = true;
         await processManager.stop();
@@ -125815,9 +125879,18 @@ class ModelManager extends EventEmitter {
             this.lifecycleState === "starting" ||
             this.lifecycleState === "errored");
     }
+    get lastExitCode() {
+        return this.lastEngineExitCode;
+    }
+    get lastExitSignal() {
+        return this.lastEngineExitSignal;
+    }
     get state() {
         return this.lifecycleState;
     }
+    get wasRunning() {
+        return this.reachedRunningState;
+    }
     async checkEngineReadiness() {
         switch (this.engine) {
             case "llama.cpp": {
@@ -125931,6 +126004,7 @@ class ModelManager extends EventEmitter {
                 if (readiness === "ready") {
                     this.clearHealthPoll();
                     this.lifecycleState = "running";
+                    this.reachedRunningState = true;
                     this.emit("engineReady");
                 }
             })
@@ -125993,6 +126067,8 @@ class ModelManager extends EventEmitter {
             }));
         });
         processManager.on("stopped", (code, signal) => {
+            this.lastEngineExitCode = code;
+            this.lastEngineExitSignal = signal;
             if (hasTerminated) {
                 return;
             }
@@ -126060,6 +126136,69 @@ class ModelManager extends EventEmitter {
     }
 }
 
+// Ordered most-specific first: a message mentioning a prompt-size failure should classify as
+// "prompt" even if it also touches memory text; memory outranks config because OOM kills frequently
+// emit sparse stderr. These are heuristics, not exhaustively enumerated engine vocabularies.
+const OOM_PATTERNS = [
+    /CUDA out of memory/i,
+    /No available memory for the cache blocks/i,
+    /\bOOMKilled\b/,
+    /out of memory/i,
+    /MemoryError/i,
+    /Cannot allocate memory/i
+];
+const PROMPT_PATTERNS = [
+    /prompt is too long/i,
+    /maximum context length exceeded/i,
+    /too many tokens/i
+];
+const CONFIG_PATTERNS = [
+    /unrecognized argument/i,
+    /invalid argument/i,
+    /error while loading state_dict/i,
+    /Architecture not understood/i,
+    /No such file or directory/i
+];
+/**
+ * Coarse engine-failure classification used only to fill `engine_execution.error_type`. Exit code and
+ * terminating signal are consulted FIRST (SIGKILL/SIGSEGV are reliable OOM signals regardless of
+ * how much stderr the engine produced); the message text then refines the category. Anything
+ * unrecognized is "unknown".
+ */
+function classifyEngineFailure({ error, exitCode, signal }) {
+    // 137 = SIGKILL (kernel OOM-killer), 139 = SIGSEGV (illegal memory access). Treat both as OOM
+    // so that memory-pressure deaths do not degrade to "unknown" when stderr is truncated.
+    if (exitCode === 137 || exitCode === 139) {
+        return "oom";
+    }
+    // Direct OS kills (SIGKILL/SIGSEGV) leave no meaningful exit code; honor them like 137/139.
+    if (signal === "SIGKILL" || signal === "SIGSEGV") {
+        return "oom";
+    }
+    const text = `${error.message}`.slice(0, 4000);
+    for (const pattern of PROMPT_PATTERNS) {
+        if (pattern.test(text)) {
+            return "prompt";
+        }
+    }
+    for (const pattern of OOM_PATTERNS) {
+        if (pattern.test(text)) {
+            return "oom";
+        }
+    }
+    for (const pattern of CONFIG_PATTERNS) {
+        if (pattern.test(text)) {
+            return "config";
+        }
+    }
+    // A process that exited abnormally with no recognizable diagnostic: treat as a crash rather
+    // than "unknown" so the two buckets distinguish "we saw nothing" from "it died badly".
+    if (exitCode !== null && exitCode !== 0) {
+        return "crash";
+    }
+    return "unknown";
+}
+
 const EXCEPTION_LINE_PATTERN = /([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception)):\s*(.+)/;
 const FALLBACK_DETAIL_MAX_LENGTH = 300;
 const FALLBACK_RAW_MAX_LENGTH = 500;
@@ -138371,6 +138510,119 @@ async function detectDockerVersion() {
     }
 }
 
+/**
+ * Flattens flat CLI extra-arg tokens into [arg, value] pairs, sorted by ARG NAME (ascending, ties by
+ * value). `--flag=value` pairs split on the first `=`; a bare `--flag` consumes the following token
+ * as its value when that token does not start with "-" (classic CLI convention); anything else
+ * (flags, non-strings) is dropped.
+ */
+function pairExtraArgs(tokens) {
+    if (!Array.isArray(tokens)) {
+        return [];
+    }
+    const list = tokens;
+    const pairs = [];
+    let index = 0;
+    while (index < list.length) {
+        const token = list[index];
+        if (typeof token !== "string" || token.length === 0 || !token.startsWith("-")) {
+            index++;
+            continue;
+        }
+        const separator = token.indexOf("=");
+        if (separator > -1) {
+            const arg = token.slice(0, separator);
+            if (arg.length > 0) {
+                pairs.push([arg, token.slice(separator + 1)]);
+            }
+            index++;
+            continue;
+        }
+        const next = list[index + 1];
+        const consumesNext = typeof next === "string" && next.length > 0 && !next.startsWith("-");
+        if (consumesNext) {
+            pairs.push([token, next]);
+            index += 2;
+        }
+        else {
+            pairs.push([token, ""]);
+            index++;
+        }
+    }
+    return pairs.sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0);
+}
+/**
+ * Files AT MOST ONE engine_execution report per engine startup. `beginStartup(runAt)` re-arms the
+ * latch on every fresh model start (initial boot or cycle) and stamps the startup epoch used as the
+ * row's `run_at`. Whichever of the three report* paths fires first wins: startup failure, in-flight
+ * crash, or first successful full prompt completion.
+ */
+class EngineExecutionReporter {
+    options;
+    currentStartupAt = null;
+    reportedForCurrentStartup = false;
+    constructor(options) {
+        this.options = options;
+    }
+    /** Re-arms the latch; the stamp becomes the row's `run_at` (moment startup began). */
+    beginStartup(runAt) {
+        this.currentStartupAt = runAt;
+        this.reportedForCurrentStartup = false;
+    }
+    /** Reports a startup failure (rejected prepare/start, readiness timeout, pre-ready death). */
+    async reportStartupFailure(report) {
+        await this.file(report, false);
+    }
+    /** Reports a spontaneous crash of an engine that had reached the running state. */
+    async reportRuntimeCrash(report) {
+        await this.file(report, false);
+    }
+    /** Reports the first fully-responded, token-bearing prompt completion since startup. */
+    async reportSuccess(report) {
+        await this.file(report, true);
+    }
+    async file(report, success) {
+        if (this.reportedForCurrentStartup || this.currentStartupAt === null) {
+            return;
+        }
+        // Latch BEFORE the network call: a thrown POST cannot double-file for this startup.
+        this.reportedForCurrentStartup = true;
+        const context = this.options.buildContext();
+        const payload = {
+            avgTps: report.throughput.avgTps,
+            completionTokens: report.usage.completionTokens,
+            durationMs: report.durationMs,
+            engineType: context.engineType,
+            engineVersion: context.engineVersion,
+            errorDetail: success ? null : report.errorDetail,
+            errorType: success ? null : report.errorType,
+            extraArgs: context.extraArgsPairs,
+            finishedAtISO: new Date().toISOString(),
+            peakTps: report.throughput.peakTps,
+            promptTokens: report.usage.promptTokens,
+            runAtISO: this.currentStartupAt.toISOString(),
+            success,
+            ttftMs: report.ttftMs,
+            totalTokens: report.usage.totalTokens
+        };
+        try {
+            await this.options.report(payload);
+            this.options.logger.info("Engine execution outcome reported", {
+                inferenceSourceID: this.options.sourceLabel,
+                success
+            });
+        }
+        catch (error) {
+            // Losing one report is preferable to filing two; the latch stays latched.
+            this.options.logger.warn("Failed to report engine execution outcome", {
+                error: asError(error),
+                inferenceSourceID: this.options.sourceLabel,
+                success
+            });
+        }
+    }
+}
+
 async function createApplication({ abortController, apiClient, configuration, logger }) {
     ensureDockerValidEnv();
     logger.info("Fetching conduit configuration");
@@ -138402,6 +138654,87 @@ async function createApplication({ abortController, apiClient, configuration, lo
             error: asError(error)
         });
     }
+    const reporter = new EngineExecutionReporter({
+        buildContext: () => {
+            const engineType = (conduitConfiguration.engineConfig?.type ??
+                "llama.cpp");
+            const versions = {
+                exllamav3: machine?.exllamav3Version ?? null,
+                "llama.cpp": machine?.llamaCppVersion ?? null,
+                "mlx-lm": machine?.mlxlmVersion ?? null,
+                sglang: machine?.sglangVersion ?? null,
+                "tensorrt-llm": machine?.tensorrtLlmVersion ?? null,
+                vllm: machine?.vllmVersion ?? null
+            };
+            return {
+                engineType,
+                engineVersion: versions[engineType] ?? null,
+                extraArgsPairs: pairExtraArgs(conduitConfiguration.engineConfig?.extraArgs)
+            };
+        },
+        logger,
+        report: payload => apiClient.reportEngineExecution(payload),
+        sourceLabel: configuration.inferenceSourceID
+    });
+    // Intercept the prompt-metrics chokepoint so the first fully-responded, token-bearing prompt of
+    // each fresh startup files the one-shot engine_execution success report. Handlers close over the
+    // SAME `apiClient` object and read `reportPromptMetrics` at request-dispatch time (which always
+    // follows this point), so the wrapped method is what they invoke.
+    const rawReportPromptMetrics = apiClient.reportPromptMetrics;
+    apiClient.reportPromptMetrics = async (payload) => {
+        if (payload.successful && payload.completionTokens > 0 && payload.latencyMs > 0) {
+            // The one-shot report is kicked off and its settlement attached HERE (before any await): if the
+            // metrics path throws below, the report promise must still be able to log its own rejection.
+            const successReport = reporter
+                .reportSuccess({
+                durationMs: payload.latencyMs,
+                errorDetail: null,
+                errorType: null,
+                throughput: {
+                    avgTps: payload.tokensPerSecond,
+                    peakTps: null
+                },
+                ttftMs: payload.timeToFirstTokenMs ?? 0,
+                usage: {
+                    completionTokens: payload.completionTokens,
+                    promptTokens: payload.promptTokens,
+                    totalTokens: payload.totalTokens
+                }
+            })
+                .catch(error => {
+                logger.warn("Engine execution success report failed", {
+                    error: asError(error)
+                });
+            });
+            await rawReportPromptMetrics(payload);
+            await successReport;
+            return;
+        }
+        await rawReportPromptMetrics(payload);
+    };
+    const SECRET_ARG_MASK_PATTERN = /(-{1,2}[A-Za-z0-9_.]*(?:api[-_]?key|hf[-_]?token|token)(?:\s+|[=:]))\S+/gi;
+    // Assembles the payload shared by the startup-failure and runtime-crash report paths. Stderr
+    // may echo secrets, so mask `--api-key`/token-looking args before they reach the DB.
+    function buildCrashReport(error, exitCode, signal) {
+        const classification = classifyEngineFailure({ error, exitCode, signal });
+        const raw = normalizeEngineError(error.message);
+        const masked = raw.replace(SECRET_ARG_MASK_PATTERN, "$1***");
+        return {
+            durationMs: 0,
+            errorDetail: masked.slice(0, 2048),
+            errorType: classification,
+            ttftMs: 0,
+            throughput: {
+                avgTps: 0,
+                peakTps: null
+            },
+            usage: {
+                completionTokens: 0,
+                promptTokens: 0,
+                totalTokens: 0
+            }
+        };
+    }
     const conduitStateManager = new ConduitStateManager({
         initialState: {
             state: "initialising"
@@ -138453,6 +138786,17 @@ async function createApplication({ abortController, apiClient, configuration, lo
             });
             stopRequestedByControl = false;
             setErrorState({ error: normalizeEngineError(err.message) });
+            // Spontaneous death of a SERVING engine → crash report, suppressed by the latch if the
+            // startup's one-shot outcome was already filed. Startup-path failures report from
+            // `startEngine`'s catch; this listener is the RUNTIME-crash path only.
+            if (modelManager.wasRunning && !err.message.includes("interrupted by stop request")) {
+                const crashReport = buildCrashReport(err, modelManager.lastExitCode, modelManager.lastExitSignal);
+                reporter.reportRuntimeCrash(crashReport).catch(crashReportError => {
+                    logger.warn("Engine execution crash report failed", {
+                        error: asError(crashReportError)
+                    });
+                });
+            }
         });
         modelManager.on("engineReady", () => {
             setOnlineState();
@@ -138512,24 +138856,40 @@ async function createApplication({ abortController, apiClient, configuration, lo
     };
     async function startEngine() {
         logger.info("Engine start requested");
-        conduitStateManager.setState({
-            modelFileName,
-            modelName,
-            state: "downloadingModelFiles",
-            totalProgress: {
-                file: 0,
-                total: 0
+        reporter.beginStartup(new Date());
+        try {
+            conduitStateManager.setState({
+                modelFileName,
+                modelName,
+                state: "downloadingModelFiles",
+                totalProgress: {
+                    file: 0,
+                    total: 0
+                }
+            });
+            await conduitStateReportManager.reportNow();
+            await modelManager.prepare({
+                onDownloadProgress: reportDownloadProgress
+            });
+            conduitStateManager.setState({
+                state: "bootingEngine"
+            });
+            await conduitStateReportManager.reportNow();
+            await modelManager.start();
+        }
+        catch (error) {
+            const parsedError = asError(error);
+            // Operator-initiated aborts are not startup failures worth reporting.
+            if (!parsedError.message.includes("interrupted by stop request")) {
+                const startupReport = buildCrashReport(parsedError, modelManager.lastExitCode, modelManager.lastExitSignal);
+                reporter.reportStartupFailure(startupReport).catch(startupReportError => {
+                    logger.warn("Engine execution startup report failed", {
+                        error: asError(startupReportError)
+                    });
+                });
             }
-        });
-        await conduitStateReportManager.reportNow();
-        await modelManager.prepare({
-            onDownloadProgress: reportDownloadProgress
-        });
-        conduitStateManager.setState({
-            state: "bootingEngine"
-        });
-        await conduitStateReportManager.reportNow();
-        await modelManager.start();
+            throw error;
+        }
     }
     async function stopEngine({ reason }) {
         if (!modelManager.canStop) {

package/dist/cli.sea.cjs

@@ -19986,6 +19986,32 @@ object$5({
     sizeBytes: number$1().int().nonnegative().nullable()
 });
 
+const EngineExecutionErrorTypeSchema = _enum$1([
+    "config",
+    "crash",
+    "oom",
+    "prompt",
+    "unknown"
+]);
+const EngineExecutionReportPayloadSchema = object$5({
+    avgTps: number$1().nonnegative().finite().default(0),
+    completionTokens: number$1().int().nonnegative().default(0),
+    durationMs: number$1().int().nonnegative().default(0),
+    engineType: LLMEngineSchema.nullable(),
+    engineVersion: string$2().max(64).nullable().default(null),
+    errorDetail: string$2().max(2048).nullable().default(null),
+    errorType: EngineExecutionErrorTypeSchema.nullable(),
+    extraArgs: array$1(tuple([string$2().min(1).max(128), string$2().max(512)]))
+        .max(256)
+        .default([]),
+    finishedAtISO: string$2().datetime({ offset: true }),
+    peakTps: number$1().nonnegative().finite().nullable().default(null),
+    promptTokens: number$1().int().nonnegative().default(0),
+    runAtISO: string$2().datetime({ offset: true }),
+    success: boolean$1(),
+    ttftMs: number$1().int().nonnegative().default(0),
+    totalTokens: number$1().int().nonnegative().default(0)
+});
 const InferenceAgentLLMMetricsPayloadSchema = object$5({
     bytes: number$1().int().nonnegative(),
     completionTokens: number$1().int().nonnegative(),
@@ -20324,6 +20350,23 @@ const API_SERVICE_CONDUIT_API_REFERENCE = {
             }
         }
     },
+    "/conduit/api/v1/source/:sourceID/engine/execution": {
+        POST: {
+            auth: {
+                type: "api-key"
+            },
+            body: EngineExecutionReportPayloadSchema,
+            parameters: {
+                sourceID: ULIDSchema
+            },
+            response: {
+                schema: object$5({
+                    acknowledged: literal(true)
+                }),
+                type: "rest"
+            }
+        }
+    },
     "/conduit/api/v1/source/:sourceID/requests/:requestID/chunk": {
         POST: {
             auth: {
@@ -113781,6 +113824,19 @@ function createAPIClient({ apiKey, apiURL, inferenceSourceID, logger }) {
                 route: "/conduit/api/v1/source/:sourceID/state"
             });
         },
+        reportEngineExecution: async (payload) => {
+            await fetchByReference({
+                baseURL: apiURL,
+                body: payload,
+                fetch: fetchWithAPIKey,
+                method: "POST",
+                parameters: {
+                    sourceID: inferenceSourceID
+                },
+                reference: API_SERVICE_CONDUIT_API_REFERENCE,
+                route: "/conduit/api/v1/source/:sourceID/engine/execution"
+            });
+        },
         reportPromptMetrics: async (payload) => {
             await fetchByReference({
                 baseURL: apiURL,
@@ -125641,6 +125697,9 @@ class ModelManager extends EventEmitter {
     lifecycleState = "stopped";
     downloadLockHandle = null;
     stopRequested = false;
+    lastEngineExitCode = null;
+    lastEngineExitSignal = null;
+    reachedRunningState = false;
     modelsDirectory;
     constructor({ contextLength, engineConfig, enginePort, engineType, logger, model, root }) {
         super();
@@ -125763,6 +125822,9 @@ class ModelManager extends EventEmitter {
         this.lifecycleState = "starting";
         this.lastEngineError = null;
         this.stopRequested = false;
+        this.lastEngineExitCode = null;
+        this.lastEngineExitSignal = null;
+        this.reachedRunningState = false;
         this.logger.info("Starting LLM engine", {
             agentEngineType: this.engine
         });
@@ -125793,6 +125855,7 @@ class ModelManager extends EventEmitter {
             throw err;
         }
         this.lifecycleState = "running";
+        this.reachedRunningState = true;
         this.emit("engineReady");
     }
     async stop() {
@@ -125817,6 +125880,7 @@ class ModelManager extends EventEmitter {
             this.lifecycleState = "stopped";
             return;
         }
+        this.reachedRunningState = false;
         this.lifecycleState = "stopping";
         this.stopRequested = true;
         await processManager.stop();
@@ -125829,9 +125893,18 @@ class ModelManager extends EventEmitter {
             this.lifecycleState === "starting" ||
             this.lifecycleState === "errored");
     }
+    get lastExitCode() {
+        return this.lastEngineExitCode;
+    }
+    get lastExitSignal() {
+        return this.lastEngineExitSignal;
+    }
     get state() {
         return this.lifecycleState;
     }
+    get wasRunning() {
+        return this.reachedRunningState;
+    }
     async checkEngineReadiness() {
         switch (this.engine) {
             case "llama.cpp": {
@@ -125945,6 +126018,7 @@ class ModelManager extends EventEmitter {
                 if (readiness === "ready") {
                     this.clearHealthPoll();
                     this.lifecycleState = "running";
+                    this.reachedRunningState = true;
                     this.emit("engineReady");
                 }
             })
@@ -126007,6 +126081,8 @@ class ModelManager extends EventEmitter {
             }));
         });
         processManager.on("stopped", (code, signal) => {
+            this.lastEngineExitCode = code;
+            this.lastEngineExitSignal = signal;
             if (hasTerminated) {
                 return;
             }
@@ -126074,6 +126150,69 @@ class ModelManager extends EventEmitter {
     }
 }
 
+// Ordered most-specific first: a message mentioning a prompt-size failure should classify as
+// "prompt" even if it also touches memory text; memory outranks config because OOM kills frequently
+// emit sparse stderr. These are heuristics, not exhaustively enumerated engine vocabularies.
+const OOM_PATTERNS = [
+    /CUDA out of memory/i,
+    /No available memory for the cache blocks/i,
+    /\bOOMKilled\b/,
+    /out of memory/i,
+    /MemoryError/i,
+    /Cannot allocate memory/i
+];
+const PROMPT_PATTERNS = [
+    /prompt is too long/i,
+    /maximum context length exceeded/i,
+    /too many tokens/i
+];
+const CONFIG_PATTERNS = [
+    /unrecognized argument/i,
+    /invalid argument/i,
+    /error while loading state_dict/i,
+    /Architecture not understood/i,
+    /No such file or directory/i
+];
+/**
+ * Coarse engine-failure classification used only to fill `engine_execution.error_type`. Exit code and
+ * terminating signal are consulted FIRST (SIGKILL/SIGSEGV are reliable OOM signals regardless of
+ * how much stderr the engine produced); the message text then refines the category. Anything
+ * unrecognized is "unknown".
+ */
+function classifyEngineFailure({ error, exitCode, signal }) {
+    // 137 = SIGKILL (kernel OOM-killer), 139 = SIGSEGV (illegal memory access). Treat both as OOM
+    // so that memory-pressure deaths do not degrade to "unknown" when stderr is truncated.
+    if (exitCode === 137 || exitCode === 139) {
+        return "oom";
+    }
+    // Direct OS kills (SIGKILL/SIGSEGV) leave no meaningful exit code; honor them like 137/139.
+    if (signal === "SIGKILL" || signal === "SIGSEGV") {
+        return "oom";
+    }
+    const text = `${error.message}`.slice(0, 4000);
+    for (const pattern of PROMPT_PATTERNS) {
+        if (pattern.test(text)) {
+            return "prompt";
+        }
+    }
+    for (const pattern of OOM_PATTERNS) {
+        if (pattern.test(text)) {
+            return "oom";
+        }
+    }
+    for (const pattern of CONFIG_PATTERNS) {
+        if (pattern.test(text)) {
+            return "config";
+        }
+    }
+    // A process that exited abnormally with no recognizable diagnostic: treat as a crash rather
+    // than "unknown" so the two buckets distinguish "we saw nothing" from "it died badly".
+    if (exitCode !== null && exitCode !== 0) {
+        return "crash";
+    }
+    return "unknown";
+}
+
 const EXCEPTION_LINE_PATTERN = /([A-Za-z_][A-Za-z0-9_]*(?:Error|Exception)):\s*(.+)/;
 const FALLBACK_DETAIL_MAX_LENGTH = 300;
 const FALLBACK_RAW_MAX_LENGTH = 500;
@@ -158600,6 +158739,119 @@ async function detectDockerVersion() {
     }
 }
 
+/**
+ * Flattens flat CLI extra-arg tokens into [arg, value] pairs, sorted by ARG NAME (ascending, ties by
+ * value). `--flag=value` pairs split on the first `=`; a bare `--flag` consumes the following token
+ * as its value when that token does not start with "-" (classic CLI convention); anything else
+ * (flags, non-strings) is dropped.
+ */
+function pairExtraArgs(tokens) {
+    if (!Array.isArray(tokens)) {
+        return [];
+    }
+    const list = tokens;
+    const pairs = [];
+    let index = 0;
+    while (index < list.length) {
+        const token = list[index];
+        if (typeof token !== "string" || token.length === 0 || !token.startsWith("-")) {
+            index++;
+            continue;
+        }
+        const separator = token.indexOf("=");
+        if (separator > -1) {
+            const arg = token.slice(0, separator);
+            if (arg.length > 0) {
+                pairs.push([arg, token.slice(separator + 1)]);
+            }
+            index++;
+            continue;
+        }
+        const next = list[index + 1];
+        const consumesNext = typeof next === "string" && next.length > 0 && !next.startsWith("-");
+        if (consumesNext) {
+            pairs.push([token, next]);
+            index += 2;
+        }
+        else {
+            pairs.push([token, ""]);
+            index++;
+        }
+    }
+    return pairs.sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0);
+}
+/**
+ * Files AT MOST ONE engine_execution report per engine startup. `beginStartup(runAt)` re-arms the
+ * latch on every fresh model start (initial boot or cycle) and stamps the startup epoch used as the
+ * row's `run_at`. Whichever of the three report* paths fires first wins: startup failure, in-flight
+ * crash, or first successful full prompt completion.
+ */
+class EngineExecutionReporter {
+    options;
+    currentStartupAt = null;
+    reportedForCurrentStartup = false;
+    constructor(options) {
+        this.options = options;
+    }
+    /** Re-arms the latch; the stamp becomes the row's `run_at` (moment startup began). */
+    beginStartup(runAt) {
+        this.currentStartupAt = runAt;
+        this.reportedForCurrentStartup = false;
+    }
+    /** Reports a startup failure (rejected prepare/start, readiness timeout, pre-ready death). */
+    async reportStartupFailure(report) {
+        await this.file(report, false);
+    }
+    /** Reports a spontaneous crash of an engine that had reached the running state. */
+    async reportRuntimeCrash(report) {
+        await this.file(report, false);
+    }
+    /** Reports the first fully-responded, token-bearing prompt completion since startup. */
+    async reportSuccess(report) {
+        await this.file(report, true);
+    }
+    async file(report, success) {
+        if (this.reportedForCurrentStartup || this.currentStartupAt === null) {
+            return;
+        }
+        // Latch BEFORE the network call: a thrown POST cannot double-file for this startup.
+        this.reportedForCurrentStartup = true;
+        const context = this.options.buildContext();
+        const payload = {
+            avgTps: report.throughput.avgTps,
+            completionTokens: report.usage.completionTokens,
+            durationMs: report.durationMs,
+            engineType: context.engineType,
+            engineVersion: context.engineVersion,
+            errorDetail: success ? null : report.errorDetail,
+            errorType: success ? null : report.errorType,
+            extraArgs: context.extraArgsPairs,
+            finishedAtISO: new Date().toISOString(),
+            peakTps: report.throughput.peakTps,
+            promptTokens: report.usage.promptTokens,
+            runAtISO: this.currentStartupAt.toISOString(),
+            success,
+            ttftMs: report.ttftMs,
+            totalTokens: report.usage.totalTokens
+        };
+        try {
+            await this.options.report(payload);
+            this.options.logger.info("Engine execution outcome reported", {
+                inferenceSourceID: this.options.sourceLabel,
+                success
+            });
+        }
+        catch (error) {
+            // Losing one report is preferable to filing two; the latch stays latched.
+            this.options.logger.warn("Failed to report engine execution outcome", {
+                error: asError(error),
+                inferenceSourceID: this.options.sourceLabel,
+                success
+            });
+        }
+    }
+}
+
 async function createApplication({ abortController, apiClient, configuration, logger }) {
     ensureDockerValidEnv();
     logger.info("Fetching conduit configuration");
@@ -158631,6 +158883,87 @@ async function createApplication({ abortController, apiClient, configuration, lo
             error: asError(error)
         });
     }
+    const reporter = new EngineExecutionReporter({
+        buildContext: () => {
+            const engineType = (conduitConfiguration.engineConfig?.type ??
+                "llama.cpp");
+            const versions = {
+                exllamav3: machine?.exllamav3Version ?? null,
+                "llama.cpp": machine?.llamaCppVersion ?? null,
+                "mlx-lm": machine?.mlxlmVersion ?? null,
+                sglang: machine?.sglangVersion ?? null,
+                "tensorrt-llm": machine?.tensorrtLlmVersion ?? null,
+                vllm: machine?.vllmVersion ?? null
+            };
+            return {
+                engineType,
+                engineVersion: versions[engineType] ?? null,
+                extraArgsPairs: pairExtraArgs(conduitConfiguration.engineConfig?.extraArgs)
+            };
+        },
+        logger,
+        report: payload => apiClient.reportEngineExecution(payload),
+        sourceLabel: configuration.inferenceSourceID
+    });
+    // Intercept the prompt-metrics chokepoint so the first fully-responded, token-bearing prompt of
+    // each fresh startup files the one-shot engine_execution success report. Handlers close over the
+    // SAME `apiClient` object and read `reportPromptMetrics` at request-dispatch time (which always
+    // follows this point), so the wrapped method is what they invoke.
+    const rawReportPromptMetrics = apiClient.reportPromptMetrics;
+    apiClient.reportPromptMetrics = async (payload) => {
+        if (payload.successful && payload.completionTokens > 0 && payload.latencyMs > 0) {
+            // The one-shot report is kicked off and its settlement attached HERE (before any await): if the
+            // metrics path throws below, the report promise must still be able to log its own rejection.
+            const successReport = reporter
+                .reportSuccess({
+                durationMs: payload.latencyMs,
+                errorDetail: null,
+                errorType: null,
+                throughput: {
+                    avgTps: payload.tokensPerSecond,
+                    peakTps: null
+                },
+                ttftMs: payload.timeToFirstTokenMs ?? 0,
+                usage: {
+                    completionTokens: payload.completionTokens,
+                    promptTokens: payload.promptTokens,
+                    totalTokens: payload.totalTokens
+                }
+            })
+                .catch(error => {
+                logger.warn("Engine execution success report failed", {
+                    error: asError(error)
+                });
+            });
+            await rawReportPromptMetrics(payload);
+            await successReport;
+            return;
+        }
+        await rawReportPromptMetrics(payload);
+    };
+    const SECRET_ARG_MASK_PATTERN = /(-{1,2}[A-Za-z0-9_.]*(?:api[-_]?key|hf[-_]?token|token)(?:\s+|[=:]))\S+/gi;
+    // Assembles the payload shared by the startup-failure and runtime-crash report paths. Stderr
+    // may echo secrets, so mask `--api-key`/token-looking args before they reach the DB.
+    function buildCrashReport(error, exitCode, signal) {
+        const classification = classifyEngineFailure({ error, exitCode, signal });
+        const raw = normalizeEngineError(error.message);
+        const masked = raw.replace(SECRET_ARG_MASK_PATTERN, "$1***");
+        return {
+            durationMs: 0,
+            errorDetail: masked.slice(0, 2048),
+            errorType: classification,
+            ttftMs: 0,
+            throughput: {
+                avgTps: 0,
+                peakTps: null
+            },
+            usage: {
+                completionTokens: 0,
+                promptTokens: 0,
+                totalTokens: 0
+            }
+        };
+    }
     const conduitStateManager = new ConduitStateManager({
         initialState: {
             state: "initialising"
@@ -158682,6 +159015,17 @@ async function createApplication({ abortController, apiClient, configuration, lo
             });
             stopRequestedByControl = false;
             setErrorState({ error: normalizeEngineError(err.message) });
+            // Spontaneous death of a SERVING engine → crash report, suppressed by the latch if the
+            // startup's one-shot outcome was already filed. Startup-path failures report from
+            // `startEngine`'s catch; this listener is the RUNTIME-crash path only.
+            if (modelManager.wasRunning && !err.message.includes("interrupted by stop request")) {
+                const crashReport = buildCrashReport(err, modelManager.lastExitCode, modelManager.lastExitSignal);
+                reporter.reportRuntimeCrash(crashReport).catch(crashReportError => {
+                    logger.warn("Engine execution crash report failed", {
+                        error: asError(crashReportError)
+                    });
+                });
+            }
         });
         modelManager.on("engineReady", () => {
             setOnlineState();
@@ -158741,24 +159085,40 @@ async function createApplication({ abortController, apiClient, configuration, lo
     };
     async function startEngine() {
         logger.info("Engine start requested");
-        conduitStateManager.setState({
-            modelFileName,
-            modelName,
-            state: "downloadingModelFiles",
-            totalProgress: {
-                file: 0,
-                total: 0
+        reporter.beginStartup(new Date());
+        try {
+            conduitStateManager.setState({
+                modelFileName,
+                modelName,
+                state: "downloadingModelFiles",
+                totalProgress: {
+                    file: 0,
+                    total: 0
+                }
+            });
+            await conduitStateReportManager.reportNow();
+            await modelManager.prepare({
+                onDownloadProgress: reportDownloadProgress
+            });
+            conduitStateManager.setState({
+                state: "bootingEngine"
+            });
+            await conduitStateReportManager.reportNow();
+            await modelManager.start();
+        }
+        catch (error) {
+            const parsedError = asError(error);
+            // Operator-initiated aborts are not startup failures worth reporting.
+            if (!parsedError.message.includes("interrupted by stop request")) {
+                const startupReport = buildCrashReport(parsedError, modelManager.lastExitCode, modelManager.lastExitSignal);
+                reporter.reportStartupFailure(startupReport).catch(startupReportError => {
+                    logger.warn("Engine execution startup report failed", {
+                        error: asError(startupReportError)
+                    });
+                });
             }
-        });
-        await conduitStateReportManager.reportNow();
-        await modelManager.prepare({
-            onDownloadProgress: reportDownloadProgress
-        });
-        conduitStateManager.setState({
-            state: "bootingEngine"
-        });
-        await conduitStateReportManager.reportNow();
-        await modelManager.start();
+            throw error;
+        }
     }
     async function stopEngine({ reason }) {
         if (!modelManager.canStop) {

package/dist/modelManagement/ModelManager.d.ts

@@ -24,6 +24,9 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
     private lifecycleState;
     private downloadLockHandle;
     private stopRequested;
+    private lastEngineExitCode;
+    private lastEngineExitSignal;
+    private reachedRunningState;
     protected readonly modelsDirectory: string;
     constructor({ contextLength, engineConfig, enginePort, engineType, logger, model, root }: {
         contextLength?: number | null;
@@ -43,7 +46,10 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
     stop(): Promise<void>;
     get canStart(): boolean;
     get canStop(): boolean;
+    get lastExitCode(): number | null;
+    get lastExitSignal(): NodeJS.Signals | null;
     get state(): EngineLifecycleState;
+    get wasRunning(): boolean;
     private checkEngineReadiness;
     private checkGenericHealthReadiness;
     private checkLlamacppReadiness;

package/dist/modelManagement/classifyEngineError.d.ts

@@ -0,0 +1,12 @@
+import { EngineExecutionErrorType } from "@infersec/definitions";
+/**
+ * Coarse engine-failure classification used only to fill `engine_execution.error_type`. Exit code and
+ * terminating signal are consulted FIRST (SIGKILL/SIGSEGV are reliable OOM signals regardless of
+ * how much stderr the engine produced); the message text then refines the category. Anything
+ * unrecognized is "unknown".
+ */
+export declare function classifyEngineFailure({ error, exitCode, signal }: {
+    error: Error;
+    exitCode: number | null;
+    signal?: NodeJS.Signals | null;
+}): EngineExecutionErrorType;

package/dist/reporting/engineExecutionReporter.d.ts

@@ -0,0 +1,54 @@
+import { EngineExecutionErrorType, EngineExecutionReportPayload, LLMEngine } from "@infersec/definitions";
+import { Logger } from "@infersec/logger";
+export interface EngineExecutionReportSource {
+    durationMs: number;
+    errorDetail: string | null;
+    errorType: EngineExecutionErrorType | null;
+    ttftMs: number;
+    throughput: {
+        avgTps: number;
+        peakTps: number | null;
+    };
+    usage: {
+        completionTokens: number;
+        promptTokens: number;
+        totalTokens: number;
+    };
+}
+/**
+ * Flattens flat CLI extra-arg tokens into [arg, value] pairs, sorted by ARG NAME (ascending, ties by
+ * value). `--flag=value` pairs split on the first `=`; a bare `--flag` consumes the following token
+ * as its value when that token does not start with "-" (classic CLI convention); anything else
+ * (flags, non-strings) is dropped.
+ */
+export declare function pairExtraArgs(tokens: unknown): Array<[string, string]>;
+/**
+ * Files AT MOST ONE engine_execution report per engine startup. `beginStartup(runAt)` re-arms the
+ * latch on every fresh model start (initial boot or cycle) and stamps the startup epoch used as the
+ * row's `run_at`. Whichever of the three report* paths fires first wins: startup failure, in-flight
+ * crash, or first successful full prompt completion.
+ */
+export declare class EngineExecutionReporter {
+    private readonly options;
+    private currentStartupAt;
+    private reportedForCurrentStartup;
+    constructor(options: {
+        buildContext: () => {
+            engineType: LLMEngine;
+            engineVersion: string | null;
+            extraArgsPairs: Array<[string, string]>;
+        };
+        logger: Logger;
+        report: (payload: EngineExecutionReportPayload) => Promise<void>;
+        sourceLabel: string;
+    });
+    /** Re-arms the latch; the stamp becomes the row's `run_at` (moment startup began). */
+    beginStartup(runAt: Date): void;
+    /** Reports a startup failure (rejected prepare/start, readiness timeout, pre-ready death). */
+    reportStartupFailure(report: EngineExecutionReportSource): Promise<void>;
+    /** Reports a spontaneous crash of an engine that had reached the running state. */
+    reportRuntimeCrash(report: EngineExecutionReportSource): Promise<void>;
+    /** Reports the first fully-responded, token-bearing prompt completion since startup. */
+    reportSuccess(report: EngineExecutionReportSource): Promise<void>;
+    private file;
+}

package/dist/reporting/index.d.ts

@@ -0,0 +1 @@
+export { EngineExecutionReporter, type EngineExecutionReportSource, pairExtraArgs } from "./engineExecutionReporter.js";

package/dist/requestHandlers/createConduitAnthropicAPIReferenceHandlers.d.ts

@@ -86,7 +86,7 @@ export declare function createPostMessagesHandler(options: {
             budget_tokens: number;
             type: "enabled";
         } | undefined;
-        tool_choice?: "none" | "any" | "auto" | {
+        tool_choice?: "any" | "none" | "auto" | {
             type: "auto";
             disable_parallel_tool_use?: boolean | undefined;
         } | {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.112.1",
+  "version": "1.113.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },