@infersec/conduit 1.114.0 → 1.115.0

package/dist/cli.js

@@ -125777,8 +125777,10 @@ class ModelManager extends EventEmitter {
     contextLength;
     logger;
     discoveredModelNames = [];
+    customProbeCount = 0;
     engineProcess = null;
     healthPollInterval = null;
+    lastCustomReadinessReport = null;
     lastEngineError = null;
     lifecycleState = "stopped";
     downloadLockHandle = null;
@@ -126016,7 +126018,9 @@ class ModelManager extends EventEmitter {
         if (this.engine !== "custom")
             return null;
         const baseUrl = this.engineConfig?.baseUrl;
-        return typeof baseUrl === "string" && baseUrl.length > 0 ? baseUrl : null;
+        if (typeof baseUrl !== "string" || baseUrl.length === 0)
+            return null;
+        return baseUrl.replace(/\/+$/, "");
     }
     async checkEngineReadiness() {
         switch (this.engine) {
@@ -126042,17 +126046,32 @@ class ModelManager extends EventEmitter {
     async checkCustomReadiness() {
         const baseURL = this.customBaseURL;
         if (!baseURL) {
+            this.reportCustomReadinessChange("unreachable", "", "no base URL configured");
             return "unreachable";
         }
+        const probeURL = joinURL(baseURL, "/v1/models");
+        this.customProbeCount++;
         try {
-            const response = await undiciExports.fetch(joinURL(baseURL, "/v1/models"), {
+            const response = await undiciExports.fetch(probeURL, {
                 method: "GET",
                 signal: AbortSignal.timeout(5000)
             });
             if (response.status === 503) {
+                this.logger.debug("Custom engine probe: server loading", {
+                    attempt: this.customProbeCount,
+                    probeURL
+                });
+                this.reportCustomReadinessChange("loading", baseURL, "server loading (HTTP 503)");
                 return "loading";
             }
             if (!response.ok) {
+                const reason = `HTTP ${response.status} from /v1/models`;
+                this.logger.debug("Custom engine probe: not ready", {
+                    attempt: this.customProbeCount,
+                    probeURL,
+                    reason
+                });
+                this.reportCustomReadinessChange("unreachable", baseURL, reason);
                 return "unreachable";
             }
             const payload = (await response.json());
@@ -126066,24 +126085,56 @@ class ModelManager extends EventEmitter {
             })
                 .filter((id) => id !== null);
             if (modelIDs.length === 0) {
-                this.logger.warn("Custom engine endpoint exposed no models via /v1/models", {
-                    engineBaseURL: baseURL
+                this.logger.debug("Custom engine probe: no models exposed yet", {
+                    attempt: this.customProbeCount,
+                    probeURL
                 });
+                this.reportCustomReadinessChange("loading", baseURL, "/v1/models returned no models");
                 return "loading";
             }
             if (modelIDs.join("\n") !== this.discoveredModelNames.join("\n")) {
+                this.discoveredModelNames = modelIDs;
                 this.logger.info("Discovered models on custom engine endpoint", {
                     engineBaseURL: baseURL,
-                    models: modelIDs
+                    models: modelIDs,
+                    selectedModel: modelIDs[0]
                 });
-                this.discoveredModelNames = modelIDs;
             }
             return "ready";
         }
-        catch (_error) {
+        catch (error) {
+            const reason = asError(error).message;
+            this.logger.debug("Custom engine probe: request failed", {
+                attempt: this.customProbeCount,
+                probeURL,
+                reason
+            });
+            this.reportCustomReadinessChange("unreachable", baseURL, reason);
             return "unreachable";
         }
     }
+    reportCustomReadinessChange(readiness, baseURL, reason) {
+        const key = `${readiness}:${reason}`;
+        if (key === this.lastCustomReadinessReport) {
+            // Same outcome as the last report: surface a heartbeat every 15
+            // attempts (~30s while booting) so progress stays visible.
+            if (this.customProbeCount % 15 === 0) {
+                this.logger.info("Still waiting for custom engine endpoint", {
+                    attempt: this.customProbeCount,
+                    engineBaseURL: baseURL,
+                    reason
+                });
+            }
+            return;
+        }
+        this.lastCustomReadinessReport = key;
+        this.logger.warn(readiness === "loading"
+            ? "Custom engine endpoint is loading — waiting for /v1/models to expose a model"
+            : "Custom engine endpoint unreachable — verify the server is running and reachable from conduit", {
+            engineBaseURL: baseURL,
+            reason
+        });
+    }
     async checkGenericHealthReadiness() {
         try {
             const response = await undiciExports.fetch(joinURL(`http://localhost:${this.enginePort}`, "/health"), {
@@ -126142,6 +126193,11 @@ class ModelManager extends EventEmitter {
         const maxWaitMs = Number.parseInt(process.env.ENGINE_STARTUP_TIMEOUT_MS ?? "", 10) || 15 * 60 * 1000;
         const pollIntervalMs = 2000;
         const start = Date.now();
+        if (this.engine === "custom") {
+            this.logger.info(`Connecting to external OpenAI-compatible server at ${this.customBaseURL ?? "(no base URL configured)"} — serving begins once /v1/models exposes a model`, {
+                engineBaseURL: this.customBaseURL ?? ""
+            });
+        }
         while (Date.now() - start < maxWaitMs) {
             if (this.lifecycleState === "stopping" || this.stopRequested) {
                 throw new Error("LLM engine startup interrupted by stop request");
@@ -126162,6 +126218,13 @@ class ModelManager extends EventEmitter {
             await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
         }
         const stderrTail = this.engineProcess?.stderr?.slice(-1e3);
+        if (this.engine === "custom") {
+            const baseURL = this.customBaseURL ?? "(missing)";
+            const discovered = this.discoveredModelNames;
+            throw new Error(discovered.length > 0
+                ? `Custom engine endpoint at ${baseURL} exposed models (${discovered.join(", ")}) but never reported ready within ${Math.round(maxWaitMs / 1000)}s`
+                : `Custom engine endpoint at ${baseURL} did not expose a model via /v1/models within ${Math.round(maxWaitMs / 1000)}s. Verify the server is running and reachable from conduit (if conduit runs in a container, use the host address instead of localhost)`);
+        }
         throw new Error(stderrTail
             ? `LLM engine failed readiness checks within timeout. Last engine output:\n${stderrTail}`
             : "LLM engine failed readiness checks within timeout");
@@ -127299,10 +127362,22 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
     });
     return {
         body: monitoredResponse.stream,
-        headers: Object.fromEntries(response.headers.entries()),
+        headers: buildProxyResponseHeaders(response.headers, Boolean(servedModelName)),
         status: response.status
     };
 }
+/**
+ * Forwards upstream headers, dropping content-length when the response body
+ * is rewritten (model-name substitution changes its length; a stale
+ * content-length desyncs the stream and aborts downstream readers).
+ */
+function buildProxyResponseHeaders(headers, rewriteActive) {
+    const forwarded = Object.fromEntries(headers.entries());
+    if (rewriteActive) {
+        delete forwarded["content-length"];
+    }
+    return forwarded;
+}
 async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointId, logger, modelID, modelManager, path, reportMetrics, signal }) {
     function normalizeTokenCount(value) {
         if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
@@ -127481,7 +127556,7 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
         });
     return {
         body: monitoredResponse.stream,
-        headers: Object.fromEntries(response.headers.entries()),
+        headers: buildProxyResponseHeaders(response.headers, Boolean(servedModelName)),
         status: response.status
     };
 }
@@ -139118,23 +139193,33 @@ async function createApplication({ abortController, apiClient, configuration, lo
         logger.info("Engine start requested");
         reporter.beginStartup(new Date());
         try {
-            conduitStateManager.setState({
-                modelFileName,
-                modelName,
-                state: "downloadingModelFiles",
-                totalProgress: {
-                    file: 0,
-                    total: 0
-                }
-            });
-            await conduitStateReportManager.reportNow();
+            if (modelManager.engine === "custom") {
+                conduitStateManager.setState({
+                    state: "bootingEngine"
+                });
+                await conduitStateReportManager.reportNow();
+            }
+            else {
+                conduitStateManager.setState({
+                    modelFileName,
+                    modelName,
+                    state: "downloadingModelFiles",
+                    totalProgress: {
+                        file: 0,
+                        total: 0
+                    }
+                });
+                await conduitStateReportManager.reportNow();
+            }
             await modelManager.prepare({
                 onDownloadProgress: reportDownloadProgress
             });
-            conduitStateManager.setState({
-                state: "bootingEngine"
-            });
-            await conduitStateReportManager.reportNow();
+            if (modelManager.engine !== "custom") {
+                conduitStateManager.setState({
+                    state: "bootingEngine"
+                });
+                await conduitStateReportManager.reportNow();
+            }
             await modelManager.start();
         }
         catch (error) {
@@ -344092,16 +344177,15 @@ function buildSourceCreateBody(options) {
     if (!options.engine) {
         throw new Error("--engine is required (engine ID from `engine create`)");
     }
-    if (!options.model) {
-        throw new Error("--model is required (model ID from `models create`)");
-    }
     validateULID({ flagName: "--engine", value: options.engine });
-    validateULID({ flagName: "--model", value: options.model });
     const body = {
         engineId: options.engine,
-        modelID: options.model,
+        modelID: options.model ?? null,
         name: options.name
     };
+    if (options.model !== undefined) {
+        validateULID({ flagName: "--model", value: options.model });
+    }
     if (options.quant !== undefined) {
         validateQuant(options.quant);
         body.quantizationLabel = options.quant;
@@ -344143,7 +344227,7 @@ function registerSourceCommands({ program }) {
         .option("--engine <id>", "Engine ID to run this source with (required)")
         .option("--id <ulid>", "Target an existing source by ID (requires --update)")
         .option("--key <value>", "API key (required, no environment variable fallback)")
-        .option("--model <id>", "Model ID to serve (required)")
+        .option("--model <id>", "Model ID to serve (required unless the engine type is custom)")
         .option("--name <name>", "Source name (matched by --update)")
         .option("--quant <label>", "Quantization variant label (eg Q4_K_M)")
         .option("--update", "Update an existing source matched by name (or --id) instead of erroring")

package/dist/cli.sea.cjs

@@ -125791,8 +125791,10 @@ class ModelManager extends EventEmitter {
     contextLength;
     logger;
     discoveredModelNames = [];
+    customProbeCount = 0;
     engineProcess = null;
     healthPollInterval = null;
+    lastCustomReadinessReport = null;
     lastEngineError = null;
     lifecycleState = "stopped";
     downloadLockHandle = null;
@@ -126030,7 +126032,9 @@ class ModelManager extends EventEmitter {
         if (this.engine !== "custom")
             return null;
         const baseUrl = this.engineConfig?.baseUrl;
-        return typeof baseUrl === "string" && baseUrl.length > 0 ? baseUrl : null;
+        if (typeof baseUrl !== "string" || baseUrl.length === 0)
+            return null;
+        return baseUrl.replace(/\/+$/, "");
     }
     async checkEngineReadiness() {
         switch (this.engine) {
@@ -126056,17 +126060,32 @@ class ModelManager extends EventEmitter {
     async checkCustomReadiness() {
         const baseURL = this.customBaseURL;
         if (!baseURL) {
+            this.reportCustomReadinessChange("unreachable", "", "no base URL configured");
             return "unreachable";
         }
+        const probeURL = joinURL(baseURL, "/v1/models");
+        this.customProbeCount++;
         try {
-            const response = await undiciExports.fetch(joinURL(baseURL, "/v1/models"), {
+            const response = await undiciExports.fetch(probeURL, {
                 method: "GET",
                 signal: AbortSignal.timeout(5000)
             });
             if (response.status === 503) {
+                this.logger.debug("Custom engine probe: server loading", {
+                    attempt: this.customProbeCount,
+                    probeURL
+                });
+                this.reportCustomReadinessChange("loading", baseURL, "server loading (HTTP 503)");
                 return "loading";
             }
             if (!response.ok) {
+                const reason = `HTTP ${response.status} from /v1/models`;
+                this.logger.debug("Custom engine probe: not ready", {
+                    attempt: this.customProbeCount,
+                    probeURL,
+                    reason
+                });
+                this.reportCustomReadinessChange("unreachable", baseURL, reason);
                 return "unreachable";
             }
             const payload = (await response.json());
@@ -126080,24 +126099,56 @@ class ModelManager extends EventEmitter {
             })
                 .filter((id) => id !== null);
             if (modelIDs.length === 0) {
-                this.logger.warn("Custom engine endpoint exposed no models via /v1/models", {
-                    engineBaseURL: baseURL
+                this.logger.debug("Custom engine probe: no models exposed yet", {
+                    attempt: this.customProbeCount,
+                    probeURL
                 });
+                this.reportCustomReadinessChange("loading", baseURL, "/v1/models returned no models");
                 return "loading";
             }
             if (modelIDs.join("\n") !== this.discoveredModelNames.join("\n")) {
+                this.discoveredModelNames = modelIDs;
                 this.logger.info("Discovered models on custom engine endpoint", {
                     engineBaseURL: baseURL,
-                    models: modelIDs
+                    models: modelIDs,
+                    selectedModel: modelIDs[0]
                 });
-                this.discoveredModelNames = modelIDs;
             }
             return "ready";
         }
-        catch (_error) {
+        catch (error) {
+            const reason = asError(error).message;
+            this.logger.debug("Custom engine probe: request failed", {
+                attempt: this.customProbeCount,
+                probeURL,
+                reason
+            });
+            this.reportCustomReadinessChange("unreachable", baseURL, reason);
             return "unreachable";
         }
     }
+    reportCustomReadinessChange(readiness, baseURL, reason) {
+        const key = `${readiness}:${reason}`;
+        if (key === this.lastCustomReadinessReport) {
+            // Same outcome as the last report: surface a heartbeat every 15
+            // attempts (~30s while booting) so progress stays visible.
+            if (this.customProbeCount % 15 === 0) {
+                this.logger.info("Still waiting for custom engine endpoint", {
+                    attempt: this.customProbeCount,
+                    engineBaseURL: baseURL,
+                    reason
+                });
+            }
+            return;
+        }
+        this.lastCustomReadinessReport = key;
+        this.logger.warn(readiness === "loading"
+            ? "Custom engine endpoint is loading — waiting for /v1/models to expose a model"
+            : "Custom engine endpoint unreachable — verify the server is running and reachable from conduit", {
+            engineBaseURL: baseURL,
+            reason
+        });
+    }
     async checkGenericHealthReadiness() {
         try {
             const response = await undiciExports.fetch(joinURL(`http://localhost:${this.enginePort}`, "/health"), {
@@ -126156,6 +126207,11 @@ class ModelManager extends EventEmitter {
         const maxWaitMs = Number.parseInt(process.env.ENGINE_STARTUP_TIMEOUT_MS ?? "", 10) || 15 * 60 * 1000;
         const pollIntervalMs = 2000;
         const start = Date.now();
+        if (this.engine === "custom") {
+            this.logger.info(`Connecting to external OpenAI-compatible server at ${this.customBaseURL ?? "(no base URL configured)"} — serving begins once /v1/models exposes a model`, {
+                engineBaseURL: this.customBaseURL ?? ""
+            });
+        }
         while (Date.now() - start < maxWaitMs) {
             if (this.lifecycleState === "stopping" || this.stopRequested) {
                 throw new Error("LLM engine startup interrupted by stop request");
@@ -126176,6 +126232,13 @@ class ModelManager extends EventEmitter {
             await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
         }
         const stderrTail = this.engineProcess?.stderr?.slice(-1e3);
+        if (this.engine === "custom") {
+            const baseURL = this.customBaseURL ?? "(missing)";
+            const discovered = this.discoveredModelNames;
+            throw new Error(discovered.length > 0
+                ? `Custom engine endpoint at ${baseURL} exposed models (${discovered.join(", ")}) but never reported ready within ${Math.round(maxWaitMs / 1000)}s`
+                : `Custom engine endpoint at ${baseURL} did not expose a model via /v1/models within ${Math.round(maxWaitMs / 1000)}s. Verify the server is running and reachable from conduit (if conduit runs in a container, use the host address instead of localhost)`);
+        }
         throw new Error(stderrTail
             ? `LLM engine failed readiness checks within timeout. Last engine output:\n${stderrTail}`
             : "LLM engine failed readiness checks within timeout");
@@ -127313,10 +127376,22 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
     });
     return {
         body: monitoredResponse.stream,
-        headers: Object.fromEntries(response.headers.entries()),
+        headers: buildProxyResponseHeaders(response.headers, Boolean(servedModelName)),
         status: response.status
     };
 }
+/**
+ * Forwards upstream headers, dropping content-length when the response body
+ * is rewritten (model-name substitution changes its length; a stale
+ * content-length desyncs the stream and aborts downstream readers).
+ */
+function buildProxyResponseHeaders(headers, rewriteActive) {
+    const forwarded = Object.fromEntries(headers.entries());
+    if (rewriteActive) {
+        delete forwarded["content-length"];
+    }
+    return forwarded;
+}
 async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointId, logger, modelID, modelManager, path, reportMetrics, signal }) {
     function normalizeTokenCount(value) {
         if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
@@ -127495,7 +127570,7 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
         });
     return {
         body: monitoredResponse.stream,
-        headers: Object.fromEntries(response.headers.entries()),
+        headers: buildProxyResponseHeaders(response.headers, Boolean(servedModelName)),
         status: response.status
     };
 }
@@ -159347,23 +159422,33 @@ async function createApplication({ abortController, apiClient, configuration, lo
         logger.info("Engine start requested");
         reporter.beginStartup(new Date());
         try {
-            conduitStateManager.setState({
-                modelFileName,
-                modelName,
-                state: "downloadingModelFiles",
-                totalProgress: {
-                    file: 0,
-                    total: 0
-                }
-            });
-            await conduitStateReportManager.reportNow();
+            if (modelManager.engine === "custom") {
+                conduitStateManager.setState({
+                    state: "bootingEngine"
+                });
+                await conduitStateReportManager.reportNow();
+            }
+            else {
+                conduitStateManager.setState({
+                    modelFileName,
+                    modelName,
+                    state: "downloadingModelFiles",
+                    totalProgress: {
+                        file: 0,
+                        total: 0
+                    }
+                });
+                await conduitStateReportManager.reportNow();
+            }
             await modelManager.prepare({
                 onDownloadProgress: reportDownloadProgress
             });
-            conduitStateManager.setState({
-                state: "bootingEngine"
-            });
-            await conduitStateReportManager.reportNow();
+            if (modelManager.engine !== "custom") {
+                conduitStateManager.setState({
+                    state: "bootingEngine"
+                });
+                await conduitStateReportManager.reportNow();
+            }
             await modelManager.start();
         }
         catch (error) {
@@ -364321,16 +364406,15 @@ function buildSourceCreateBody(options) {
     if (!options.engine) {
         throw new Error("--engine is required (engine ID from `engine create`)");
     }
-    if (!options.model) {
-        throw new Error("--model is required (model ID from `models create`)");
-    }
     validateULID({ flagName: "--engine", value: options.engine });
-    validateULID({ flagName: "--model", value: options.model });
     const body = {
         engineId: options.engine,
-        modelID: options.model,
+        modelID: options.model ?? null,
         name: options.name
     };
+    if (options.model !== undefined) {
+        validateULID({ flagName: "--model", value: options.model });
+    }
     if (options.quant !== undefined) {
         validateQuant(options.quant);
         body.quantizationLabel = options.quant;
@@ -364372,7 +364456,7 @@ function registerSourceCommands({ program }) {
         .option("--engine <id>", "Engine ID to run this source with (required)")
         .option("--id <ulid>", "Target an existing source by ID (requires --update)")
         .option("--key <value>", "API key (required, no environment variable fallback)")
-        .option("--model <id>", "Model ID to serve (required)")
+        .option("--model <id>", "Model ID to serve (required unless the engine type is custom)")
         .option("--name <name>", "Source name (matched by --update)")
         .option("--quant <label>", "Quantization variant label (eg Q4_K_M)")
         .option("--update", "Update an existing source matched by name (or --id) instead of erroring")

package/dist/modelManagement/ModelManager.d.ts

@@ -19,8 +19,10 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
     readonly contextLength: number | null;
     protected readonly logger: Logger;
     private discoveredModelNames;
+    private customProbeCount;
     private engineProcess;
     private healthPollInterval;
+    private lastCustomReadinessReport;
     private lastEngineError;
     private lifecycleState;
     private downloadLockHandle;
@@ -55,6 +57,7 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
     private get customBaseURL();
     private checkEngineReadiness;
     private checkCustomReadiness;
+    private reportCustomReadinessChange;
     private checkGenericHealthReadiness;
     private checkLlamacppReadiness;
     private checkVLLMReadiness;

package/dist/utils/openai.d.ts

@@ -44,6 +44,12 @@ export declare function proxyEmbeddingsRoute({ body, conduitConfiguration, endpo
     status: number;
     statusText: string;
 }>;
+/**
+ * Forwards upstream headers, dropping content-length when the response body
+ * is rewritten (model-name substitution changes its length; a stale
+ * content-length desyncs the stream and aborts downstream readers).
+ */
+export declare function buildProxyResponseHeaders(headers: Headers, rewriteActive: boolean): Record<string, string>;
 export declare function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointId, logger, modelID, modelManager, path, reportMetrics, signal }: {
     body: unknown;
     conduitConfiguration: InferenceAgentConfiguration;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.114.0",
+  "version": "1.115.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },