@infersec/conduit 1.100.0 → 1.102.0

package/dist/modelManagement/ModelManager.d.ts

@@ -33,6 +33,7 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
         model: LLMModel;
         root: string;
     });
+    verifyChatTemplate(): Promise<string | null>;
     fetchOpenAI(path: string, opts?: RequestInit): Promise<Response>;
     prepare({ onDownloadProgress }?: {
         onDownloadProgress?: (update: ModelDownloadProgressUpdate) => void;

package/dist/modelManagement/chatTemplate.d.ts

@@ -0,0 +1,41 @@
+import { type LLMModel } from "@infersec/definitions";
+export declare function getChatTemplateLocalPath(targetDirectory: string): string;
+/**
+ * Materializes the model's chat template override into the model directory:
+ * - Always writes the canonical copy for flag-based engines.
+ * - For engines that read the model directory (exllamav3, mlx-lm), also writes
+ *   the transformers-style `chat_template.jinja` drop-in.
+ * - When no override is configured, removes any previously materialized files,
+ *   restoring the model repo's own `chat_template.jinja` if we replaced it.
+ */
+export declare function materializeChatTemplate({ engine, huggingFaceToken, model, targetDirectory }: {
+    engine: string;
+    huggingFaceToken?: string | null;
+    model: LLMModel;
+    targetDirectory: string;
+}): Promise<void>;
+/**
+ * CLI arguments applying the materialized chat template for flag-based engines.
+ * Returns an empty array when no override exists, the engine applies templates
+ * from the model directory, or the engine does not support overrides.
+ */
+export declare function getChatTemplateEngineArgs({ engine, model, targetDirectory }: {
+    engine: string;
+    model: LLMModel;
+    targetDirectory: string;
+}): Promise<string[]>;
+/**
+ * Post-start verification for llama.cpp: compares the template the server
+ * reports via `/props` against the materialized override file. Returns a
+ * warning message on mismatch, or null when the template applied cleanly
+ * (or verification is not applicable).
+ */
+export declare function verifyEngineChatTemplate({ engine, enginePort, logger, model, targetDirectory }: {
+    engine: string;
+    enginePort: number;
+    logger: {
+        warn: (message: string, meta?: Record<string, unknown>) => void;
+    };
+    model: LLMModel;
+    targetDirectory: string;
+}): Promise<string | null>;

package/dist/requestHandlers/createConduitOpenAIAPIReferenceHandlers.d.ts

@@ -87,7 +87,7 @@ export declare function createPostChatCompletionsHandler(options: {
                 type: "image_url";
                 image_url: {
                     url: string;
-                    detail?: "auto" | "low" | "high" | undefined;
+                    detail?: "low" | "high" | "auto" | undefined;
                 };
             })[];
             name?: string | undefined;
@@ -118,6 +118,7 @@ export declare function createPostChatCompletionsHandler(options: {
             name: string;
         })[];
         model: string;
+        chat_template_kwargs?: Record<string, unknown> | null | undefined;
         frequency_penalty?: number | null | undefined;
         function_call?: "none" | "auto" | {
             name: string;
@@ -133,6 +134,7 @@ export declare function createPostChatCompletionsHandler(options: {
         max_tokens?: number | null | undefined;
         n?: number | null | undefined;
         presence_penalty?: number | null | undefined;
+        reasoning_effort?: "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | null | undefined;
         response_format?: {
             type: "text" | "json_object";
         } | undefined;

package/dist/utils/openai.d.ts

@@ -1,8 +1,24 @@
 import { Readable } from "node:stream";
-import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definitions";
+import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, type LLMModel, type ULID } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
+type PlainObject = Record<string, unknown>;
+/**
+ * Builds `chat_template_kwargs` for engines that read template variables at
+ * render time. Only active when the model carries a template override or
+ * thinking config; otherwise the body forwards untouched (engines like vLLM
+ * consume top-level `reasoning_effort` natively for supported models).
+ *
+ * Precedence (highest wins):
+ * 1. Request `chat_template_kwargs` (per key)
+ * 2. Request top-level `reasoning_effort`
+ * 3. Model thinking config defaults
+ */
+export declare function applyChatTemplateKwargs({ body, model }: {
+    body: PlainObject;
+    model: LLMModel;
+}): PlainObject;
 export declare function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, logger, modelID, modelManager, reportMetrics, signal }: {
     body: unknown;
     conduitConfiguration: InferenceAgentConfiguration;
@@ -39,3 +55,4 @@ export declare function proxyOpenAIStreamingRoute({ body, conduitConfiguration,
     status: number;
     statusText: string;
 }>;
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.100.0",
+  "version": "1.102.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },

package/dist/modelManagement/extraArgs.d.ts

@@ -1 +0,0 @@
-export declare function parseExtraArgs(extraArgs: unknown): Array<string>;