@infersec/conduit 1.67.0 → 1.69.0

package/dist/configuration.d.ts

@@ -1,12 +1,11 @@
-import { LLMEngine, ULID } from "@infersec/definitions";
 import { z } from "zod";
+import { ULID } from "@infersec/definitions";
 declare const StartModeSchema: z.ZodEnum<{
     idle: "idle";
     auto: "auto";
 }>;
 export type StartMode = z.infer<typeof StartModeSchema>;
 export interface Configuration {
-    agentEngineType: LLMEngine;
     apiKey: string;
     apiURL: string;
     enginePort: number;
@@ -16,7 +15,6 @@ export interface Configuration {
     startMode: StartMode;
 }
 export interface ConfigurationOverrides {
-    agentEngineType?: string;
     apiKey?: string;
     apiURL?: string;
     enginePort?: number;
@@ -27,5 +25,5 @@ export interface ConfigurationOverrides {
 }
 export declare function getConfiguration({ overrides }?: {
     overrides?: ConfigurationOverrides;
-}): Configuration;
+}): Promise<Configuration>;
 export {};

package/dist/modelManagement/ModelManager.d.ts

@@ -1,4 +1,4 @@
-import { LLMEngine, LLMModel } from "@infersec/definitions";
+import { LLMModel } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import EventEmitter from "eventemitter3";
 import { Response } from "undici";
@@ -11,10 +11,10 @@ interface ModelManagerEvents {
 }
 type EngineLifecycleState = "errored" | "running" | "starting" | "stopped" | "stopping";
 export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
-    readonly engine: LLMEngine;
+    readonly engine: string;
+    readonly engineConfig: Record<string, unknown> | null;
     readonly enginePort: number;
     readonly model: LLMModel;
-    readonly parallelism: number | null;
     private uniqueName;
     readonly contextLength: number | null;
     protected readonly logger: Logger;
@@ -23,13 +23,13 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
     private lifecycleState;
     private stopRequested;
     protected readonly modelsDirectory: string;
-    constructor({ contextLength, engine, enginePort, logger, model, parallelism, root }: {
+    constructor({ contextLength, engineConfig, enginePort, engineType, logger, model, root }: {
         contextLength?: number | null;
-        engine: LLMEngine;
+        engineConfig?: Record<string, unknown> | null;
         enginePort: number;
+        engineType: string;
         logger: Logger;
         model: LLMModel;
-        parallelism?: number | null;
         root: string;
     });
     fetchOpenAI(path: string, opts?: RequestInit): Promise<Response>;

package/dist/requestHandlers/createConduitAnthropicAPIReferenceHandlers.d.ts

@@ -1,12 +1,13 @@
-import { API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE } from "@infersec/definitions";
+import { API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE, type InferenceAgentConfiguration } from "@infersec/definitions";
 import { implementAPIReference, type APIRequest } from "@infersec/fetch";
 import { Logger } from "@infersec/logger";
 import { APIClient } from "../apiClient/index.js";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
 type ConduitAnthropicAPIReferenceHandlers = Parameters<typeof implementAPIReference<typeof API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE>>[0]["api"];
-export declare function createConduitAnthropicAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger }: {
+export declare function createConduitAnthropicAPIReferenceHandlers({ apiClient, conduitConfiguration, configuration, getModelID, getModelManager, logger }: {
     apiClient: APIClient;
+    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     getModelID: () => string;
     getModelManager: () => ModelManager;
@@ -14,6 +15,7 @@ export declare function createConduitAnthropicAPIReferenceHandlers({ apiClient,
 }): ConduitAnthropicAPIReferenceHandlers;
 export declare function createPostMessagesHandler(options: {
     apiClient: APIClient;
+    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     getModelID: () => string;
     getModelManager: () => ModelManager;

package/dist/requestHandlers/createConduitOpenAIAPIReferenceHandlers.d.ts

@@ -1,12 +1,13 @@
-import { API_CLIENT_CONDUIT_OPENAI_REFERENCE } from "@infersec/definitions";
+import { API_CLIENT_CONDUIT_OPENAI_REFERENCE, type InferenceAgentConfiguration } from "@infersec/definitions";
 import { implementAPIReference, type APIRequest } from "@infersec/fetch";
 import { Logger } from "@infersec/logger";
 import { APIClient } from "../apiClient/index.js";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
 type ConduitOpenAIAPIReferenceHandlers = Parameters<typeof implementAPIReference<typeof API_CLIENT_CONDUIT_OPENAI_REFERENCE>>[0]["api"];
-export declare function createConduitOpenAIAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger, startup }: {
+export declare function createConduitOpenAIAPIReferenceHandlers({ apiClient, conduitConfiguration, configuration, getModelID, getModelManager, logger, startup }: {
     apiClient: APIClient;
+    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     getModelID: () => string;
     getModelManager: () => ModelManager;
@@ -15,6 +16,7 @@ export declare function createConduitOpenAIAPIReferenceHandlers({ apiClient, con
 }): ConduitOpenAIAPIReferenceHandlers;
 export declare function createGetModelsHandler(options: {
     apiClient: APIClient;
+    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     getModelID: () => string;
     getModelManager: () => ModelManager;
@@ -60,6 +62,7 @@ export declare function createGetModelsHandler(options: {
 }>;
 export declare function createPostChatCompletionsHandler(options: {
     apiClient: APIClient;
+    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     getModelID: () => string;
     getModelManager: () => ModelManager;
@@ -166,6 +169,7 @@ export declare function createPostChatCompletionsHandler(options: {
 }>;
 export declare function createPostCompletionsHandler(options: {
     apiClient: APIClient;
+    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     getModelID: () => string;
     getModelManager: () => ModelManager;

package/dist/sse/handler.d.ts

@@ -1,8 +1,9 @@
-import { InferenceAgentLLMMetricsPayload, type ULID, type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
+import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, type ULID, type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
-export declare function handleSSERequests({ apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }: {
+export declare function handleSSERequests({ apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }: {
     apiURL: string;
+    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     logger: Logger;
     modelID: ULID;

package/dist/utils/anthropic.d.ts

@@ -1,10 +1,11 @@
 import { Readable } from "node:stream";
-import { InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definitions";
+import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
-export declare function proxyAnthropicStreamingRoute({ body, configuration, endpointId, logger, modelID, modelManager, reportMetrics, signal }: {
+export declare function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpointId, logger, modelID, modelManager, reportMetrics, signal }: {
     body: unknown;
+    conduitConfiguration: InferenceAgentConfiguration;
     configuration: Configuration;
     endpointId?: ULID | null;
     logger: Logger;

package/dist/utils/engineMetrics.d.ts

@@ -1,5 +1,4 @@
 import { Readable } from "node:stream";
-import { LLMEngine } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 export interface EngineUsageMetrics {
     completionTokens: number | null;
@@ -8,7 +7,7 @@ export interface EngineUsageMetrics {
     totalTokens: number | null;
 }
 interface EngineMetricsLoggerOptions {
-    agentEngineType: LLMEngine;
+    agentEngineType: string;
     logger: Logger;
     requestBodyBytes: number;
     requestPath: string;
@@ -24,9 +23,9 @@ interface EngineMetricsCompletion {
 interface MonitorEngineResponseOptions extends EngineMetricsLoggerOptions {
     body: Readable;
     contextLength: number | null;
-    engine: LLMEngine;
+    engineConfig: Record<string, unknown> | null;
+    engineType: string;
     onComplete?: (result: EngineMetricsCompletion) => void | Promise<void>;
-    parallelism: number | null;
     requestStartedAt?: number;
 }
 interface EngineMetricsLogOptions extends EngineMetricsLoggerOptions {
@@ -38,7 +37,7 @@ interface EngineMetricsLogOptions extends EngineMetricsLoggerOptions {
 interface MonitorEngineResponseResult {
     stream: Readable;
 }
-export declare function monitorEngineResponseStream({ agentEngineType, body, contextLength, engine, logger, onComplete, parallelism, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
-export declare function monitorEngineResponseSingle({ agentEngineType, body, contextLength, engine, logger, onComplete, parallelism, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
+export declare function monitorEngineResponseStream({ agentEngineType, body, contextLength, engineConfig, engineType, logger, onComplete, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
+export declare function monitorEngineResponseSingle({ agentEngineType, body, contextLength, engineConfig, engineType, logger, onComplete, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
 export declare function logEngineMetrics({ agentEngineType, error, level, logger, requestBodyBytes, requestPath, responseBytes, usage }: EngineMetricsLogOptions): void;
 export {};

package/dist/utils/openai.d.ts

@@ -1,10 +1,11 @@
 import { Readable } from "node:stream";
-import { InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definitions";
+import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
-export declare function proxyOpenAIStreamingRoute({ body, configuration, endpointId, logger, modelID, modelManager, path, reportMetrics, signal }: {
+export declare function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointId, logger, modelID, modelManager, path, reportMetrics, signal }: {
     body: unknown;
+    conduitConfiguration: InferenceAgentConfiguration;
     configuration: Configuration;
     endpointId?: ULID | null;
     logger: Logger;

package/dist/utils/portAllocator.d.ts

@@ -0,0 +1 @@
+export declare function allocatePort(startFrom: number, maxTries?: number): Promise<number>;

package/dist/utils/stripImages.d.ts

@@ -0,0 +1 @@
+export declare function stripImagesFromBody(body: unknown): unknown;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.67.0",
+  "version": "1.69.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },