@infersec/conduit 1.34.0 → 1.35.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-BXwggMaM.js';
+import { a as asError, s as startInferenceAgent } from './start-CdILFvRO.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-BXwggMaM.js';
+import { s as startInferenceAgent, a as asError } from './start-CdILFvRO.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/requestHandlers/createConduitAnthropicAPIReferenceHandlers.d.ts

@@ -0,0 +1,119 @@
+import { API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE } from "@infersec/definitions";
+import { implementAPIReference } from "@infersec/fetch";
+import { Logger } from "@infersec/logger";
+import { APIClient } from "../apiClient/index.js";
+import { Configuration } from "../configuration.js";
+import { ModelManager } from "../modelManagement/ModelManager.js";
+type ConduitAnthropicAPIReferenceHandlers = Parameters<typeof implementAPIReference<typeof API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE>>[0]["api"];
+export declare function createConduitAnthropicAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger }: {
+    apiClient: APIClient;
+    configuration: Configuration;
+    getModelID: () => string;
+    getModelManager: () => ModelManager;
+    logger: Logger;
+}): ConduitAnthropicAPIReferenceHandlers;
+export declare function createPostMessagesHandler(options: {
+    apiClient: APIClient;
+    configuration: Configuration;
+    getModelID: () => string;
+    getModelManager: () => ModelManager;
+    logger: Logger;
+}): (params: {
+    req: import("@infersec/fetch").APIRequest;
+    res: import("@infersec/fetch").APIResponse;
+    parameters: Record<string, never>;
+    query: Record<string, never>;
+    body: {
+        max_tokens: number;
+        messages: ({
+            content: string | ({
+                text: string;
+                type: "text";
+            } | {
+                source: {
+                    data: string;
+                    media_type: "image/gif" | "image/jpeg" | "image/png" | "image/webp";
+                    type: "base64";
+                } | {
+                    type: "url";
+                    url: string;
+                };
+                type: "image";
+            } | {
+                tool_use_id: string;
+                content: string | {
+                    text: string;
+                    type: "text";
+                }[];
+                type: "tool_result";
+                is_error?: boolean | undefined;
+            })[];
+            role: "user";
+        } | {
+            content: string | ({
+                text: string;
+                type: "text";
+            } | {
+                id: string;
+                input: Record<string, unknown>;
+                name: string;
+                type: "tool_use";
+            } | {
+                [x: string]: unknown;
+                thinking: string;
+                type: "thinking";
+            } | {
+                [x: string]: unknown;
+                data: string;
+                type: "redacted_thinking";
+            })[];
+            role: "assistant";
+        })[];
+        model: string;
+        metadata?: {
+            user_id?: string | undefined;
+        } | undefined;
+        stop_sequences?: string[] | undefined;
+        stream?: boolean | undefined;
+        system?: string | {
+            text: string;
+            type: "text";
+        }[] | undefined;
+        temperature?: number | undefined;
+        thinking?: {
+            budget_tokens: number;
+            type: "enabled";
+        } | undefined;
+        tool_choice?: "none" | "any" | "auto" | {
+            type: "auto";
+            disable_parallel_tool_use?: boolean | undefined;
+        } | {
+            type: "any";
+            disable_parallel_tool_use?: boolean | undefined;
+        } | {
+            type: "none";
+            disable_parallel_tool_use?: boolean | undefined;
+        } | {
+            name: string;
+            type: "tool";
+            disable_parallel_tool_use?: boolean | undefined;
+        } | undefined;
+        tools?: {
+            input_schema: Record<string, unknown>;
+            name: string;
+            description?: string | undefined;
+        }[] | undefined;
+        top_k?: number | undefined;
+        top_p?: number | undefined;
+    };
+    responseSchema: undefined;
+}) => Promise<{
+    body: import("stream").Readable;
+    headers?: Record<string, string>;
+    status: number;
+} | {
+    headers?: Record<string, string>;
+    status: number;
+    statusText: string;
+}>;
+export {};

package/dist/{start-BXwggMaM.js → start-CdILFvRO.js}

@@ -460,7 +460,7 @@ const allowsEval = cached(() => {
         return false;
     }
 });
-function isPlainObject$2(o) {
+function isPlainObject$3(o) {
     if (isObject$1(o) === false)
         return false;
     // modified constructor
@@ -480,7 +480,7 @@ function isPlainObject$2(o) {
     return true;
 }
 function shallowClone(o) {
-    if (isPlainObject$2(o))
+    if (isPlainObject$3(o))
         return { ...o };
     if (Array.isArray(o))
         return [...o];
@@ -665,7 +665,7 @@ function omit(schema, mask) {
     return clone(schema, def);
 }
 function extend(schema, shape) {
-    if (!isPlainObject$2(shape)) {
+    if (!isPlainObject$3(shape)) {
         throw new Error("Invalid input to extend: expected a plain object");
     }
     const checks = schema._zod.def.checks;
@@ -684,7 +684,7 @@ function extend(schema, shape) {
     return clone(schema, def);
 }
 function safeExtend(schema, shape) {
-    if (!isPlainObject$2(shape)) {
+    if (!isPlainObject$3(shape)) {
         throw new Error("Invalid input to safeExtend: expected a plain object");
     }
     const def = {
@@ -944,7 +944,7 @@ var util$6 = /*#__PURE__*/Object.freeze({
     getSizableOrigin: getSizableOrigin,
     hexToUint8Array: hexToUint8Array,
     isObject: isObject$1,
-    isPlainObject: isPlainObject$2,
+    isPlainObject: isPlainObject$3,
     issue: issue,
     joinValues: joinValues,
     jsonStringifyReplacer: jsonStringifyReplacer,
@@ -3154,7 +3154,7 @@ function mergeValues(a, b) {
     if (a instanceof Date && b instanceof Date && +a === +b) {
         return { valid: true, data: a };
     }
-    if (isPlainObject$2(a) && isPlainObject$2(b)) {
+    if (isPlainObject$3(a) && isPlainObject$3(b)) {
         const bKeys = Object.keys(b);
         const sharedKeys = Object.keys(a).filter((key) => bKeys.indexOf(key) !== -1);
         const newObj = { ...a, ...b };
@@ -3286,7 +3286,7 @@ const $ZodRecord = /*@__PURE__*/ $constructor("$ZodRecord", (inst, def) => {
     $ZodType.init(inst, def);
     inst._zod.parse = (payload, ctx) => {
         const input = payload.value;
-        if (!isPlainObject$2(input)) {
+        if (!isPlainObject$3(input)) {
             payload.issues.push({
                 expected: "record",
                 code: "invalid_type",
@@ -15004,6 +15004,167 @@ const API_SERVICE_CONDUIT_API_REFERENCE = {
     }
 };
 
+// ==================== CONTENT BLOCKS ====================
+const AnthropicTextContentBlockSchema = object({
+    text: string$1(),
+    type: literal("text")
+});
+const AnthropicToolUseContentBlockSchema = object({
+    id: string$1(),
+    input: record(string$1(), unknown()),
+    name: string$1(),
+    type: literal("tool_use")
+});
+const AnthropicToolResultContentBlockSchema = object({
+    tool_use_id: string$1(),
+    content: union([
+        string$1(),
+        array(object({ text: string$1(), type: literal("text") }))
+    ]),
+    is_error: boolean$1().optional(),
+    type: literal("tool_result")
+});
+const AnthropicImageContentBlockSchema = object({
+    source: discriminatedUnion("type", [
+        object({
+            data: string$1(),
+            media_type: _enum(["image/gif", "image/jpeg", "image/png", "image/webp"]),
+            type: literal("base64")
+        }),
+        object({
+            type: literal("url"),
+            url: string$1()
+        })
+    ]),
+    type: literal("image")
+});
+const AnthropicThinkingContentBlockSchema = object({
+    thinking: string$1(),
+    type: literal("thinking")
+})
+    .passthrough();
+const AnthropicRedactedThinkingContentBlockSchema = object({
+    data: string$1(),
+    type: literal("redacted_thinking")
+})
+    .passthrough();
+// ==================== INPUT CONTENT (for messages) ====================
+const AnthropicInputContentSchema = union([
+    AnthropicTextContentBlockSchema,
+    AnthropicImageContentBlockSchema,
+    AnthropicToolResultContentBlockSchema
+]);
+// ==================== MESSAGE PARAMS ====================
+const AnthropicUserMessageParamSchema = object({
+    content: union([string$1(), array(AnthropicInputContentSchema)]),
+    role: literal("user")
+});
+const AnthropicAssistantMessageParamSchema = object({
+    content: union([
+        string$1(),
+        array(union([
+            AnthropicTextContentBlockSchema,
+            AnthropicToolUseContentBlockSchema,
+            AnthropicThinkingContentBlockSchema,
+            AnthropicRedactedThinkingContentBlockSchema
+        ]))
+    ]),
+    role: literal("assistant")
+});
+const AnthropicMessageParamSchema = discriminatedUnion("role", [
+    AnthropicUserMessageParamSchema,
+    AnthropicAssistantMessageParamSchema
+]);
+// ==================== TOOL DEFINITIONS ====================
+const AnthropicToolSchema = object({
+    description: string$1().optional(),
+    input_schema: record(string$1(), unknown()),
+    name: string$1()
+});
+// ==================== MESSAGES CREATE PARAMS ====================
+const AnthropicMessagesCreateParamsSchema = object({
+    max_tokens: number$1().int().positive(),
+    messages: array(AnthropicMessageParamSchema),
+    model: string$1(),
+    metadata: object({
+        user_id: string$1().optional()
+    })
+        .optional(),
+    stop_sequences: array(string$1()).optional(),
+    stream: boolean$1().optional(),
+    system: union([string$1(), array(object({ text: string$1(), type: literal("text") }))])
+        .optional(),
+    temperature: number$1().min(0).max(1).optional(),
+    thinking: object({
+        budget_tokens: number$1().int().min(1024),
+        type: literal("enabled")
+    })
+        .optional(),
+    tool_choice: union([
+        literal("auto"),
+        literal("any"),
+        literal("none"),
+        object({
+            disable_parallel_tool_use: boolean$1().optional(),
+            type: literal("auto")
+        }),
+        object({
+            disable_parallel_tool_use: boolean$1().optional(),
+            type: literal("any")
+        }),
+        object({
+            disable_parallel_tool_use: boolean$1().optional(),
+            type: literal("none")
+        }),
+        object({
+            disable_parallel_tool_use: boolean$1().optional(),
+            name: string$1(),
+            type: literal("tool")
+        })
+    ])
+        .optional(),
+    tools: array(AnthropicToolSchema).optional(),
+    top_k: number$1().int().positive().optional(),
+    top_p: number$1().min(0).max(1).optional()
+});
+// ==================== RESPONSE SCHEMAS ====================
+const AnthropicUsageSchema = object({
+    cache_creation_input_tokens: number$1().optional(),
+    cache_read_input_tokens: number$1().optional(),
+    input_tokens: number$1(),
+    output_tokens: number$1()
+});
+object({
+    content: array(union([
+        AnthropicTextContentBlockSchema,
+        AnthropicToolUseContentBlockSchema,
+        AnthropicThinkingContentBlockSchema,
+        AnthropicRedactedThinkingContentBlockSchema
+    ])),
+    id: string$1(),
+    model: string$1(),
+    role: literal("assistant"),
+    stop_reason: _enum(["end_turn", "max_tokens", "pause_turn", "refusal", "stop_sequence", "tool_use"])
+        .nullable(),
+    stop_sequence: string$1().nullable(),
+    type: literal("message"),
+    usage: AnthropicUsageSchema
+});
+
+const API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE = {
+    "/v1/messages": {
+        POST: {
+            auth: {
+                type: "shared-secret"
+            },
+            body: AnthropicMessagesCreateParamsSchema,
+            response: {
+                type: "text-stream"
+            }
+        }
+    }
+};
+
 /**
  * Coerce non-string values to JSON strings. Some LLM backends (e.g. llama.cpp)
  * return tool_calls arguments as parsed objects instead of JSON strings, which
@@ -15323,6 +15484,15 @@ const API_CLIENT_CONDUIT_OPENAI_REFERENCE = {
     }
 };
 
+({
+    "/api/inferencing/:endpointID/anthropic/v1/messages": {
+        POST: {
+            parameters: {
+                endpointID: ULIDSchema.describe("Endpoint identifier")
+            }}
+    }
+});
+
 ({
     "/api/inferencing/:endpointID/oai/v1/chat/completions": {
         POST: {
@@ -109253,11 +109423,11 @@ function logEngineMetrics({ agentEngineType, error, level, logger, requestBodyBy
     logger[level](metricsMessage, attributes);
 }
 
-function isPlainObject$1(value) {
+function isPlainObject$2(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-function serializeRequestBody(body) {
-    if (!isPlainObject$1(body)) {
+function serializeRequestBody$1(body) {
+    if (!isPlainObject$2(body)) {
         const payload = typeof body === "string" ? body : JSON.stringify(body);
         return {
             bytes: Buffer.byteLength(payload, "utf8"),
@@ -109266,7 +109436,7 @@ function serializeRequestBody(body) {
     }
     const requestPayload = { ...body };
     const streamOptions = requestPayload.stream_options;
-    const normalizedStreamOptions = isPlainObject$1(streamOptions)
+    const normalizedStreamOptions = isPlainObject$2(streamOptions)
         ? { ...streamOptions }
         : {};
     normalizedStreamOptions.include_usage = true;
@@ -109277,7 +109447,7 @@ function serializeRequestBody(body) {
         payload
     };
 }
-function calculateTokensPerSecond$1({ durationMs, totalTokens }) {
+function calculateTokensPerSecond$2({ durationMs, totalTokens }) {
     if (durationMs <= 0) {
         return 0;
     }
@@ -109302,7 +109472,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
             });
         });
     }
-    const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody(body);
+    const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody$1(body);
     const requestStartedAt = Date.now();
     const requestBody = JSON.parse(serializedBody);
     const streamRequested = requestBody.stream === true;
@@ -109326,7 +109496,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
             responseBytes,
             successful: !error,
             timeToFirstTokenMs,
-            tokensPerSecond: calculateTokensPerSecond$1({
+            tokensPerSecond: calculateTokensPerSecond$2({
                 durationMs: latencyMs,
                 totalTokens
             }),
@@ -109383,8 +109553,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
             error: responseError,
             requestUrl: path,
             statusCode: response.status,
-            statusText: responseStatusText,
-            responseBody: responseBody ?? undefined
+            statusText: responseStatusText
         });
         if (!response.body) {
             return {
@@ -109531,6 +109700,385 @@ function createPostCompletionsHandler(options) {
     return createConduitOpenAIAPIReferenceHandlers(options)["/v1/completions"].POST;
 }
 
+function isPlainObject$1(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function serializeRequestBody(body) {
+    const payload = typeof body === "string" ? body : JSON.stringify(body);
+    return {
+        bytes: Buffer.byteLength(payload, "utf8"),
+        payload
+    };
+}
+function calculateTokensPerSecond$1({ durationMs, totalTokens }) {
+    if (durationMs <= 0)
+        return 0;
+    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0)
+        return 0;
+    return Math.round(tokensPerSecond);
+}
+function normalizeTokenCount(value) {
+    if (typeof value === "number" && Number.isFinite(value) && value >= 0)
+        return value;
+    return 0;
+}
+function extractAnthropicStreamUsage(line) {
+    if (!line.startsWith("data:"))
+        return null;
+    const payload = line.slice(5).trim();
+    if (!payload)
+        return null;
+    try {
+        const parsed = JSON.parse(payload);
+        if (!isPlainObject$1(parsed))
+            return null;
+        if (parsed.type === "message_start" && isPlainObject$1(parsed.message)) {
+            const msgObj = parsed.message;
+            const usage = msgObj.usage;
+            if (isPlainObject$1(usage)) {
+                return {
+                    inputTokens: typeof usage.input_tokens === "number" ? usage.input_tokens : null
+                };
+            }
+        }
+        if (parsed.type === "message_delta" && isPlainObject$1(parsed.usage)) {
+            return {
+                outputTokens: typeof parsed.usage.output_tokens === "number"
+                    ? parsed.usage.output_tokens
+                    : null
+            };
+        }
+    }
+    catch {
+        // ignore
+    }
+    return null;
+}
+function extractAnthropicNonStreamUsage(body) {
+    try {
+        const parsed = JSON.parse(body);
+        if (!isPlainObject$1(parsed) || !isPlainObject$1(parsed.usage))
+            return null;
+        const usage = parsed.usage;
+        return {
+            inputTokens: typeof usage.input_tokens === "number" ? usage.input_tokens : null,
+            outputTokens: typeof usage.output_tokens === "number" ? usage.output_tokens : null
+        };
+    }
+    catch {
+        return null;
+    }
+}
+async function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics }) {
+    function reportMetricsSafe(payload) {
+        reportMetrics(payload).catch(error => {
+            logger.warn("Failed to upload LLM prompt metrics", {
+                error: asError(error),
+                requestUrl: "/v1/messages"
+            });
+        });
+    }
+    const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody(body);
+    const requestStartedAt = Date.now();
+    const requestBody = JSON.parse(serializedBody);
+    const streamRequested = requestBody.stream === true;
+    const onMonitoringComplete = ({ durationMs, error, responseBytes, usage }) => {
+        const promptTokens = normalizeTokenCount(usage?.inputTokens);
+        const completionTokens = normalizeTokenCount(usage?.outputTokens);
+        const totalTokens = promptTokens + completionTokens;
+        const latencyMs = Math.max(0, durationMs);
+        reportMetricsSafe({
+            bytes: requestBodyBytes + responseBytes,
+            completionTokens,
+            engine: configuration.agentEngineType,
+            endpointId: null,
+            latencyMs,
+            modelId: modelID,
+            promptTokens,
+            requestBytes: requestBodyBytes,
+            requestId: null,
+            requestMethod: "POST",
+            requestPath: "/v1/messages",
+            responseBytes,
+            successful: !error,
+            timeToFirstTokenMs: null,
+            tokensPerSecond: calculateTokensPerSecond$1({
+                durationMs: latencyMs,
+                totalTokens
+            }),
+            totalTokens
+        });
+    };
+    const response = await modelManager
+        .fetchOpenAI("/v1/messages", {
+        body: serializedBody,
+        headers: {
+            "Content-Type": "application/json"
+        },
+        method: "POST"
+    })
+        .catch(error => {
+        logEngineMetrics({
+            agentEngineType: configuration.agentEngineType,
+            error: asError(error),
+            level: "error",
+            logger,
+            requestBodyBytes,
+            requestPath: "/v1/messages",
+            responseBytes: 0,
+            usage: null
+        });
+        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
+        reportMetricsSafe({
+            bytes: requestBodyBytes,
+            completionTokens: 0,
+            engine: configuration.agentEngineType,
+            endpointId: null,
+            latencyMs,
+            modelId: modelID,
+            promptTokens: 0,
+            requestBytes: requestBodyBytes,
+            requestId: null,
+            requestMethod: "POST",
+            requestPath: "/v1/messages",
+            responseBytes: 0,
+            successful: false,
+            timeToFirstTokenMs: null,
+            tokensPerSecond: 0,
+            totalTokens: 0
+        });
+        throw error;
+    });
+    const responseStatusText = response.statusText ?? "Upstream request failed";
+    if (!response.ok) {
+        const responseClone = response.clone();
+        const responseBody = await responseClone.text().catch(() => null);
+        const responseError = new Error(responseBody
+            ? `Upstream error response: ${responseBody}`
+            : "Upstream error response: empty body");
+        logger.error("LLM engine request failed", {
+            error: responseError,
+            requestUrl: "/v1/messages",
+            statusCode: response.status,
+            statusText: responseStatusText
+        });
+    }
+    if (!response.body) {
+        logEngineMetrics({
+            agentEngineType: configuration.agentEngineType,
+            level: response.ok ? "info" : "error",
+            logger,
+            requestBodyBytes,
+            requestPath: "/v1/messages",
+            responseBytes: 0,
+            usage: null
+        });
+        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
+        reportMetricsSafe({
+            bytes: requestBodyBytes,
+            completionTokens: 0,
+            engine: configuration.agentEngineType,
+            endpointId: null,
+            latencyMs,
+            modelId: modelID,
+            promptTokens: 0,
+            requestBytes: requestBodyBytes,
+            requestId: null,
+            requestMethod: "POST",
+            requestPath: "/v1/messages",
+            responseBytes: 0,
+            successful: false,
+            timeToFirstTokenMs: null,
+            tokensPerSecond: 0,
+            totalTokens: 0
+        });
+        return {
+            status: response.status,
+            statusText: responseStatusText
+        };
+    }
+    const passThrough = new PassThrough();
+    let responseBytes = 0;
+    let completed = false;
+    const usage = { inputTokens: null, outputTokens: null };
+    const upstreamError = response.ok
+        ? null
+        : new Error(`Upstream error: ${response.status} ${responseStatusText}`);
+    function finalize(error) {
+        if (completed)
+            return;
+        completed = true;
+        onMonitoringComplete({
+            durationMs: Math.max(0, Date.now() - requestStartedAt),
+            error,
+            responseBytes,
+            usage
+        });
+    }
+    const rawBody = Readable.fromWeb(response.body);
+    if (streamRequested) {
+        let buffer = "";
+        rawBody.on("data", (chunk) => {
+            const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            responseBytes += chunkBuffer.length;
+            buffer += chunkBuffer.toString("utf8");
+            const lines = buffer.split("\n");
+            buffer = lines.pop() ?? "";
+            for (const line of lines) {
+                const extracted = extractAnthropicStreamUsage(line.trim());
+                if (extracted?.inputTokens !== undefined && extracted.inputTokens !== null) {
+                    usage.inputTokens = extracted.inputTokens;
+                }
+                if (extracted?.outputTokens !== undefined && extracted.outputTokens !== null) {
+                    usage.outputTokens = extracted.outputTokens;
+                }
+            }
+            passThrough.write(chunkBuffer);
+        });
+        rawBody.once("error", err => {
+            const normalizedError = asError(err);
+            logEngineMetrics({
+                agentEngineType: configuration.agentEngineType,
+                error: normalizedError,
+                level: "error",
+                logger,
+                requestBodyBytes,
+                requestPath: "/v1/messages",
+                responseBytes,
+                usage: null
+            });
+            finalize(normalizedError);
+            passThrough.destroy(normalizedError);
+        });
+        rawBody.once("end", () => {
+            logEngineMetrics({
+                agentEngineType: configuration.agentEngineType,
+                level: upstreamError ? "error" : "info",
+                logger,
+                requestBodyBytes,
+                requestPath: "/v1/messages",
+                responseBytes,
+                usage: null
+            });
+            finalize(upstreamError);
+            passThrough.end();
+        });
+        rawBody.once("close", () => {
+            if (completed) {
+                if (!passThrough.writableEnded)
+                    passThrough.end();
+                return;
+            }
+            const closeError = new Error("Engine response stream closed before completion");
+            logEngineMetrics({
+                agentEngineType: configuration.agentEngineType,
+                error: closeError,
+                level: "error",
+                logger,
+                requestBodyBytes,
+                requestPath: "/v1/messages",
+                responseBytes,
+                usage: null
+            });
+            finalize(closeError);
+            if (!passThrough.writableEnded)
+                passThrough.end();
+        });
+    }
+    else {
+        const chunks = [];
+        rawBody.on("data", (chunk) => {
+            const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            responseBytes += chunkBuffer.length;
+            chunks.push(chunkBuffer);
+            passThrough.write(chunkBuffer);
+        });
+        rawBody.once("error", err => {
+            const normalizedError = asError(err);
+            logEngineMetrics({
+                agentEngineType: configuration.agentEngineType,
+                error: normalizedError,
+                level: "error",
+                logger,
+                requestBodyBytes,
+                requestPath: "/v1/messages",
+                responseBytes,
+                usage: null
+            });
+            finalize(normalizedError);
+            passThrough.destroy(normalizedError);
+        });
+        rawBody.once("end", () => {
+            const fullBody = Buffer.concat(chunks).toString("utf8");
+            const extractedUsage = extractAnthropicNonStreamUsage(fullBody);
+            if (extractedUsage) {
+                usage.inputTokens = extractedUsage.inputTokens;
+                usage.outputTokens = extractedUsage.outputTokens;
+            }
+            logEngineMetrics({
+                agentEngineType: configuration.agentEngineType,
+                level: upstreamError ? "error" : "info",
+                logger,
+                requestBodyBytes,
+                requestPath: "/v1/messages",
+                responseBytes,
+                usage: null
+            });
+            finalize(upstreamError);
+            passThrough.end();
+        });
+        rawBody.once("close", () => {
+            if (completed) {
+                if (!passThrough.writableEnded)
+                    passThrough.end();
+                return;
+            }
+            const closeError = new Error("Engine response stream closed before completion");
+            logEngineMetrics({
+                agentEngineType: configuration.agentEngineType,
+                error: closeError,
+                level: "error",
+                logger,
+                requestBodyBytes,
+                requestPath: "/v1/messages",
+                responseBytes,
+                usage: null
+            });
+            finalize(closeError);
+            if (!passThrough.writableEnded)
+                passThrough.end();
+        });
+    }
+    return {
+        body: passThrough,
+        headers: Object.fromEntries(response.headers.entries()),
+        status: response.status
+    };
+}
+
+function createConduitAnthropicAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger }) {
+    return {
+        "/v1/messages": {
+            POST: async ({ body }) => {
+                const modelID = getModelID();
+                const modelManager = getModelManager();
+                return proxyAnthropicStreamingRoute({
+                    body,
+                    configuration,
+                    logger,
+                    modelID,
+                    modelManager,
+                    reportMetrics: apiClient.reportPromptMetrics
+                });
+            }
+        }
+    };
+}
+function createPostMessagesHandler(options) {
+    return createConduitAnthropicAPIReferenceHandlers(options)["/v1/messages"].POST;
+}
+
 function createHealthHandler() {
     return (_req, res) => {
         res.status(200).send("OK");
@@ -119544,6 +120092,22 @@ async function createApplication({ abortController, apiClient, configuration, lo
         mount: publicRouter,
         reference: API_CLIENT_CONDUIT_OPENAI_REFERENCE
     });
+    implementAPIReference({
+        api: {
+            "/v1/messages": {
+                POST: createPostMessagesHandler({
+                    apiClient,
+                    configuration,
+                    getModelID: () => conduitConfiguration.targetModel.id,
+                    getModelManager: () => modelManager,
+                    logger
+                })
+            }
+        },
+        logger,
+        mount: publicRouter,
+        reference: API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE
+    });
     handleSSERequests({
         apiURL: configuration.apiURL,
         configuration,

package/dist/utils/anthropic.d.ts

@@ -0,0 +1,20 @@
+import { Readable } from "node:stream";
+import { InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definitions";
+import { Logger } from "@infersec/logger";
+import { Configuration } from "../configuration.js";
+import { ModelManager } from "../modelManagement/ModelManager.js";
+export declare function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics }: {
+    body: unknown;
+    configuration: Configuration;
+    logger: Logger;
+    modelID: ULID;
+    modelManager: ModelManager;
+    reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
+}): Promise<{
+    body: Readable;
+    headers: Record<string, string>;
+    status: number;
+} | {
+    status: number;
+    statusText: string;
+}>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.34.0",
+  "version": "1.35.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },