@livekit/agents-plugin-anthropic 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +21 -0
- package/dist/index.cjs +45 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +3 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -0
- package/dist/llm.cjs +357 -0
- package/dist/llm.cjs.map +1 -0
- package/dist/llm.d.cts +92 -0
- package/dist/llm.d.ts +92 -0
- package/dist/llm.d.ts.map +1 -0
- package/dist/llm.js +328 -0
- package/dist/llm.js.map +1 -0
- package/dist/llm.test.cjs +259 -0
- package/dist/llm.test.cjs.map +1 -0
- package/dist/llm.test.d.cts +2 -0
- package/dist/llm.test.d.ts +2 -0
- package/dist/llm.test.d.ts.map +1 -0
- package/dist/llm.test.js +258 -0
- package/dist/llm.test.js.map +1 -0
- package/dist/models.cjs +17 -0
- package/dist/models.cjs.map +1 -0
- package/dist/models.d.cts +8 -0
- package/dist/models.d.ts +8 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +1 -0
- package/dist/models.js.map +1 -0
- package/package.json +52 -0
- package/src/index.ts +19 -0
- package/src/llm.test.ts +317 -0
- package/src/llm.ts +447 -0
- package/src/models.ts +21 -0
package/dist/llm.cjs.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport Anthropic from '@anthropic-ai/sdk';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n APITimeoutError,\n DEFAULT_API_CONNECT_OPTIONS,\n llm,\n} from '@livekit/agents';\nimport type { ChatModels } from './models.js';\n\n/** Configuration options for the Anthropic LLM plugin. */\nexport interface LLMOptions {\n /** The model identifier to use. */\n model: string | ChatModels;\n /** Anthropic API key. Falls back to the `ANTHROPIC_API_KEY` environment variable. */\n apiKey?: string;\n /** Custom base URL for the Anthropic API. */\n baseURL?: string;\n /** Sampling temperature. */\n temperature?: number;\n /** Pre-configured Anthropic client instance. */\n client?: Anthropic;\n /** Tool selection strategy. */\n toolChoice?: llm.ToolChoice;\n /** Whether to allow parallel tool calls. */\n parallelToolCalls?: boolean;\n /** Maximum number of tokens in the response. Defaults to 4096. */\n maxTokens?: number;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n model: 'claude-sonnet-4-6',\n /* eslint-disable-next-line turbo/no-undeclared-env-vars */\n apiKey: process.env.ANTHROPIC_API_KEY,\n parallelToolCalls: true,\n};\n\n/**\n * Anthropic LLM provider for LiveKit Agents.\n *\n * @remarks\n * Implements the {@link llm.LLM} interface using the Anthropic Messages API.\n * Supports streaming, tool calling, and system prompt isolation required by\n * Claude 3.5+ models.\n */\nexport class LLM extends llm.LLM {\n #opts: LLMOptions;\n #client: Anthropic;\n\n constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n super();\n\n this.#opts = { ...defaultLLMOptions, ...opts };\n if (!this.#opts.apiKey && !this.#opts.client) {\n throw new Error(\n 'Anthropic API key is required, whether as an argument or as $ANTHROPIC_API_KEY',\n );\n }\n\n this.#client =\n this.#opts.client ||\n new Anthropic({\n baseURL: this.#opts.baseURL,\n apiKey: this.#opts.apiKey,\n });\n }\n\n /** @returns Human-readable label for logging. */\n label(): string {\n return 'anthropic.LLM';\n }\n\n /** @returns The model identifier being used. */\n get model(): string {\n return this.#opts.model;\n }\n\n /** @returns The API provider host. */\n get provider(): string {\n try {\n const url = new URL(this.#client.baseURL);\n return url.host;\n } catch {\n return 'api.anthropic.com';\n }\n }\n\n /**\n * Converts a framework ChatContext into Anthropic's message format.\n *\n * @remarks\n * - System prompts are isolated into a separate `TextBlockParam[]` array\n * (required by Claude 3.5+).\n * - `function_call` items are mapped to Anthropic `tool_use` content blocks.\n * - `function_call_output` items are mapped to `tool_result` content blocks.\n * - Consecutive same-role messages are merged to satisfy Anthropic's\n * strict alternating-turn requirement.\n * - A dummy `(empty)` user message is injected if the conversation doesn't\n * start with a user turn.\n * - A dummy user message is appended if the conversation ends with an\n * assistant turn, since Claude 4.6+ does not support prefilling.\n */\n protected _buildAnthropicContext(chatCtx: llm.ChatContext): {\n system: Anthropic.TextBlockParam[];\n messages: Anthropic.MessageParam[];\n } {\n const system: Anthropic.TextBlockParam[] = [];\n const rawMessages: Anthropic.MessageParam[] = [];\n\n for (const msg of chatCtx.items) {\n if (msg.type === 'message') {\n const textContent = msg.textContent || '';\n if (msg.role === 'system' || msg.role === 'developer') {\n system.push({ type: 'text', text: textContent });\n } else if (msg.role === 'user' || msg.role === 'assistant') {\n rawMessages.push({\n role: msg.role,\n content: textContent,\n });\n }\n } else if (msg.type === 'function_call') {\n // Map to Anthropic's tool_use content block (assistant role)\n rawMessages.push({\n role: 'assistant',\n content: [\n {\n type: 'tool_use',\n id: msg.callId,\n name: msg.name,\n input: JSON.parse(msg.args || '{}'),\n },\n ],\n });\n } else if (msg.type === 'function_call_output') {\n // Map to Anthropic's tool_result content block (user role)\n rawMessages.push({\n role: 'user',\n content: [\n {\n type: 'tool_result',\n tool_use_id: msg.callId,\n content: msg.output,\n is_error: msg.isError,\n },\n ],\n });\n }\n }\n\n // Merge consecutive same-role messages (Anthropic requires alternating turns)\n const messages: Anthropic.MessageParam[] = [];\n for (const msg of rawMessages) {\n const prev = messages[messages.length - 1];\n if (prev && prev.role === msg.role) {\n // Merge content into a single content array\n const prevContent = Array.isArray(prev.content)\n ? prev.content\n : [{ type: 'text' as const, text: prev.content as string }];\n const curContent = Array.isArray(msg.content)\n ? msg.content\n : [{ type: 'text' as const, text: msg.content as string }];\n prev.content = [...prevContent, ...curContent] as Anthropic.ContentBlockParam[];\n } else {\n messages.push({ ...msg });\n }\n }\n\n // Anthropic requires conversations to start with a user turn\n if (messages.length === 0 || messages[0]!.role !== 'user') {\n messages.unshift({ role: 'user', content: '(empty)' });\n }\n\n // Claude 4.6+ does not support prefilling (trailing assistant messages).\n if (messages.length > 0 && messages[messages.length - 1]!.role === 'assistant') {\n messages.push({ role: 'user', content: [{ type: 'text', text: '.' }] });\n }\n\n return { system, messages };\n }\n\n /**\n * Creates a streaming chat completion.\n *\n * @remarks\n * Maps `toolChoice` to Anthropic's format:\n * - `\"required\"` → `{ type: \"any\" }`\n * - `\"none\"` → clears tools\n * - `{ type: \"function\", function: { name } }` → `{ type: \"tool\", name }`\n */\n chat({\n chatCtx,\n toolCtx: toolCtxInput,\n connOptions = DEFAULT_API_CONNECT_OPTIONS,\n parallelToolCalls,\n toolChoice,\n extraKwargs,\n }: {\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContextLike;\n connOptions?: APIConnectOptions;\n parallelToolCalls?: boolean;\n toolChoice?: llm.ToolChoice;\n extraKwargs?: Record<string, unknown>;\n }): LLMStream {\n const extras: Record<string, unknown> = { ...extraKwargs };\n\n if (this.#opts.temperature !== undefined) extras.temperature = this.#opts.temperature;\n\n const { system, messages } = this._buildAnthropicContext(chatCtx);\n\n // Build Anthropic tool schemas\n const toolCtx = llm.toToolContext(toolCtxInput);\n const anthropicTools: Anthropic.Tool[] = [];\n if (toolCtx) {\n for (const [name, tool] of llm.sortedToolEntries(toolCtx)) {\n anthropicTools.push({\n name: name,\n description: tool.description || '',\n input_schema: (tool.parameters\n ? llm.toJsonSchema(tool.parameters, false)\n : { type: 'object', properties: {} }) as Anthropic.Tool.InputSchema,\n });\n }\n }\n\n // Map toolChoice and parallelToolCalls to Anthropic format\n const resolvedToolChoice = toolChoice ?? this.#opts.toolChoice;\n const resolvedParallel = parallelToolCalls ?? this.#opts.parallelToolCalls;\n\n if ((resolvedToolChoice || resolvedParallel !== undefined) && anthropicTools.length > 0) {\n let anthropicToolChoice: Record<string, unknown> | undefined = { type: 'auto' };\n\n if (typeof resolvedToolChoice === 'string') {\n if (resolvedToolChoice === 'required') {\n anthropicToolChoice = { type: 'any' };\n } else if (resolvedToolChoice === 'none') {\n // Clear tools entirely when none is requested\n anthropicTools.length = 0;\n anthropicToolChoice = undefined;\n }\n } else if (\n typeof resolvedToolChoice === 'object' &&\n 'type' in resolvedToolChoice &&\n resolvedToolChoice.type === 'function'\n ) {\n const fn = (resolvedToolChoice as { function: { name: string } }).function;\n anthropicToolChoice = { type: 'tool', name: fn.name };\n }\n\n if (anthropicToolChoice) {\n // Map parallelToolCalls\n if (resolvedParallel !== undefined) {\n anthropicToolChoice.disable_parallel_tool_use = !resolvedParallel;\n }\n extras.tool_choice = anthropicToolChoice;\n }\n }\n\n const requestParams: Anthropic.MessageCreateParamsStreaming = {\n model: this.#opts.model,\n messages: messages,\n system: system.length > 0 ? system : undefined,\n tools: anthropicTools.length > 0 ? anthropicTools : undefined,\n stream: true,\n max_tokens: this.#opts.maxTokens || 4096,\n ...extras,\n };\n\n return new LLMStream(this, this.#client, requestParams, chatCtx, toolCtx, connOptions);\n }\n}\n\n/**\n * Streaming response handler for Anthropic Messages API.\n *\n * @remarks\n * Parses SSE events including:\n * - `content_block_start` / `content_block_delta` / `content_block_stop` for text and tool use\n * - `message_start` / `message_delta` for token usage tracking\n * - Chain-of-thought `<thinking>` block filtering when tools are active\n */\nexport class LLMStream extends llm.LLMStream {\n #client: Anthropic;\n #requestParams: Anthropic.MessageCreateParamsStreaming;\n #toolCallId?: string;\n #fncName?: string;\n #fncRawArgs?: string;\n #requestId = '';\n #ignoringCoT = false;\n #inputTokens = 0;\n #outputTokens = 0;\n #cacheCreationTokens = 0;\n #cacheReadTokens = 0;\n #toolCtx?: llm.ToolContext;\n\n constructor(\n llmInst: LLM,\n client: Anthropic,\n requestParams: Anthropic.MessageCreateParamsStreaming,\n chatCtx: llm.ChatContext,\n toolCtx: llm.ToolContext | undefined,\n connOptions: APIConnectOptions,\n ) {\n super(llmInst, { chatCtx, toolCtx, connOptions });\n this.#client = client;\n this.#requestParams = requestParams;\n this.#toolCtx = toolCtx;\n }\n\n protected async run(): Promise<void> {\n let retryable = true;\n this.#toolCallId = undefined;\n this.#fncName = undefined;\n this.#fncRawArgs = undefined;\n this.#requestId = '';\n this.#ignoringCoT = false;\n this.#inputTokens = 0;\n this.#outputTokens = 0;\n this.#cacheCreationTokens = 0;\n this.#cacheReadTokens = 0;\n\n try {\n const stream = await this.#client.messages.create(this.#requestParams, {\n timeout: this.connOptions.timeoutMs,\n });\n for await (const event of stream) {\n if (event.type === 'message_start') {\n this.#requestId = event.message.id;\n this.#inputTokens = event.message.usage.input_tokens;\n this.#outputTokens = event.message.usage.output_tokens;\n this.#cacheCreationTokens = event.message.usage.cache_creation_input_tokens ?? 0;\n this.#cacheReadTokens = event.message.usage.cache_read_input_tokens ?? 0;\n } else if (event.type === 'message_delta') {\n this.#outputTokens = event.usage.output_tokens;\n } else if (\n event.type === 'content_block_start' &&\n event.content_block.type === 'tool_use'\n ) {\n this.#toolCallId = event.content_block.id;\n this.#fncName = event.content_block.name;\n this.#fncRawArgs = '';\n } else if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {\n let text = event.delta.text;\n\n // Filter chain-of-thought <thinking> blocks when tools are active\n if (this.#toolCtx) {\n const thinkingStart = text.indexOf('<thinking>');\n if (thinkingStart >= 0) {\n const preThinking = text.slice(0, thinkingStart);\n if (preThinking) {\n this.queue.put({\n id: this.#requestId,\n delta: { role: 'assistant', content: preThinking },\n });\n retryable = false;\n }\n text = text.slice(thinkingStart + '<thinking>'.length);\n this.#ignoringCoT = true;\n }\n if (this.#ignoringCoT) {\n const thinkingEnd = text.indexOf('</thinking>');\n if (thinkingEnd >= 0) {\n text = text.slice(thinkingEnd + '</thinking>'.length);\n this.#ignoringCoT = false;\n }\n }\n }\n\n if (this.#ignoringCoT) {\n continue;\n }\n\n // A delta can be reduced to an empty string when it was entirely a\n // <thinking> block; don't emit it or mark the stream non-retryable.\n if (text) {\n this.queue.put({\n id: this.#requestId,\n delta: { role: 'assistant', content: text },\n });\n retryable = false;\n }\n } else if (\n event.type === 'content_block_delta' &&\n event.delta.type === 'input_json_delta'\n ) {\n this.#fncRawArgs += event.delta.partial_json;\n } else if (event.type === 'content_block_stop' && this.#toolCallId) {\n this.queue.put({\n id: this.#requestId,\n delta: {\n role: 'assistant',\n toolCalls: [\n llm.FunctionCall.create({\n callId: this.#toolCallId,\n name: this.#fncName || '',\n args: this.#fncRawArgs || '',\n }),\n ],\n },\n });\n this.#toolCallId = undefined;\n this.#fncName = undefined;\n this.#fncRawArgs = undefined;\n retryable = false;\n }\n }\n\n // Emit final usage chunk. Anthropic reports cached tokens separately from\n // input_tokens, so fold them into promptTokens like the Python plugin does.\n const promptTokens = this.#inputTokens + this.#cacheCreationTokens + this.#cacheReadTokens;\n this.queue.put({\n id: this.#requestId,\n usage: {\n completionTokens: this.#outputTokens,\n promptTokens,\n totalTokens: promptTokens + this.#outputTokens,\n promptCachedTokens: this.#cacheReadTokens,\n },\n });\n } catch (e: unknown) {\n if (e instanceof Anthropic.APIError) {\n if (e.status === 408) {\n throw new APITimeoutError({\n message: e.message,\n options: { retryable },\n });\n }\n throw new APIStatusError({\n message: e.message,\n options: {\n statusCode: e.status,\n body: e.error as object,\n retryable: retryable && (e.status === 429 || e.status >= 500),\n },\n });\n }\n throw new APIConnectionError({\n message: e instanceof Error ? e.message : String(e),\n options: { retryable },\n });\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAAsB;AAEtB,oBAMO;AAuBP,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA;AAAA,EAEP,QAAQ,QAAQ,IAAI;AAAA,EACpB,mBAAmB;AACrB;AAUO,MAAM,YAAY,kBAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,CAAC,KAAK,MAAM,UAAU,CAAC,KAAK,MAAM,QAAQ;AAC5C,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,WAAAA,QAAU;AAAA,MACZ,SAAS,KAAK,MAAM;AAAA,MACpB,QAAQ,KAAK,MAAM;AAAA,IACrB,CAAC;AAAA,EACL;AAAA;AAAA,EAGA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,IAAI,QAAgB;AAClB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,WAAmB;AACrB,QAAI;AACF,YAAM,MAAM,IAAI,IAAI,KAAK,QAAQ,OAAO;AACxC,aAAO,IAAI;AAAA,IACb,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiBU,uBAAuB,SAG/B;AACA,UAAM,SAAqC,CAAC;AAC5C,UAAM,cAAwC,CAAC;AAE/C,eAAW,OAAO,QAAQ,OAAO;AAC/B,UAAI,IAAI,SAAS,WAAW;AAC1B,cAAM,cAAc,IAAI,eAAe;AACvC,YAAI,IAAI,SAAS,YAAY,IAAI,SAAS,aAAa;AACrD,iBAAO,KAAK,EAAE,MAAM,QAAQ,MAAM,YAAY,CAAC;AAAA,QACjD,WAAW,IAAI,SAAS,UAAU,IAAI,SAAS,aAAa;AAC1D,sBAAY,KAAK;AAAA,YACf,MAAM,IAAI;AAAA,YACV,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAAA,MACF,WAAW,IAAI,SAAS,iBAAiB;AAEvC,oBAAY,KAAK;AAAA,UACf,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,cACE,MAAM;AAAA,cACN,IAAI,IAAI;AAAA,cACR,MAAM,IAAI;AAAA,cACV,OAAO,KAAK,MAAM,IAAI,QAAQ,IAAI;AAAA,YACpC;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH,WAAW,IAAI,SAAS,wBAAwB;AAE9C,oBAAY,KAAK;AAAA,UACf,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,cACE,MAAM;AAAA,cACN,aAAa,IAAI;AAAA,cACjB,SAAS,IAAI;AAAA,cACb,UAAU,IAAI;AAAA,YAChB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAGA,UAAM,WAAqC,CAAC;AAC5C,eAAW,OAAO,aAAa;AAC7B,YAAM,OAAO,SAAS,SAAS,SAAS,CAAC;AACzC,UAAI,QAAQ,KAAK,SAAS,IAAI,MAAM;AAElC,cAAM,cAAc,MAAM,QAAQ,KAAK,OAAO,IAC1C,KAAK,UACL,CAAC,EAAE,MAAM,QAAiB,MAAM,KAAK,QAAkB,CAAC;AAC5D,cAAM,aAAa,MAAM,QAAQ,IAAI,OAAO,IACxC,IAAI,UACJ,CAAC,EAAE,MAAM,QAAiB,MAAM,IAAI,QAAkB,CAAC;AAC3D,aAAK,UAAU,CAAC,GAAG,aAAa,GAAG,UAAU;AAAA,MAC/C,OAAO;AACL,iBAAS,KAAK,EAAE,GAAG,IAAI,CAAC;AAAA,MAC1B;AAAA,IACF;AAGA,QAAI,SAAS,WAAW,KAAK,SAAS,CAAC,EAAG,SAAS,QAAQ;AACzD,eAAS,QAAQ,EAAE,MAAM,QAAQ,SAAS,UAAU,CAAC;AAAA,IACvD;AAGA,QAAI,SAAS,SAAS,KAAK,SAAS,SAAS,SAAS,CAAC,EAAG,SAAS,aAAa;AAC9E,eAAS,KAAK,EAAE,MAAM,QAAQ,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,IACxE;AAEA,WAAO,EAAE,QAAQ,SAAS;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,KAAK;AAAA,IACH;AAAA,IACA,SAAS;AAAA,IACT,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAOc;AACZ,UAAM,SAAkC,EAAE,GAAG,YAAY;AAEzD,QAAI,KAAK,MAAM,gBAAgB,OAAW,QAAO,cAAc,KAAK,MAAM;AAE1E,UAAM,EAAE,QAAQ,SAAS,IAAI,KAAK,uBAAuB,OAAO;AAGhE,UAAM,UAAU,kBAAI,cAAc,YAAY;AAC9C,UAAM,iBAAmC,CAAC;AAC1C,QAAI,SAAS;AACX,iBAAW,CAAC,MAAM,IAAI,KAAK,kBAAI,kBAAkB,OAAO,GAAG;AACzD,uBAAe,KAAK;AAAA,UAClB;AAAA,UACA,aAAa,KAAK,eAAe;AAAA,UACjC,cAAe,KAAK,aAChB,kBAAI,aAAa,KAAK,YAAY,KAAK,IACvC,EAAE,MAAM,UAAU,YAAY,CAAC,EAAE;AAAA,QACvC,CAAC;AAAA,MACH;AAAA,IACF;AAGA,UAAM,qBAAqB,cAAc,KAAK,MAAM;AACpD,UAAM,mBAAmB,qBAAqB,KAAK,MAAM;AAEzD,SAAK,sBAAsB,qBAAqB,WAAc,eAAe,SAAS,GAAG;AACvF,UAAI,sBAA2D,EAAE,MAAM,OAAO;AAE9E,UAAI,OAAO,uBAAuB,UAAU;AAC1C,YAAI,uBAAuB,YAAY;AACrC,gCAAsB,EAAE,MAAM,MAAM;AAAA,QACtC,WAAW,uBAAuB,QAAQ;AAExC,yBAAe,SAAS;AACxB,gCAAsB;AAAA,QACxB;AAAA,MACF,WACE,OAAO,uBAAuB,YAC9B,UAAU,sBACV,mBAAmB,SAAS,YAC5B;AACA,cAAM,KAAM,mBAAsD;AAClE,8BAAsB,EAAE,MAAM,QAAQ,MAAM,GAAG,KAAK;AAAA,MACtD;AAEA,UAAI,qBAAqB;AAEvB,YAAI,qBAAqB,QAAW;AAClC,8BAAoB,4BAA4B,CAAC;AAAA,QACnD;AACA,eAAO,cAAc;AAAA,MACvB;AAAA,IACF;AAEA,UAAM,gBAAwD;AAAA,MAC5D,OAAO,KAAK,MAAM;AAAA,MAClB;AAAA,MACA,QAAQ,OAAO,SAAS,IAAI,SAAS;AAAA,MACrC,OAAO,eAAe,SAAS,IAAI,iBAAiB;AAAA,MACpD,QAAQ;AAAA,MACR,YAAY,KAAK,MAAM,aAAa;AAAA,MACpC,GAAG;AAAA,IACL;AAEA,WAAO,IAAI,UAAU,MAAM,KAAK,SAAS,eAAe,SAAS,SAAS,WAAW;AAAA,EACvF;AACF;AAWO,MAAM,kBAAkB,kBAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa;AAAA,EACb,eAAe;AAAA,EACf,eAAe;AAAA,EACf,gBAAgB;AAAA,EAChB,uBAAuB;AAAA,EACvB,mBAAmB;AAAA,EACnB;AAAA,EAEA,YACE,SACA,QACA,eACA,SACA,SACA,aACA;AACA,UAAM,SAAS,EAAE,SAAS,SAAS,YAAY,CAAC;AAChD,SAAK,UAAU;AACf,SAAK,iBAAiB;AACtB,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAgB,MAAqB;AACnC,QAAI,YAAY;AAChB,SAAK,cAAc;AACnB,SAAK,WAAW;AAChB,SAAK,cAAc;AACnB,SAAK,aAAa;AAClB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AACrB,SAAK,uBAAuB;AAC5B,SAAK,mBAAmB;AAExB,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,SAAS,OAAO,KAAK,gBAAgB;AAAA,QACrE,SAAS,KAAK,YAAY;AAAA,MAC5B,CAAC;AACD,uBAAiB,SAAS,QAAQ;AAChC,YAAI,MAAM,SAAS,iBAAiB;AAClC,eAAK,aAAa,MAAM,QAAQ;AAChC,eAAK,eAAe,MAAM,QAAQ,MAAM;AACxC,eAAK,gBAAgB,MAAM,QAAQ,MAAM;AACzC,eAAK,uBAAuB,MAAM,QAAQ,MAAM,+BAA+B;AAC/E,eAAK,mBAAmB,MAAM,QAAQ,MAAM,2BAA2B;AAAA,QACzE,WAAW,MAAM,SAAS,iBAAiB;AACzC,eAAK,gBAAgB,MAAM,MAAM;AAAA,QACnC,WACE,MAAM,SAAS,yBACf,MAAM,cAAc,SAAS,YAC7B;AACA,eAAK,cAAc,MAAM,cAAc;AACvC,eAAK,WAAW,MAAM,cAAc;AACpC,eAAK,cAAc;AAAA,QACrB,WAAW,MAAM,SAAS,yBAAyB,MAAM,MAAM,SAAS,cAAc;AACpF,cAAI,OAAO,MAAM,MAAM;AAGvB,cAAI,KAAK,UAAU;AACjB,kBAAM,gBAAgB,KAAK,QAAQ,YAAY;AAC/C,gBAAI,iBAAiB,GAAG;AACtB,oBAAM,cAAc,KAAK,MAAM,GAAG,aAAa;AAC/C,kBAAI,aAAa;AACf,qBAAK,MAAM,IAAI;AAAA,kBACb,IAAI,KAAK;AAAA,kBACT,OAAO,EAAE,MAAM,aAAa,SAAS,YAAY;AAAA,gBACnD,CAAC;AACD,4BAAY;AAAA,cACd;AACA,qBAAO,KAAK,MAAM,gBAAgB,aAAa,MAAM;AACrD,mBAAK,eAAe;AAAA,YACtB;AACA,gBAAI,KAAK,cAAc;AACrB,oBAAM,cAAc,KAAK,QAAQ,aAAa;AAC9C,kBAAI,eAAe,GAAG;AACpB,uBAAO,KAAK,MAAM,cAAc,cAAc,MAAM;AACpD,qBAAK,eAAe;AAAA,cACtB;AAAA,YACF;AAAA,UACF;AAEA,cAAI,KAAK,cAAc;AACrB;AAAA,UACF;AAIA,cAAI,MAAM;AACR,iBAAK,MAAM,IAAI;AAAA,cACb,IAAI,KAAK;AAAA,cACT,OAAO,EAAE,MAAM,aAAa,SAAS,KAAK;AAAA,YAC5C,CAAC;AACD,wBAAY;AAAA,UACd;AAAA,QACF,WACE,MAAM,SAAS,yBACf,MAAM,MAAM,SAAS,oBACrB;AACA,eAAK,eAAe,MAAM,MAAM;AAAA,QAClC,WAAW,MAAM,SAAS,wBAAwB,KAAK,aAAa;AAClE,eAAK,MAAM,IAAI;AAAA,YACb,IAAI,KAAK;AAAA,YACT,OAAO;AAAA,cACL,MAAM;AAAA,cACN,WAAW;AAAA,gBACT,kBAAI,aAAa,OAAO;AAAA,kBACtB,QAAQ,KAAK;AAAA,kBACb,MAAM,KAAK,YAAY;AAAA,kBACvB,MAAM,KAAK,eAAe;AAAA,gBAC5B,CAAC;AAAA,cACH;AAAA,YACF;AAAA,UACF,CAAC;AACD,eAAK,cAAc;AACnB,eAAK,WAAW;AAChB,eAAK,cAAc;AACnB,sBAAY;AAAA,QACd;AAAA,MACF;AAIA,YAAM,eAAe,KAAK,eAAe,KAAK,uBAAuB,KAAK;AAC1E,WAAK,MAAM,IAAI;AAAA,QACb,IAAI,KAAK;AAAA,QACT,OAAO;AAAA,UACL,kBAAkB,KAAK;AAAA,UACvB;AAAA,UACA,aAAa,eAAe,KAAK;AAAA,UACjC,oBAAoB,KAAK;AAAA,QAC3B;AAAA,MACF,CAAC;AAAA,IACH,SAAS,GAAY;AACnB,UAAI,aAAa,WAAAA,QAAU,UAAU;AACnC,YAAI,EAAE,WAAW,KAAK;AACpB,gBAAM,IAAI,8BAAgB;AAAA,YACxB,SAAS,EAAE;AAAA,YACX,SAAS,EAAE,UAAU;AAAA,UACvB,CAAC;AAAA,QACH;AACA,cAAM,IAAI,6BAAe;AAAA,UACvB,SAAS,EAAE;AAAA,UACX,SAAS;AAAA,YACP,YAAY,EAAE;AAAA,YACd,MAAM,EAAE;AAAA,YACR,WAAW,cAAc,EAAE,WAAW,OAAO,EAAE,UAAU;AAAA,UAC3D;AAAA,QACF,CAAC;AAAA,MACH;AACA,YAAM,IAAI,iCAAmB;AAAA,QAC3B,SAAS,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;AAAA,QAClD,SAAS,EAAE,UAAU;AAAA,MACvB,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":["Anthropic"]}
|
package/dist/llm.d.cts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
import type { APIConnectOptions } from '@livekit/agents';
|
|
3
|
+
import { llm } from '@livekit/agents';
|
|
4
|
+
import type { ChatModels } from './models.js';
|
|
5
|
+
/** Configuration options for the Anthropic LLM plugin. */
|
|
6
|
+
export interface LLMOptions {
|
|
7
|
+
/** The model identifier to use. */
|
|
8
|
+
model: string | ChatModels;
|
|
9
|
+
/** Anthropic API key. Falls back to the `ANTHROPIC_API_KEY` environment variable. */
|
|
10
|
+
apiKey?: string;
|
|
11
|
+
/** Custom base URL for the Anthropic API. */
|
|
12
|
+
baseURL?: string;
|
|
13
|
+
/** Sampling temperature. */
|
|
14
|
+
temperature?: number;
|
|
15
|
+
/** Pre-configured Anthropic client instance. */
|
|
16
|
+
client?: Anthropic;
|
|
17
|
+
/** Tool selection strategy. */
|
|
18
|
+
toolChoice?: llm.ToolChoice;
|
|
19
|
+
/** Whether to allow parallel tool calls. */
|
|
20
|
+
parallelToolCalls?: boolean;
|
|
21
|
+
/** Maximum number of tokens in the response. Defaults to 4096. */
|
|
22
|
+
maxTokens?: number;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Anthropic LLM provider for LiveKit Agents.
|
|
26
|
+
*
|
|
27
|
+
* @remarks
|
|
28
|
+
* Implements the {@link llm.LLM} interface using the Anthropic Messages API.
|
|
29
|
+
* Supports streaming, tool calling, and system prompt isolation required by
|
|
30
|
+
* Claude 3.5+ models.
|
|
31
|
+
*/
|
|
32
|
+
export declare class LLM extends llm.LLM {
|
|
33
|
+
#private;
|
|
34
|
+
constructor(opts?: Partial<LLMOptions>);
|
|
35
|
+
/** @returns Human-readable label for logging. */
|
|
36
|
+
label(): string;
|
|
37
|
+
/** @returns The model identifier being used. */
|
|
38
|
+
get model(): string;
|
|
39
|
+
/** @returns The API provider host. */
|
|
40
|
+
get provider(): string;
|
|
41
|
+
/**
|
|
42
|
+
* Converts a framework ChatContext into Anthropic's message format.
|
|
43
|
+
*
|
|
44
|
+
* @remarks
|
|
45
|
+
* - System prompts are isolated into a separate `TextBlockParam[]` array
|
|
46
|
+
* (required by Claude 3.5+).
|
|
47
|
+
* - `function_call` items are mapped to Anthropic `tool_use` content blocks.
|
|
48
|
+
* - `function_call_output` items are mapped to `tool_result` content blocks.
|
|
49
|
+
* - Consecutive same-role messages are merged to satisfy Anthropic's
|
|
50
|
+
* strict alternating-turn requirement.
|
|
51
|
+
* - A dummy `(empty)` user message is injected if the conversation doesn't
|
|
52
|
+
* start with a user turn.
|
|
53
|
+
* - A dummy user message is appended if the conversation ends with an
|
|
54
|
+
* assistant turn, since Claude 4.6+ does not support prefilling.
|
|
55
|
+
*/
|
|
56
|
+
protected _buildAnthropicContext(chatCtx: llm.ChatContext): {
|
|
57
|
+
system: Anthropic.TextBlockParam[];
|
|
58
|
+
messages: Anthropic.MessageParam[];
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Creates a streaming chat completion.
|
|
62
|
+
*
|
|
63
|
+
* @remarks
|
|
64
|
+
* Maps `toolChoice` to Anthropic's format:
|
|
65
|
+
* - `"required"` → `{ type: "any" }`
|
|
66
|
+
* - `"none"` → clears tools
|
|
67
|
+
* - `{ type: "function", function: { name } }` → `{ type: "tool", name }`
|
|
68
|
+
*/
|
|
69
|
+
chat({ chatCtx, toolCtx: toolCtxInput, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
|
|
70
|
+
chatCtx: llm.ChatContext;
|
|
71
|
+
toolCtx?: llm.ToolContextLike;
|
|
72
|
+
connOptions?: APIConnectOptions;
|
|
73
|
+
parallelToolCalls?: boolean;
|
|
74
|
+
toolChoice?: llm.ToolChoice;
|
|
75
|
+
extraKwargs?: Record<string, unknown>;
|
|
76
|
+
}): LLMStream;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Streaming response handler for Anthropic Messages API.
|
|
80
|
+
*
|
|
81
|
+
* @remarks
|
|
82
|
+
* Parses SSE events including:
|
|
83
|
+
* - `content_block_start` / `content_block_delta` / `content_block_stop` for text and tool use
|
|
84
|
+
* - `message_start` / `message_delta` for token usage tracking
|
|
85
|
+
* - Chain-of-thought `<thinking>` block filtering when tools are active
|
|
86
|
+
*/
|
|
87
|
+
export declare class LLMStream extends llm.LLMStream {
|
|
88
|
+
#private;
|
|
89
|
+
constructor(llmInst: LLM, client: Anthropic, requestParams: Anthropic.MessageCreateParamsStreaming, chatCtx: llm.ChatContext, toolCtx: llm.ToolContext | undefined, connOptions: APIConnectOptions);
|
|
90
|
+
protected run(): Promise<void>;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=llm.d.ts.map
|
package/dist/llm.d.ts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
import type { APIConnectOptions } from '@livekit/agents';
|
|
3
|
+
import { llm } from '@livekit/agents';
|
|
4
|
+
import type { ChatModels } from './models.js';
|
|
5
|
+
/** Configuration options for the Anthropic LLM plugin. */
|
|
6
|
+
export interface LLMOptions {
|
|
7
|
+
/** The model identifier to use. */
|
|
8
|
+
model: string | ChatModels;
|
|
9
|
+
/** Anthropic API key. Falls back to the `ANTHROPIC_API_KEY` environment variable. */
|
|
10
|
+
apiKey?: string;
|
|
11
|
+
/** Custom base URL for the Anthropic API. */
|
|
12
|
+
baseURL?: string;
|
|
13
|
+
/** Sampling temperature. */
|
|
14
|
+
temperature?: number;
|
|
15
|
+
/** Pre-configured Anthropic client instance. */
|
|
16
|
+
client?: Anthropic;
|
|
17
|
+
/** Tool selection strategy. */
|
|
18
|
+
toolChoice?: llm.ToolChoice;
|
|
19
|
+
/** Whether to allow parallel tool calls. */
|
|
20
|
+
parallelToolCalls?: boolean;
|
|
21
|
+
/** Maximum number of tokens in the response. Defaults to 4096. */
|
|
22
|
+
maxTokens?: number;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Anthropic LLM provider for LiveKit Agents.
|
|
26
|
+
*
|
|
27
|
+
* @remarks
|
|
28
|
+
* Implements the {@link llm.LLM} interface using the Anthropic Messages API.
|
|
29
|
+
* Supports streaming, tool calling, and system prompt isolation required by
|
|
30
|
+
* Claude 3.5+ models.
|
|
31
|
+
*/
|
|
32
|
+
export declare class LLM extends llm.LLM {
|
|
33
|
+
#private;
|
|
34
|
+
constructor(opts?: Partial<LLMOptions>);
|
|
35
|
+
/** @returns Human-readable label for logging. */
|
|
36
|
+
label(): string;
|
|
37
|
+
/** @returns The model identifier being used. */
|
|
38
|
+
get model(): string;
|
|
39
|
+
/** @returns The API provider host. */
|
|
40
|
+
get provider(): string;
|
|
41
|
+
/**
|
|
42
|
+
* Converts a framework ChatContext into Anthropic's message format.
|
|
43
|
+
*
|
|
44
|
+
* @remarks
|
|
45
|
+
* - System prompts are isolated into a separate `TextBlockParam[]` array
|
|
46
|
+
* (required by Claude 3.5+).
|
|
47
|
+
* - `function_call` items are mapped to Anthropic `tool_use` content blocks.
|
|
48
|
+
* - `function_call_output` items are mapped to `tool_result` content blocks.
|
|
49
|
+
* - Consecutive same-role messages are merged to satisfy Anthropic's
|
|
50
|
+
* strict alternating-turn requirement.
|
|
51
|
+
* - A dummy `(empty)` user message is injected if the conversation doesn't
|
|
52
|
+
* start with a user turn.
|
|
53
|
+
* - A dummy user message is appended if the conversation ends with an
|
|
54
|
+
* assistant turn, since Claude 4.6+ does not support prefilling.
|
|
55
|
+
*/
|
|
56
|
+
protected _buildAnthropicContext(chatCtx: llm.ChatContext): {
|
|
57
|
+
system: Anthropic.TextBlockParam[];
|
|
58
|
+
messages: Anthropic.MessageParam[];
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Creates a streaming chat completion.
|
|
62
|
+
*
|
|
63
|
+
* @remarks
|
|
64
|
+
* Maps `toolChoice` to Anthropic's format:
|
|
65
|
+
* - `"required"` → `{ type: "any" }`
|
|
66
|
+
* - `"none"` → clears tools
|
|
67
|
+
* - `{ type: "function", function: { name } }` → `{ type: "tool", name }`
|
|
68
|
+
*/
|
|
69
|
+
chat({ chatCtx, toolCtx: toolCtxInput, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
|
|
70
|
+
chatCtx: llm.ChatContext;
|
|
71
|
+
toolCtx?: llm.ToolContextLike;
|
|
72
|
+
connOptions?: APIConnectOptions;
|
|
73
|
+
parallelToolCalls?: boolean;
|
|
74
|
+
toolChoice?: llm.ToolChoice;
|
|
75
|
+
extraKwargs?: Record<string, unknown>;
|
|
76
|
+
}): LLMStream;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Streaming response handler for Anthropic Messages API.
|
|
80
|
+
*
|
|
81
|
+
* @remarks
|
|
82
|
+
* Parses SSE events including:
|
|
83
|
+
* - `content_block_start` / `content_block_delta` / `content_block_stop` for text and tool use
|
|
84
|
+
* - `message_start` / `message_delta` for token usage tracking
|
|
85
|
+
* - Chain-of-thought `<thinking>` block filtering when tools are active
|
|
86
|
+
*/
|
|
87
|
+
export declare class LLMStream extends llm.LLMStream {
|
|
88
|
+
#private;
|
|
89
|
+
constructor(llmInst: LLM, client: Anthropic, requestParams: Anthropic.MessageCreateParamsStreaming, chatCtx: llm.ChatContext, toolCtx: llm.ToolContext | undefined, connOptions: APIConnectOptions);
|
|
90
|
+
protected run(): Promise<void>;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=llm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAGA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAKL,GAAG,EACJ,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,WAAW,UAAU;IACzB,mCAAmC;IACnC,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;IAC3B,qFAAqF;IACrF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gDAAgD;IAChD,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;IAC5B,4CAA4C;IAC5C,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AASD;;;;;;;GAOG;AACH,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;gBAIlB,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAkBzD,iDAAiD;IACjD,KAAK,IAAI,MAAM;IAIf,gDAAgD;IAChD,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,sCAAsC;IACtC,IAAI,QAAQ,IAAI,MAAM,CAOrB;IAED;;;;;;;;;;;;;;OAcG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG;QAC1D,MAAM,EAAE,SAAS,CAAC,cAAc,EAAE,CAAC;QACnC,QAAQ,EAAE,SAAS,CAAC,YAAY,EAAE,CAAC;KACpC;IA2ED;;;;;;;;OAQG;IACH,IAAI,CAAC,EACH,OAAO,EACP,OAAO,EAAE,YAAY,EACrB,WAAyC,EACzC,iBAAiB,EACjB,UAAU,EACV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC;QAC9B,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;QAC5B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACvC,GAAG,SAAS;CAmEd;AAED;;;;;;;;GAQG;AACH,qBAAa,SAAU,SAAQ,GAAG,CAAC,SAAS;;gBAexC,OAAO,EAAE,GAAG,EACZ,MAAM,EAAE,SAAS,EACjB,aAAa,EAAE,SAAS,CAAC,4BAA4B,EACrD,OAAO,EAAE,GAAG,CAAC,WAAW,EACxB,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,SAAS,EACpC,WAAW,EAAE,iBAAiB;cAQhB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAqIrC"}
|
package/dist/llm.js
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
2
|
+
import {
|
|
3
|
+
APIConnectionError,
|
|
4
|
+
APIStatusError,
|
|
5
|
+
APITimeoutError,
|
|
6
|
+
DEFAULT_API_CONNECT_OPTIONS,
|
|
7
|
+
llm
|
|
8
|
+
} from "@livekit/agents";
|
|
9
|
+
const defaultLLMOptions = {
|
|
10
|
+
model: "claude-sonnet-4-6",
|
|
11
|
+
/* eslint-disable-next-line turbo/no-undeclared-env-vars */
|
|
12
|
+
apiKey: process.env.ANTHROPIC_API_KEY,
|
|
13
|
+
parallelToolCalls: true
|
|
14
|
+
};
|
|
15
|
+
class LLM extends llm.LLM {
|
|
16
|
+
#opts;
|
|
17
|
+
#client;
|
|
18
|
+
constructor(opts = defaultLLMOptions) {
|
|
19
|
+
super();
|
|
20
|
+
this.#opts = { ...defaultLLMOptions, ...opts };
|
|
21
|
+
if (!this.#opts.apiKey && !this.#opts.client) {
|
|
22
|
+
throw new Error(
|
|
23
|
+
"Anthropic API key is required, whether as an argument or as $ANTHROPIC_API_KEY"
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
this.#client = this.#opts.client || new Anthropic({
|
|
27
|
+
baseURL: this.#opts.baseURL,
|
|
28
|
+
apiKey: this.#opts.apiKey
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
/** @returns Human-readable label for logging. */
|
|
32
|
+
label() {
|
|
33
|
+
return "anthropic.LLM";
|
|
34
|
+
}
|
|
35
|
+
/** @returns The model identifier being used. */
|
|
36
|
+
get model() {
|
|
37
|
+
return this.#opts.model;
|
|
38
|
+
}
|
|
39
|
+
/** @returns The API provider host. */
|
|
40
|
+
get provider() {
|
|
41
|
+
try {
|
|
42
|
+
const url = new URL(this.#client.baseURL);
|
|
43
|
+
return url.host;
|
|
44
|
+
} catch {
|
|
45
|
+
return "api.anthropic.com";
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Converts a framework ChatContext into Anthropic's message format.
|
|
50
|
+
*
|
|
51
|
+
* @remarks
|
|
52
|
+
* - System prompts are isolated into a separate `TextBlockParam[]` array
|
|
53
|
+
* (required by Claude 3.5+).
|
|
54
|
+
* - `function_call` items are mapped to Anthropic `tool_use` content blocks.
|
|
55
|
+
* - `function_call_output` items are mapped to `tool_result` content blocks.
|
|
56
|
+
* - Consecutive same-role messages are merged to satisfy Anthropic's
|
|
57
|
+
* strict alternating-turn requirement.
|
|
58
|
+
* - A dummy `(empty)` user message is injected if the conversation doesn't
|
|
59
|
+
* start with a user turn.
|
|
60
|
+
* - A dummy user message is appended if the conversation ends with an
|
|
61
|
+
* assistant turn, since Claude 4.6+ does not support prefilling.
|
|
62
|
+
*/
|
|
63
|
+
_buildAnthropicContext(chatCtx) {
|
|
64
|
+
const system = [];
|
|
65
|
+
const rawMessages = [];
|
|
66
|
+
for (const msg of chatCtx.items) {
|
|
67
|
+
if (msg.type === "message") {
|
|
68
|
+
const textContent = msg.textContent || "";
|
|
69
|
+
if (msg.role === "system" || msg.role === "developer") {
|
|
70
|
+
system.push({ type: "text", text: textContent });
|
|
71
|
+
} else if (msg.role === "user" || msg.role === "assistant") {
|
|
72
|
+
rawMessages.push({
|
|
73
|
+
role: msg.role,
|
|
74
|
+
content: textContent
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
} else if (msg.type === "function_call") {
|
|
78
|
+
rawMessages.push({
|
|
79
|
+
role: "assistant",
|
|
80
|
+
content: [
|
|
81
|
+
{
|
|
82
|
+
type: "tool_use",
|
|
83
|
+
id: msg.callId,
|
|
84
|
+
name: msg.name,
|
|
85
|
+
input: JSON.parse(msg.args || "{}")
|
|
86
|
+
}
|
|
87
|
+
]
|
|
88
|
+
});
|
|
89
|
+
} else if (msg.type === "function_call_output") {
|
|
90
|
+
rawMessages.push({
|
|
91
|
+
role: "user",
|
|
92
|
+
content: [
|
|
93
|
+
{
|
|
94
|
+
type: "tool_result",
|
|
95
|
+
tool_use_id: msg.callId,
|
|
96
|
+
content: msg.output,
|
|
97
|
+
is_error: msg.isError
|
|
98
|
+
}
|
|
99
|
+
]
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
const messages = [];
|
|
104
|
+
for (const msg of rawMessages) {
|
|
105
|
+
const prev = messages[messages.length - 1];
|
|
106
|
+
if (prev && prev.role === msg.role) {
|
|
107
|
+
const prevContent = Array.isArray(prev.content) ? prev.content : [{ type: "text", text: prev.content }];
|
|
108
|
+
const curContent = Array.isArray(msg.content) ? msg.content : [{ type: "text", text: msg.content }];
|
|
109
|
+
prev.content = [...prevContent, ...curContent];
|
|
110
|
+
} else {
|
|
111
|
+
messages.push({ ...msg });
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (messages.length === 0 || messages[0].role !== "user") {
|
|
115
|
+
messages.unshift({ role: "user", content: "(empty)" });
|
|
116
|
+
}
|
|
117
|
+
if (messages.length > 0 && messages[messages.length - 1].role === "assistant") {
|
|
118
|
+
messages.push({ role: "user", content: [{ type: "text", text: "." }] });
|
|
119
|
+
}
|
|
120
|
+
return { system, messages };
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Creates a streaming chat completion.
|
|
124
|
+
*
|
|
125
|
+
* @remarks
|
|
126
|
+
* Maps `toolChoice` to Anthropic's format:
|
|
127
|
+
* - `"required"` → `{ type: "any" }`
|
|
128
|
+
* - `"none"` → clears tools
|
|
129
|
+
* - `{ type: "function", function: { name } }` → `{ type: "tool", name }`
|
|
130
|
+
*/
|
|
131
|
+
chat({
|
|
132
|
+
chatCtx,
|
|
133
|
+
toolCtx: toolCtxInput,
|
|
134
|
+
connOptions = DEFAULT_API_CONNECT_OPTIONS,
|
|
135
|
+
parallelToolCalls,
|
|
136
|
+
toolChoice,
|
|
137
|
+
extraKwargs
|
|
138
|
+
}) {
|
|
139
|
+
const extras = { ...extraKwargs };
|
|
140
|
+
if (this.#opts.temperature !== void 0) extras.temperature = this.#opts.temperature;
|
|
141
|
+
const { system, messages } = this._buildAnthropicContext(chatCtx);
|
|
142
|
+
const toolCtx = llm.toToolContext(toolCtxInput);
|
|
143
|
+
const anthropicTools = [];
|
|
144
|
+
if (toolCtx) {
|
|
145
|
+
for (const [name, tool] of llm.sortedToolEntries(toolCtx)) {
|
|
146
|
+
anthropicTools.push({
|
|
147
|
+
name,
|
|
148
|
+
description: tool.description || "",
|
|
149
|
+
input_schema: tool.parameters ? llm.toJsonSchema(tool.parameters, false) : { type: "object", properties: {} }
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
const resolvedToolChoice = toolChoice ?? this.#opts.toolChoice;
|
|
154
|
+
const resolvedParallel = parallelToolCalls ?? this.#opts.parallelToolCalls;
|
|
155
|
+
if ((resolvedToolChoice || resolvedParallel !== void 0) && anthropicTools.length > 0) {
|
|
156
|
+
let anthropicToolChoice = { type: "auto" };
|
|
157
|
+
if (typeof resolvedToolChoice === "string") {
|
|
158
|
+
if (resolvedToolChoice === "required") {
|
|
159
|
+
anthropicToolChoice = { type: "any" };
|
|
160
|
+
} else if (resolvedToolChoice === "none") {
|
|
161
|
+
anthropicTools.length = 0;
|
|
162
|
+
anthropicToolChoice = void 0;
|
|
163
|
+
}
|
|
164
|
+
} else if (typeof resolvedToolChoice === "object" && "type" in resolvedToolChoice && resolvedToolChoice.type === "function") {
|
|
165
|
+
const fn = resolvedToolChoice.function;
|
|
166
|
+
anthropicToolChoice = { type: "tool", name: fn.name };
|
|
167
|
+
}
|
|
168
|
+
if (anthropicToolChoice) {
|
|
169
|
+
if (resolvedParallel !== void 0) {
|
|
170
|
+
anthropicToolChoice.disable_parallel_tool_use = !resolvedParallel;
|
|
171
|
+
}
|
|
172
|
+
extras.tool_choice = anthropicToolChoice;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const requestParams = {
|
|
176
|
+
model: this.#opts.model,
|
|
177
|
+
messages,
|
|
178
|
+
system: system.length > 0 ? system : void 0,
|
|
179
|
+
tools: anthropicTools.length > 0 ? anthropicTools : void 0,
|
|
180
|
+
stream: true,
|
|
181
|
+
max_tokens: this.#opts.maxTokens || 4096,
|
|
182
|
+
...extras
|
|
183
|
+
};
|
|
184
|
+
return new LLMStream(this, this.#client, requestParams, chatCtx, toolCtx, connOptions);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
class LLMStream extends llm.LLMStream {
|
|
188
|
+
#client;
|
|
189
|
+
#requestParams;
|
|
190
|
+
#toolCallId;
|
|
191
|
+
#fncName;
|
|
192
|
+
#fncRawArgs;
|
|
193
|
+
#requestId = "";
|
|
194
|
+
#ignoringCoT = false;
|
|
195
|
+
#inputTokens = 0;
|
|
196
|
+
#outputTokens = 0;
|
|
197
|
+
#cacheCreationTokens = 0;
|
|
198
|
+
#cacheReadTokens = 0;
|
|
199
|
+
#toolCtx;
|
|
200
|
+
constructor(llmInst, client, requestParams, chatCtx, toolCtx, connOptions) {
|
|
201
|
+
super(llmInst, { chatCtx, toolCtx, connOptions });
|
|
202
|
+
this.#client = client;
|
|
203
|
+
this.#requestParams = requestParams;
|
|
204
|
+
this.#toolCtx = toolCtx;
|
|
205
|
+
}
|
|
206
|
+
async run() {
|
|
207
|
+
let retryable = true;
|
|
208
|
+
this.#toolCallId = void 0;
|
|
209
|
+
this.#fncName = void 0;
|
|
210
|
+
this.#fncRawArgs = void 0;
|
|
211
|
+
this.#requestId = "";
|
|
212
|
+
this.#ignoringCoT = false;
|
|
213
|
+
this.#inputTokens = 0;
|
|
214
|
+
this.#outputTokens = 0;
|
|
215
|
+
this.#cacheCreationTokens = 0;
|
|
216
|
+
this.#cacheReadTokens = 0;
|
|
217
|
+
try {
|
|
218
|
+
const stream = await this.#client.messages.create(this.#requestParams, {
|
|
219
|
+
timeout: this.connOptions.timeoutMs
|
|
220
|
+
});
|
|
221
|
+
for await (const event of stream) {
|
|
222
|
+
if (event.type === "message_start") {
|
|
223
|
+
this.#requestId = event.message.id;
|
|
224
|
+
this.#inputTokens = event.message.usage.input_tokens;
|
|
225
|
+
this.#outputTokens = event.message.usage.output_tokens;
|
|
226
|
+
this.#cacheCreationTokens = event.message.usage.cache_creation_input_tokens ?? 0;
|
|
227
|
+
this.#cacheReadTokens = event.message.usage.cache_read_input_tokens ?? 0;
|
|
228
|
+
} else if (event.type === "message_delta") {
|
|
229
|
+
this.#outputTokens = event.usage.output_tokens;
|
|
230
|
+
} else if (event.type === "content_block_start" && event.content_block.type === "tool_use") {
|
|
231
|
+
this.#toolCallId = event.content_block.id;
|
|
232
|
+
this.#fncName = event.content_block.name;
|
|
233
|
+
this.#fncRawArgs = "";
|
|
234
|
+
} else if (event.type === "content_block_delta" && event.delta.type === "text_delta") {
|
|
235
|
+
let text = event.delta.text;
|
|
236
|
+
if (this.#toolCtx) {
|
|
237
|
+
const thinkingStart = text.indexOf("<thinking>");
|
|
238
|
+
if (thinkingStart >= 0) {
|
|
239
|
+
const preThinking = text.slice(0, thinkingStart);
|
|
240
|
+
if (preThinking) {
|
|
241
|
+
this.queue.put({
|
|
242
|
+
id: this.#requestId,
|
|
243
|
+
delta: { role: "assistant", content: preThinking }
|
|
244
|
+
});
|
|
245
|
+
retryable = false;
|
|
246
|
+
}
|
|
247
|
+
text = text.slice(thinkingStart + "<thinking>".length);
|
|
248
|
+
this.#ignoringCoT = true;
|
|
249
|
+
}
|
|
250
|
+
if (this.#ignoringCoT) {
|
|
251
|
+
const thinkingEnd = text.indexOf("</thinking>");
|
|
252
|
+
if (thinkingEnd >= 0) {
|
|
253
|
+
text = text.slice(thinkingEnd + "</thinking>".length);
|
|
254
|
+
this.#ignoringCoT = false;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
if (this.#ignoringCoT) {
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
if (text) {
|
|
262
|
+
this.queue.put({
|
|
263
|
+
id: this.#requestId,
|
|
264
|
+
delta: { role: "assistant", content: text }
|
|
265
|
+
});
|
|
266
|
+
retryable = false;
|
|
267
|
+
}
|
|
268
|
+
} else if (event.type === "content_block_delta" && event.delta.type === "input_json_delta") {
|
|
269
|
+
this.#fncRawArgs += event.delta.partial_json;
|
|
270
|
+
} else if (event.type === "content_block_stop" && this.#toolCallId) {
|
|
271
|
+
this.queue.put({
|
|
272
|
+
id: this.#requestId,
|
|
273
|
+
delta: {
|
|
274
|
+
role: "assistant",
|
|
275
|
+
toolCalls: [
|
|
276
|
+
llm.FunctionCall.create({
|
|
277
|
+
callId: this.#toolCallId,
|
|
278
|
+
name: this.#fncName || "",
|
|
279
|
+
args: this.#fncRawArgs || ""
|
|
280
|
+
})
|
|
281
|
+
]
|
|
282
|
+
}
|
|
283
|
+
});
|
|
284
|
+
this.#toolCallId = void 0;
|
|
285
|
+
this.#fncName = void 0;
|
|
286
|
+
this.#fncRawArgs = void 0;
|
|
287
|
+
retryable = false;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
const promptTokens = this.#inputTokens + this.#cacheCreationTokens + this.#cacheReadTokens;
|
|
291
|
+
this.queue.put({
|
|
292
|
+
id: this.#requestId,
|
|
293
|
+
usage: {
|
|
294
|
+
completionTokens: this.#outputTokens,
|
|
295
|
+
promptTokens,
|
|
296
|
+
totalTokens: promptTokens + this.#outputTokens,
|
|
297
|
+
promptCachedTokens: this.#cacheReadTokens
|
|
298
|
+
}
|
|
299
|
+
});
|
|
300
|
+
} catch (e) {
|
|
301
|
+
if (e instanceof Anthropic.APIError) {
|
|
302
|
+
if (e.status === 408) {
|
|
303
|
+
throw new APITimeoutError({
|
|
304
|
+
message: e.message,
|
|
305
|
+
options: { retryable }
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
throw new APIStatusError({
|
|
309
|
+
message: e.message,
|
|
310
|
+
options: {
|
|
311
|
+
statusCode: e.status,
|
|
312
|
+
body: e.error,
|
|
313
|
+
retryable: retryable && (e.status === 429 || e.status >= 500)
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
throw new APIConnectionError({
|
|
318
|
+
message: e instanceof Error ? e.message : String(e),
|
|
319
|
+
options: { retryable }
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
export {
|
|
325
|
+
LLM,
|
|
326
|
+
LLMStream
|
|
327
|
+
};
|
|
328
|
+
//# sourceMappingURL=llm.js.map
|
package/dist/llm.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport Anthropic from '@anthropic-ai/sdk';\nimport type { APIConnectOptions } from '@livekit/agents';\nimport {\n APIConnectionError,\n APIStatusError,\n APITimeoutError,\n DEFAULT_API_CONNECT_OPTIONS,\n llm,\n} from '@livekit/agents';\nimport type { ChatModels } from './models.js';\n\n/** Configuration options for the Anthropic LLM plugin. */\nexport interface LLMOptions {\n /** The model identifier to use. */\n model: string | ChatModels;\n /** Anthropic API key. Falls back to the `ANTHROPIC_API_KEY` environment variable. */\n apiKey?: string;\n /** Custom base URL for the Anthropic API. */\n baseURL?: string;\n /** Sampling temperature. */\n temperature?: number;\n /** Pre-configured Anthropic client instance. */\n client?: Anthropic;\n /** Tool selection strategy. */\n toolChoice?: llm.ToolChoice;\n /** Whether to allow parallel tool calls. */\n parallelToolCalls?: boolean;\n /** Maximum number of tokens in the response. Defaults to 4096. */\n maxTokens?: number;\n}\n\nconst defaultLLMOptions: LLMOptions = {\n model: 'claude-sonnet-4-6',\n /* eslint-disable-next-line turbo/no-undeclared-env-vars */\n apiKey: process.env.ANTHROPIC_API_KEY,\n parallelToolCalls: true,\n};\n\n/**\n * Anthropic LLM provider for LiveKit Agents.\n *\n * @remarks\n * Implements the {@link llm.LLM} interface using the Anthropic Messages API.\n * Supports streaming, tool calling, and system prompt isolation required by\n * Claude 3.5+ models.\n */\nexport class LLM extends llm.LLM {\n #opts: LLMOptions;\n #client: Anthropic;\n\n constructor(opts: Partial<LLMOptions> = defaultLLMOptions) {\n super();\n\n this.#opts = { ...defaultLLMOptions, ...opts };\n if (!this.#opts.apiKey && !this.#opts.client) {\n throw new Error(\n 'Anthropic API key is required, whether as an argument or as $ANTHROPIC_API_KEY',\n );\n }\n\n this.#client =\n this.#opts.client ||\n new Anthropic({\n baseURL: this.#opts.baseURL,\n apiKey: this.#opts.apiKey,\n });\n }\n\n /** @returns Human-readable label for logging. */\n label(): string {\n return 'anthropic.LLM';\n }\n\n /** @returns The model identifier being used. */\n get model(): string {\n return this.#opts.model;\n }\n\n /** @returns The API provider host. */\n get provider(): string {\n try {\n const url = new URL(this.#client.baseURL);\n return url.host;\n } catch {\n return 'api.anthropic.com';\n }\n }\n\n /**\n * Converts a framework ChatContext into Anthropic's message format.\n *\n * @remarks\n * - System prompts are isolated into a separate `TextBlockParam[]` array\n * (required by Claude 3.5+).\n * - `function_call` items are mapped to Anthropic `tool_use` content blocks.\n * - `function_call_output` items are mapped to `tool_result` content blocks.\n * - Consecutive same-role messages are merged to satisfy Anthropic's\n * strict alternating-turn requirement.\n * - A dummy `(empty)` user message is injected if the conversation doesn't\n * start with a user turn.\n * - A dummy user message is appended if the conversation ends with an\n * assistant turn, since Claude 4.6+ does not support prefilling.\n */\n protected _buildAnthropicContext(chatCtx: llm.ChatContext): {\n system: Anthropic.TextBlockParam[];\n messages: Anthropic.MessageParam[];\n } {\n const system: Anthropic.TextBlockParam[] = [];\n const rawMessages: Anthropic.MessageParam[] = [];\n\n for (const msg of chatCtx.items) {\n if (msg.type === 'message') {\n const textContent = msg.textContent || '';\n if (msg.role === 'system' || msg.role === 'developer') {\n system.push({ type: 'text', text: textContent });\n } else if (msg.role === 'user' || msg.role === 'assistant') {\n rawMessages.push({\n role: msg.role,\n content: textContent,\n });\n }\n } else if (msg.type === 'function_call') {\n // Map to Anthropic's tool_use content block (assistant role)\n rawMessages.push({\n role: 'assistant',\n content: [\n {\n type: 'tool_use',\n id: msg.callId,\n name: msg.name,\n input: JSON.parse(msg.args || '{}'),\n },\n ],\n });\n } else if (msg.type === 'function_call_output') {\n // Map to Anthropic's tool_result content block (user role)\n rawMessages.push({\n role: 'user',\n content: [\n {\n type: 'tool_result',\n tool_use_id: msg.callId,\n content: msg.output,\n is_error: msg.isError,\n },\n ],\n });\n }\n }\n\n // Merge consecutive same-role messages (Anthropic requires alternating turns)\n const messages: Anthropic.MessageParam[] = [];\n for (const msg of rawMessages) {\n const prev = messages[messages.length - 1];\n if (prev && prev.role === msg.role) {\n // Merge content into a single content array\n const prevContent = Array.isArray(prev.content)\n ? prev.content\n : [{ type: 'text' as const, text: prev.content as string }];\n const curContent = Array.isArray(msg.content)\n ? msg.content\n : [{ type: 'text' as const, text: msg.content as string }];\n prev.content = [...prevContent, ...curContent] as Anthropic.ContentBlockParam[];\n } else {\n messages.push({ ...msg });\n }\n }\n\n // Anthropic requires conversations to start with a user turn\n if (messages.length === 0 || messages[0]!.role !== 'user') {\n messages.unshift({ role: 'user', content: '(empty)' });\n }\n\n // Claude 4.6+ does not support prefilling (trailing assistant messages).\n if (messages.length > 0 && messages[messages.length - 1]!.role === 'assistant') {\n messages.push({ role: 'user', content: [{ type: 'text', text: '.' }] });\n }\n\n return { system, messages };\n }\n\n /**\n * Creates a streaming chat completion.\n *\n * @remarks\n * Maps `toolChoice` to Anthropic's format:\n * - `\"required\"` → `{ type: \"any\" }`\n * - `\"none\"` → clears tools\n * - `{ type: \"function\", function: { name } }` → `{ type: \"tool\", name }`\n */\n chat({\n chatCtx,\n toolCtx: toolCtxInput,\n connOptions = DEFAULT_API_CONNECT_OPTIONS,\n parallelToolCalls,\n toolChoice,\n extraKwargs,\n }: {\n chatCtx: llm.ChatContext;\n toolCtx?: llm.ToolContextLike;\n connOptions?: APIConnectOptions;\n parallelToolCalls?: boolean;\n toolChoice?: llm.ToolChoice;\n extraKwargs?: Record<string, unknown>;\n }): LLMStream {\n const extras: Record<string, unknown> = { ...extraKwargs };\n\n if (this.#opts.temperature !== undefined) extras.temperature = this.#opts.temperature;\n\n const { system, messages } = this._buildAnthropicContext(chatCtx);\n\n // Build Anthropic tool schemas\n const toolCtx = llm.toToolContext(toolCtxInput);\n const anthropicTools: Anthropic.Tool[] = [];\n if (toolCtx) {\n for (const [name, tool] of llm.sortedToolEntries(toolCtx)) {\n anthropicTools.push({\n name: name,\n description: tool.description || '',\n input_schema: (tool.parameters\n ? llm.toJsonSchema(tool.parameters, false)\n : { type: 'object', properties: {} }) as Anthropic.Tool.InputSchema,\n });\n }\n }\n\n // Map toolChoice and parallelToolCalls to Anthropic format\n const resolvedToolChoice = toolChoice ?? this.#opts.toolChoice;\n const resolvedParallel = parallelToolCalls ?? this.#opts.parallelToolCalls;\n\n if ((resolvedToolChoice || resolvedParallel !== undefined) && anthropicTools.length > 0) {\n let anthropicToolChoice: Record<string, unknown> | undefined = { type: 'auto' };\n\n if (typeof resolvedToolChoice === 'string') {\n if (resolvedToolChoice === 'required') {\n anthropicToolChoice = { type: 'any' };\n } else if (resolvedToolChoice === 'none') {\n // Clear tools entirely when none is requested\n anthropicTools.length = 0;\n anthropicToolChoice = undefined;\n }\n } else if (\n typeof resolvedToolChoice === 'object' &&\n 'type' in resolvedToolChoice &&\n resolvedToolChoice.type === 'function'\n ) {\n const fn = (resolvedToolChoice as { function: { name: string } }).function;\n anthropicToolChoice = { type: 'tool', name: fn.name };\n }\n\n if (anthropicToolChoice) {\n // Map parallelToolCalls\n if (resolvedParallel !== undefined) {\n anthropicToolChoice.disable_parallel_tool_use = !resolvedParallel;\n }\n extras.tool_choice = anthropicToolChoice;\n }\n }\n\n const requestParams: Anthropic.MessageCreateParamsStreaming = {\n model: this.#opts.model,\n messages: messages,\n system: system.length > 0 ? system : undefined,\n tools: anthropicTools.length > 0 ? anthropicTools : undefined,\n stream: true,\n max_tokens: this.#opts.maxTokens || 4096,\n ...extras,\n };\n\n return new LLMStream(this, this.#client, requestParams, chatCtx, toolCtx, connOptions);\n }\n}\n\n/**\n * Streaming response handler for Anthropic Messages API.\n *\n * @remarks\n * Parses SSE events including:\n * - `content_block_start` / `content_block_delta` / `content_block_stop` for text and tool use\n * - `message_start` / `message_delta` for token usage tracking\n * - Chain-of-thought `<thinking>` block filtering when tools are active\n */\nexport class LLMStream extends llm.LLMStream {\n #client: Anthropic;\n #requestParams: Anthropic.MessageCreateParamsStreaming;\n #toolCallId?: string;\n #fncName?: string;\n #fncRawArgs?: string;\n #requestId = '';\n #ignoringCoT = false;\n #inputTokens = 0;\n #outputTokens = 0;\n #cacheCreationTokens = 0;\n #cacheReadTokens = 0;\n #toolCtx?: llm.ToolContext;\n\n constructor(\n llmInst: LLM,\n client: Anthropic,\n requestParams: Anthropic.MessageCreateParamsStreaming,\n chatCtx: llm.ChatContext,\n toolCtx: llm.ToolContext | undefined,\n connOptions: APIConnectOptions,\n ) {\n super(llmInst, { chatCtx, toolCtx, connOptions });\n this.#client = client;\n this.#requestParams = requestParams;\n this.#toolCtx = toolCtx;\n }\n\n protected async run(): Promise<void> {\n let retryable = true;\n this.#toolCallId = undefined;\n this.#fncName = undefined;\n this.#fncRawArgs = undefined;\n this.#requestId = '';\n this.#ignoringCoT = false;\n this.#inputTokens = 0;\n this.#outputTokens = 0;\n this.#cacheCreationTokens = 0;\n this.#cacheReadTokens = 0;\n\n try {\n const stream = await this.#client.messages.create(this.#requestParams, {\n timeout: this.connOptions.timeoutMs,\n });\n for await (const event of stream) {\n if (event.type === 'message_start') {\n this.#requestId = event.message.id;\n this.#inputTokens = event.message.usage.input_tokens;\n this.#outputTokens = event.message.usage.output_tokens;\n this.#cacheCreationTokens = event.message.usage.cache_creation_input_tokens ?? 0;\n this.#cacheReadTokens = event.message.usage.cache_read_input_tokens ?? 0;\n } else if (event.type === 'message_delta') {\n this.#outputTokens = event.usage.output_tokens;\n } else if (\n event.type === 'content_block_start' &&\n event.content_block.type === 'tool_use'\n ) {\n this.#toolCallId = event.content_block.id;\n this.#fncName = event.content_block.name;\n this.#fncRawArgs = '';\n } else if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {\n let text = event.delta.text;\n\n // Filter chain-of-thought <thinking> blocks when tools are active\n if (this.#toolCtx) {\n const thinkingStart = text.indexOf('<thinking>');\n if (thinkingStart >= 0) {\n const preThinking = text.slice(0, thinkingStart);\n if (preThinking) {\n this.queue.put({\n id: this.#requestId,\n delta: { role: 'assistant', content: preThinking },\n });\n retryable = false;\n }\n text = text.slice(thinkingStart + '<thinking>'.length);\n this.#ignoringCoT = true;\n }\n if (this.#ignoringCoT) {\n const thinkingEnd = text.indexOf('</thinking>');\n if (thinkingEnd >= 0) {\n text = text.slice(thinkingEnd + '</thinking>'.length);\n this.#ignoringCoT = false;\n }\n }\n }\n\n if (this.#ignoringCoT) {\n continue;\n }\n\n // A delta can be reduced to an empty string when it was entirely a\n // <thinking> block; don't emit it or mark the stream non-retryable.\n if (text) {\n this.queue.put({\n id: this.#requestId,\n delta: { role: 'assistant', content: text },\n });\n retryable = false;\n }\n } else if (\n event.type === 'content_block_delta' &&\n event.delta.type === 'input_json_delta'\n ) {\n this.#fncRawArgs += event.delta.partial_json;\n } else if (event.type === 'content_block_stop' && this.#toolCallId) {\n this.queue.put({\n id: this.#requestId,\n delta: {\n role: 'assistant',\n toolCalls: [\n llm.FunctionCall.create({\n callId: this.#toolCallId,\n name: this.#fncName || '',\n args: this.#fncRawArgs || '',\n }),\n ],\n },\n });\n this.#toolCallId = undefined;\n this.#fncName = undefined;\n this.#fncRawArgs = undefined;\n retryable = false;\n }\n }\n\n // Emit final usage chunk. Anthropic reports cached tokens separately from\n // input_tokens, so fold them into promptTokens like the Python plugin does.\n const promptTokens = this.#inputTokens + this.#cacheCreationTokens + this.#cacheReadTokens;\n this.queue.put({\n id: this.#requestId,\n usage: {\n completionTokens: this.#outputTokens,\n promptTokens,\n totalTokens: promptTokens + this.#outputTokens,\n promptCachedTokens: this.#cacheReadTokens,\n },\n });\n } catch (e: unknown) {\n if (e instanceof Anthropic.APIError) {\n if (e.status === 408) {\n throw new APITimeoutError({\n message: e.message,\n options: { retryable },\n });\n }\n throw new APIStatusError({\n message: e.message,\n options: {\n statusCode: e.status,\n body: e.error as object,\n retryable: retryable && (e.status === 429 || e.status >= 500),\n },\n });\n }\n throw new APIConnectionError({\n message: e instanceof Error ? e.message : String(e),\n options: { retryable },\n });\n }\n }\n}\n"],"mappings":"AAGA,OAAO,eAAe;AAEtB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAuBP,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA;AAAA,EAEP,QAAQ,QAAQ,IAAI;AAAA,EACpB,mBAAmB;AACrB;AAUO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EAEA,YAAY,OAA4B,mBAAmB;AACzD,UAAM;AAEN,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,CAAC,KAAK,MAAM,UAAU,CAAC,KAAK,MAAM,QAAQ;AAC5C,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,UAAU;AAAA,MACZ,SAAS,KAAK,MAAM;AAAA,MACpB,QAAQ,KAAK,MAAM;AAAA,IACrB,CAAC;AAAA,EACL;AAAA;AAAA,EAGA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,IAAI,QAAgB;AAClB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,WAAmB;AACrB,QAAI;AACF,YAAM,MAAM,IAAI,IAAI,KAAK,QAAQ,OAAO;AACxC,aAAO,IAAI;AAAA,IACb,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiBU,uBAAuB,SAG/B;AACA,UAAM,SAAqC,CAAC;AAC5C,UAAM,cAAwC,CAAC;AAE/C,eAAW,OAAO,QAAQ,OAAO;AAC/B,UAAI,IAAI,SAAS,WAAW;AAC1B,cAAM,cAAc,IAAI,eAAe;AACvC,YAAI,IAAI,SAAS,YAAY,IAAI,SAAS,aAAa;AACrD,iBAAO,KAAK,EAAE,MAAM,QAAQ,MAAM,YAAY,CAAC;AAAA,QACjD,WAAW,IAAI,SAAS,UAAU,IAAI,SAAS,aAAa;AAC1D,sBAAY,KAAK;AAAA,YACf,MAAM,IAAI;AAAA,YACV,SAAS;AAAA,UACX,CAAC;AAAA,QACH;AAAA,MACF,WAAW,IAAI,SAAS,iBAAiB;AAEvC,oBAAY,KAAK;AAAA,UACf,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,cACE,MAAM;AAAA,cACN,IAAI,IAAI;AAAA,cACR,MAAM,IAAI;AAAA,cACV,OAAO,KAAK,MAAM,IAAI,QAAQ,IAAI;AAAA,YACpC;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH,WAAW,IAAI,SAAS,wBAAwB;AAE9C,oBAAY,KAAK;AAAA,UACf,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,cACE,MAAM;AAAA,cACN,aAAa,IAAI;AAAA,cACjB,SAAS,IAAI;AAAA,cACb,UAAU,IAAI;AAAA,YAChB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAGA,UAAM,WAAqC,CAAC;AAC5C,eAAW,OAAO,aAAa;AAC7B,YAAM,OAAO,SAAS,SAAS,SAAS,CAAC;AACzC,UAAI,QAAQ,KAAK,SAAS,IAAI,MAAM;AAElC,cAAM,cAAc,MAAM,QAAQ,KAAK,OAAO,IAC1C,KAAK,UACL,CAAC,EAAE,MAAM,QAAiB,MAAM,KAAK,QAAkB,CAAC;AAC5D,cAAM,aAAa,MAAM,QAAQ,IAAI,OAAO,IACxC,IAAI,UACJ,CAAC,EAAE,MAAM,QAAiB,MAAM,IAAI,QAAkB,CAAC;AAC3D,aAAK,UAAU,CAAC,GAAG,aAAa,GAAG,UAAU;AAAA,MAC/C,OAAO;AACL,iBAAS,KAAK,EAAE,GAAG,IAAI,CAAC;AAAA,MAC1B;AAAA,IACF;AAGA,QAAI,SAAS,WAAW,KAAK,SAAS,CAAC,EAAG,SAAS,QAAQ;AACzD,eAAS,QAAQ,EAAE,MAAM,QAAQ,SAAS,UAAU,CAAC;AAAA,IACvD;AAGA,QAAI,SAAS,SAAS,KAAK,SAAS,SAAS,SAAS,CAAC,EAAG,SAAS,aAAa;AAC9E,eAAS,KAAK,EAAE,MAAM,QAAQ,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,IACxE;AAEA,WAAO,EAAE,QAAQ,SAAS;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,KAAK;AAAA,IACH;AAAA,IACA,SAAS;AAAA,IACT,cAAc;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAOc;AACZ,UAAM,SAAkC,EAAE,GAAG,YAAY;AAEzD,QAAI,KAAK,MAAM,gBAAgB,OAAW,QAAO,cAAc,KAAK,MAAM;AAE1E,UAAM,EAAE,QAAQ,SAAS,IAAI,KAAK,uBAAuB,OAAO;AAGhE,UAAM,UAAU,IAAI,cAAc,YAAY;AAC9C,UAAM,iBAAmC,CAAC;AAC1C,QAAI,SAAS;AACX,iBAAW,CAAC,MAAM,IAAI,KAAK,IAAI,kBAAkB,OAAO,GAAG;AACzD,uBAAe,KAAK;AAAA,UAClB;AAAA,UACA,aAAa,KAAK,eAAe;AAAA,UACjC,cAAe,KAAK,aAChB,IAAI,aAAa,KAAK,YAAY,KAAK,IACvC,EAAE,MAAM,UAAU,YAAY,CAAC,EAAE;AAAA,QACvC,CAAC;AAAA,MACH;AAAA,IACF;AAGA,UAAM,qBAAqB,cAAc,KAAK,MAAM;AACpD,UAAM,mBAAmB,qBAAqB,KAAK,MAAM;AAEzD,SAAK,sBAAsB,qBAAqB,WAAc,eAAe,SAAS,GAAG;AACvF,UAAI,sBAA2D,EAAE,MAAM,OAAO;AAE9E,UAAI,OAAO,uBAAuB,UAAU;AAC1C,YAAI,uBAAuB,YAAY;AACrC,gCAAsB,EAAE,MAAM,MAAM;AAAA,QACtC,WAAW,uBAAuB,QAAQ;AAExC,yBAAe,SAAS;AACxB,gCAAsB;AAAA,QACxB;AAAA,MACF,WACE,OAAO,uBAAuB,YAC9B,UAAU,sBACV,mBAAmB,SAAS,YAC5B;AACA,cAAM,KAAM,mBAAsD;AAClE,8BAAsB,EAAE,MAAM,QAAQ,MAAM,GAAG,KAAK;AAAA,MACtD;AAEA,UAAI,qBAAqB;AAEvB,YAAI,qBAAqB,QAAW;AAClC,8BAAoB,4BAA4B,CAAC;AAAA,QACnD;AACA,eAAO,cAAc;AAAA,MACvB;AAAA,IACF;AAEA,UAAM,gBAAwD;AAAA,MAC5D,OAAO,KAAK,MAAM;AAAA,MAClB;AAAA,MACA,QAAQ,OAAO,SAAS,IAAI,SAAS;AAAA,MACrC,OAAO,eAAe,SAAS,IAAI,iBAAiB;AAAA,MACpD,QAAQ;AAAA,MACR,YAAY,KAAK,MAAM,aAAa;AAAA,MACpC,GAAG;AAAA,IACL;AAEA,WAAO,IAAI,UAAU,MAAM,KAAK,SAAS,eAAe,SAAS,SAAS,WAAW;AAAA,EACvF;AACF;AAWO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa;AAAA,EACb,eAAe;AAAA,EACf,eAAe;AAAA,EACf,gBAAgB;AAAA,EAChB,uBAAuB;AAAA,EACvB,mBAAmB;AAAA,EACnB;AAAA,EAEA,YACE,SACA,QACA,eACA,SACA,SACA,aACA;AACA,UAAM,SAAS,EAAE,SAAS,SAAS,YAAY,CAAC;AAChD,SAAK,UAAU;AACf,SAAK,iBAAiB;AACtB,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAgB,MAAqB;AACnC,QAAI,YAAY;AAChB,SAAK,cAAc;AACnB,SAAK,WAAW;AAChB,SAAK,cAAc;AACnB,SAAK,aAAa;AAClB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB;AACrB,SAAK,uBAAuB;AAC5B,SAAK,mBAAmB;AAExB,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,SAAS,OAAO,KAAK,gBAAgB;AAAA,QACrE,SAAS,KAAK,YAAY;AAAA,MAC5B,CAAC;AACD,uBAAiB,SAAS,QAAQ;AAChC,YAAI,MAAM,SAAS,iBAAiB;AAClC,eAAK,aAAa,MAAM,QAAQ;AAChC,eAAK,eAAe,MAAM,QAAQ,MAAM;AACxC,eAAK,gBAAgB,MAAM,QAAQ,MAAM;AACzC,eAAK,uBAAuB,MAAM,QAAQ,MAAM,+BAA+B;AAC/E,eAAK,mBAAmB,MAAM,QAAQ,MAAM,2BAA2B;AAAA,QACzE,WAAW,MAAM,SAAS,iBAAiB;AACzC,eAAK,gBAAgB,MAAM,MAAM;AAAA,QACnC,WACE,MAAM,SAAS,yBACf,MAAM,cAAc,SAAS,YAC7B;AACA,eAAK,cAAc,MAAM,cAAc;AACvC,eAAK,WAAW,MAAM,cAAc;AACpC,eAAK,cAAc;AAAA,QACrB,WAAW,MAAM,SAAS,yBAAyB,MAAM,MAAM,SAAS,cAAc;AACpF,cAAI,OAAO,MAAM,MAAM;AAGvB,cAAI,KAAK,UAAU;AACjB,kBAAM,gBAAgB,KAAK,QAAQ,YAAY;AAC/C,gBAAI,iBAAiB,GAAG;AACtB,oBAAM,cAAc,KAAK,MAAM,GAAG,aAAa;AAC/C,kBAAI,aAAa;AACf,qBAAK,MAAM,IAAI;AAAA,kBACb,IAAI,KAAK;AAAA,kBACT,OAAO,EAAE,MAAM,aAAa,SAAS,YAAY;AAAA,gBACnD,CAAC;AACD,4BAAY;AAAA,cACd;AACA,qBAAO,KAAK,MAAM,gBAAgB,aAAa,MAAM;AACrD,mBAAK,eAAe;AAAA,YACtB;AACA,gBAAI,KAAK,cAAc;AACrB,oBAAM,cAAc,KAAK,QAAQ,aAAa;AAC9C,kBAAI,eAAe,GAAG;AACpB,uBAAO,KAAK,MAAM,cAAc,cAAc,MAAM;AACpD,qBAAK,eAAe;AAAA,cACtB;AAAA,YACF;AAAA,UACF;AAEA,cAAI,KAAK,cAAc;AACrB;AAAA,UACF;AAIA,cAAI,MAAM;AACR,iBAAK,MAAM,IAAI;AAAA,cACb,IAAI,KAAK;AAAA,cACT,OAAO,EAAE,MAAM,aAAa,SAAS,KAAK;AAAA,YAC5C,CAAC;AACD,wBAAY;AAAA,UACd;AAAA,QACF,WACE,MAAM,SAAS,yBACf,MAAM,MAAM,SAAS,oBACrB;AACA,eAAK,eAAe,MAAM,MAAM;AAAA,QAClC,WAAW,MAAM,SAAS,wBAAwB,KAAK,aAAa;AAClE,eAAK,MAAM,IAAI;AAAA,YACb,IAAI,KAAK;AAAA,YACT,OAAO;AAAA,cACL,MAAM;AAAA,cACN,WAAW;AAAA,gBACT,IAAI,aAAa,OAAO;AAAA,kBACtB,QAAQ,KAAK;AAAA,kBACb,MAAM,KAAK,YAAY;AAAA,kBACvB,MAAM,KAAK,eAAe;AAAA,gBAC5B,CAAC;AAAA,cACH;AAAA,YACF;AAAA,UACF,CAAC;AACD,eAAK,cAAc;AACnB,eAAK,WAAW;AAChB,eAAK,cAAc;AACnB,sBAAY;AAAA,QACd;AAAA,MACF;AAIA,YAAM,eAAe,KAAK,eAAe,KAAK,uBAAuB,KAAK;AAC1E,WAAK,MAAM,IAAI;AAAA,QACb,IAAI,KAAK;AAAA,QACT,OAAO;AAAA,UACL,kBAAkB,KAAK;AAAA,UACvB;AAAA,UACA,aAAa,eAAe,KAAK;AAAA,UACjC,oBAAoB,KAAK;AAAA,QAC3B;AAAA,MACF,CAAC;AAAA,IACH,SAAS,GAAY;AACnB,UAAI,aAAa,UAAU,UAAU;AACnC,YAAI,EAAE,WAAW,KAAK;AACpB,gBAAM,IAAI,gBAAgB;AAAA,YACxB,SAAS,EAAE;AAAA,YACX,SAAS,EAAE,UAAU;AAAA,UACvB,CAAC;AAAA,QACH;AACA,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,EAAE;AAAA,UACX,SAAS;AAAA,YACP,YAAY,EAAE;AAAA,YACd,MAAM,EAAE;AAAA,YACR,WAAW,cAAc,EAAE,WAAW,OAAO,EAAE,UAAU;AAAA,UAC3D;AAAA,QACF,CAAC;AAAA,MACH;AACA,YAAM,IAAI,mBAAmB;AAAA,QAC3B,SAAS,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;AAAA,QAClD,SAAS,EAAE,UAAU;AAAA,MACvB,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}
|