llmapi-v2 2.1.0

package/src/converter/stream.ts

@@ -0,0 +1,373 @@
+import type { Response } from 'express';
+import type { IncomingMessage } from 'http';
+import type { OpenAIStreamChunk, OpenAIStreamToolCall } from './types';
+import { v4 as uuidv4 } from 'uuid';
+import { logger } from '../utils/logger';
+
+interface ToolCallAccumulator {
+  id: string;
+  name: string;
+  arguments: string;
+  blockIndex: number;
+  started: boolean; // Whether content_block_start has been sent
+}
+
+export interface StreamResult {
+  inputTokens: number;
+  outputTokens: number;
+  thinkingTokens: number;
+  ttftMs: number;
+  tokensPerSec: number;
+  durationMs: number;
+}
+
+/**
+ * Transforms an OpenAI SSE stream into Anthropic SSE stream format.
+ *
+ * This is the most critical component of the relay service.
+ * Claude Code expects Anthropic's streaming format exactly:
+ *   message_start -> content_block_start/delta/stop (repeated) -> message_delta -> message_stop
+ */
+export class AnthropicStreamTransformer {
+  // Block tracking
+  private blockIndex = 0;
+  private thinkingBlockOpen = false;
+  private textBlockOpen = false;
+  private toolCalls = new Map<number, ToolCallAccumulator>();
+
+  // State
+  private hasToolUse = false;
+  private finished = false;
+  private buffer = '';
+
+  // Metrics
+  private usage = { inputTokens: 0, outputTokens: 0 };
+  private thinkingCharCount = 0;
+  private outputCharCount = 0;
+  private firstTokenTime = 0;
+  private startTime = Date.now();
+
+  // Message identity
+  private messageId: string;
+
+  constructor(
+    private res: Response,
+    private displayModel: string,
+  ) {
+    this.messageId = `msg_${uuidv4().replace(/-/g, '')}`;
+  }
+
+  /**
+   * Pipe an OpenAI streaming response through this transformer.
+   * Returns usage stats when complete.
+   */
+  async pipe(backendRes: IncomingMessage): Promise<StreamResult> {
+    // Set SSE headers
+    this.res.setHeader('Content-Type', 'text/event-stream');
+    this.res.setHeader('Cache-Control', 'no-cache');
+    this.res.setHeader('Connection', 'keep-alive');
+    this.res.flushHeaders();
+
+    // Handle client disconnect
+    let clientDisconnected = false;
+    this.res.on('close', () => {
+      clientDisconnected = true;
+      backendRes.destroy();
+    });
+
+    // Send message_start
+    this.sendSSE('message_start', {
+      type: 'message_start',
+      message: {
+        id: this.messageId,
+        type: 'message',
+        role: 'assistant',
+        model: this.displayModel,
+        content: [],
+        stop_reason: null,
+        stop_sequence: null,
+        usage: { input_tokens: 0, output_tokens: 0 },
+      },
+    });
+
+    // Send initial ping
+    this.sendSSE('ping', { type: 'ping' });
+
+    return new Promise<StreamResult>((resolve, reject) => {
+      backendRes.setEncoding('utf8');
+
+      backendRes.on('data', (chunk: string) => {
+        if (clientDisconnected) return;
+        this.buffer += chunk;
+        this.processBuffer();
+      });
+
+      backendRes.on('end', () => {
+        // Process any remaining buffer
+        if (this.buffer.trim()) {
+          this.processBuffer();
+        }
+        if (!this.finished) {
+          this.finish();
+        }
+        resolve(this.getResult());
+      });
+
+      backendRes.on('error', (err) => {
+        logger.error({ err }, 'Backend stream error');
+        if (!this.finished) {
+          this.finish();
+        }
+        resolve(this.getResult());
+      });
+    });
+  }
+
+  private processBuffer(): void {
+    const lines = this.buffer.split('\n');
+    this.buffer = lines.pop() || ''; // Keep incomplete last line
+
+    for (const line of lines) {
+      if (!line.startsWith('data: ')) continue;
+      const payload = line.slice(6).trim();
+
+      if (payload === '[DONE]') {
+        this.finish();
+        return;
+      }
+
+      let chunk: OpenAIStreamChunk;
+      try {
+        chunk = JSON.parse(payload);
+      } catch {
+        continue;
+      }
+
+      this.processChunk(chunk);
+    }
+  }
+
+  private processChunk(chunk: OpenAIStreamChunk): void {
+    // Track usage from provider
+    if (chunk.usage) {
+      if (chunk.usage.prompt_tokens) this.usage.inputTokens = chunk.usage.prompt_tokens;
+      if (chunk.usage.completion_tokens) this.usage.outputTokens = chunk.usage.completion_tokens;
+    }
+
+    const choice = chunk.choices?.[0];
+    if (!choice) return;
+
+    const delta = choice.delta;
+
+    // Handle reasoning/thinking content
+    if (delta.reasoning_content) {
+      this.handleThinking(delta.reasoning_content);
+    }
+
+    // Handle text content
+    if (delta.content) {
+      this.handleText(delta.content);
+    }
+
+    // Handle tool calls
+    if (delta.tool_calls) {
+      this.handleToolCalls(delta.tool_calls);
+    }
+
+    // Handle finish_reason (some providers send this instead of [DONE])
+    if (choice.finish_reason && !this.finished) {
+      this.finish();
+    }
+  }
+
+  private handleThinking(text: string): void {
+    this.thinkingCharCount += text.length;
+
+    if (!this.thinkingBlockOpen) {
+      this.sendSSE('content_block_start', {
+        type: 'content_block_start',
+        index: this.blockIndex,
+        content_block: { type: 'thinking', thinking: '' },
+      });
+      this.thinkingBlockOpen = true;
+    }
+
+    this.sendSSE('content_block_delta', {
+      type: 'content_block_delta',
+      index: this.blockIndex,
+      delta: { type: 'thinking_delta', thinking: text },
+    });
+  }
+
+  private handleText(text: string): void {
+    if (!this.firstTokenTime) this.firstTokenTime = Date.now();
+    this.outputCharCount += text.length;
+
+    // Close thinking block if open (thinking always comes before text)
+    this.closeThinkingBlock();
+
+    if (!this.textBlockOpen) {
+      this.sendSSE('content_block_start', {
+        type: 'content_block_start',
+        index: this.blockIndex,
+        content_block: { type: 'text', text: '' },
+      });
+      this.textBlockOpen = true;
+    }
+
+    this.sendSSE('content_block_delta', {
+      type: 'content_block_delta',
+      index: this.blockIndex,
+      delta: { type: 'text_delta', text },
+    });
+  }
+
+  private handleToolCalls(toolCalls: OpenAIStreamToolCall[]): void {
+    // Close thinking and text blocks before tool use
+    this.closeThinkingBlock();
+    this.closeTextBlock();
+    this.hasToolUse = true;
+
+    for (const tc of toolCalls) {
+      const tcIndex = tc.index;
+      let acc = this.toolCalls.get(tcIndex);
+
+      // New tool call: create accumulator
+      if (!acc && tc.function?.name) {
+        acc = {
+          id: tc.id || `toolu_${uuidv4().replace(/-/g, '').slice(0, 24)}`,
+          name: tc.function.name,
+          arguments: '',
+          blockIndex: this.blockIndex,
+          started: false,
+        };
+        this.toolCalls.set(tcIndex, acc);
+        this.blockIndex++; // Reserve a block index for this tool
+      }
+
+      if (!acc) continue;
+
+      // Start the content block if not yet started
+      if (!acc.started) {
+        this.sendSSE('content_block_start', {
+          type: 'content_block_start',
+          index: acc.blockIndex,
+          content_block: {
+            type: 'tool_use',
+            id: acc.id,
+            name: acc.name,
+            input: {},
+          },
+        });
+        acc.started = true;
+      }
+
+      // Stream argument chunks
+      if (tc.function?.arguments) {
+        acc.arguments += tc.function.arguments;
+        this.sendSSE('content_block_delta', {
+          type: 'content_block_delta',
+          index: acc.blockIndex,
+          delta: { type: 'input_json_delta', partial_json: tc.function.arguments },
+        });
+      }
+    }
+  }
+
+  private closeThinkingBlock(): void {
+    if (this.thinkingBlockOpen) {
+      this.sendSSE('content_block_stop', {
+        type: 'content_block_stop',
+        index: this.blockIndex,
+      });
+      this.blockIndex++;
+      this.thinkingBlockOpen = false;
+    }
+  }
+
+  private closeTextBlock(): void {
+    if (this.textBlockOpen) {
+      this.sendSSE('content_block_stop', {
+        type: 'content_block_stop',
+        index: this.blockIndex,
+      });
+      this.blockIndex++;
+      this.textBlockOpen = false;
+    }
+  }
+
+  private closeAllToolBlocks(): void {
+    for (const [, acc] of this.toolCalls) {
+      if (acc.started) {
+        this.sendSSE('content_block_stop', {
+          type: 'content_block_stop',
+          index: acc.blockIndex,
+        });
+      }
+    }
+    this.toolCalls.clear();
+  }
+
+  /**
+   * Finalize the stream. Called once when [DONE] or finish_reason is received.
+   */
+  private finish(): void {
+    if (this.finished) return;
+    this.finished = true;
+
+    // Determine stop_reason BEFORE closing blocks
+    const stopReason = this.hasToolUse ? 'tool_use' : 'end_turn';
+
+    // Close all open blocks
+    this.closeThinkingBlock();
+    this.closeTextBlock();
+    this.closeAllToolBlocks();
+
+    // Send message_delta with final stop_reason
+    this.sendSSE('message_delta', {
+      type: 'message_delta',
+      delta: { stop_reason: stopReason, stop_sequence: null },
+      usage: { output_tokens: this.usage.outputTokens },
+    });
+
+    // Send message_stop
+    this.sendSSE('message_stop', { type: 'message_stop' });
+
+    // End the response
+    if (!this.res.writableEnded) {
+      this.res.end();
+    }
+  }
+
+  private sendSSE(event: string, data: object): void {
+    if (this.res.writableEnded || this.res.destroyed) return;
+    try {
+      this.res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+    } catch {
+      // Client disconnected, ignore
+    }
+  }
+
+  private getResult(): StreamResult {
+    const elapsed = Date.now() - this.startTime;
+    const ttft = this.firstTokenTime ? this.firstTokenTime - this.startTime : elapsed;
+
+    // Estimate tokens from char count if provider didn't report usage
+    if (this.usage.outputTokens === 0 && this.outputCharCount > 0) {
+      this.usage.outputTokens = Math.ceil(this.outputCharCount / 3);
+    }
+
+    const tokensPerSec = elapsed > 0
+      ? Math.round((this.usage.outputTokens / (elapsed / 1000)) * 100) / 100
+      : 0;
+
+    return {
+      inputTokens: this.usage.inputTokens,
+      outputTokens: this.usage.outputTokens,
+      thinkingTokens: Math.ceil(this.thinkingCharCount / 3),
+      ttftMs: ttft,
+      tokensPerSec,
+      durationMs: elapsed,
+    };
+  }
+}

package/src/converter/types.ts

@@ -0,0 +1,257 @@
+// ============================================================================
+// Anthropic Messages API Types (what Claude Code sends/expects)
+// ============================================================================
+
+// --- Request ---
+
+export interface AnthropicRequest {
+  model: string;
+  messages: AnthropicMessage[];
+  system?: string | AnthropicSystemBlock[];
+  max_tokens: number;
+  temperature?: number;
+  top_p?: number;
+  tools?: AnthropicToolDefinition[];
+  tool_choice?: AnthropicToolChoice;
+  stream?: boolean;
+  stop_sequences?: string[];
+  metadata?: { user_id?: string };
+  thinking?: { type: 'enabled'; budget_tokens: number };
+}
+
+export interface AnthropicMessage {
+  role: 'user' | 'assistant';
+  content: string | AnthropicContentBlock[];
+}
+
+export type AnthropicContentBlock =
+  | AnthropicTextBlock
+  | AnthropicThinkingBlock
+  | AnthropicToolUseBlock
+  | AnthropicToolResultBlock
+  | AnthropicImageBlock;
+
+export interface AnthropicTextBlock {
+  type: 'text';
+  text: string;
+}
+
+export interface AnthropicThinkingBlock {
+  type: 'thinking';
+  thinking: string;
+}
+
+export interface AnthropicToolUseBlock {
+  type: 'tool_use';
+  id: string;
+  name: string;
+  input: Record<string, unknown>;
+}
+
+export interface AnthropicToolResultBlock {
+  type: 'tool_result';
+  tool_use_id: string;
+  content: string | AnthropicContentBlock[];
+  is_error?: boolean;
+}
+
+export interface AnthropicImageBlock {
+  type: 'image';
+  source: {
+    type: 'base64';
+    media_type: string;
+    data: string;
+  };
+}
+
+export interface AnthropicSystemBlock {
+  type: 'text';
+  text: string;
+  cache_control?: { type: 'ephemeral' };
+}
+
+export interface AnthropicToolDefinition {
+  name: string;
+  description?: string;
+  input_schema: Record<string, unknown>;
+}
+
+export type AnthropicToolChoice =
+  | { type: 'auto' }
+  | { type: 'any' }
+  | { type: 'tool'; name: string };
+
+// --- Response ---
+
+export interface AnthropicResponse {
+  id: string;
+  type: 'message';
+  role: 'assistant';
+  model: string;
+  content: AnthropicResponseBlock[];
+  stop_reason: 'end_turn' | 'max_tokens' | 'stop_sequence' | 'tool_use';
+  stop_sequence: string | null;
+  usage: AnthropicUsage;
+}
+
+export type AnthropicResponseBlock =
+  | AnthropicTextBlock
+  | AnthropicThinkingBlock
+  | AnthropicToolUseBlock;
+
+export interface AnthropicUsage {
+  input_tokens: number;
+  output_tokens: number;
+  cache_creation_input_tokens?: number;
+  cache_read_input_tokens?: number;
+}
+
+// --- Streaming Events ---
+
+export type AnthropicStreamEvent =
+  | { type: 'message_start'; message: AnthropicResponse }
+  | { type: 'content_block_start'; index: number; content_block: AnthropicResponseBlock }
+  | { type: 'content_block_delta'; index: number; delta: AnthropicDelta }
+  | { type: 'content_block_stop'; index: number }
+  | { type: 'message_delta'; delta: { stop_reason: string; stop_sequence: string | null }; usage: { output_tokens: number } }
+  | { type: 'message_stop' }
+  | { type: 'ping' };
+
+export type AnthropicDelta =
+  | { type: 'text_delta'; text: string }
+  | { type: 'thinking_delta'; thinking: string }
+  | { type: 'input_json_delta'; partial_json: string };
+
+
+// ============================================================================
+// OpenAI Chat Completions API Types (what providers accept/return)
+// ============================================================================
+
+// --- Request ---
+
+export interface OpenAIRequest {
+  model: string;
+  messages: OpenAIMessage[];
+  max_tokens?: number;
+  temperature?: number;
+  top_p?: number;
+  tools?: OpenAITool[];
+  tool_choice?: string | OpenAIToolChoiceObject;
+  stream: boolean;
+  stream_options?: { include_usage: boolean };
+  stop?: string[];
+}
+
+export type OpenAIMessage =
+  | OpenAISystemMessage
+  | OpenAIUserMessage
+  | OpenAIAssistantMessage
+  | OpenAIToolMessage;
+
+export interface OpenAISystemMessage {
+  role: 'system';
+  content: string;
+}
+
+export interface OpenAIUserMessage {
+  role: 'user';
+  content: string | OpenAIContentPart[];
+}
+
+export type OpenAIContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image_url'; image_url: { url: string } };
+
+export interface OpenAIAssistantMessage {
+  role: 'assistant';
+  content: string | null;
+  tool_calls?: OpenAIToolCall[];
+}
+
+export interface OpenAIToolMessage {
+  role: 'tool';
+  tool_call_id: string;
+  content: string;
+}
+
+export interface OpenAITool {
+  type: 'function';
+  function: {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+  };
+}
+
+export interface OpenAIToolChoiceObject {
+  type: 'function';
+  function: { name: string };
+}
+
+export interface OpenAIToolCall {
+  id: string;
+  type: 'function';
+  function: {
+    name: string;
+    arguments: string;
+  };
+}
+
+// --- Response ---
+
+export interface OpenAIResponse {
+  id: string;
+  object: string;
+  created: number;
+  model: string;
+  choices: OpenAIChoice[];
+  usage?: OpenAIUsage;
+}
+
+export interface OpenAIChoice {
+  index: number;
+  message: OpenAIAssistantMessage & {
+    reasoning_content?: string;
+  };
+  finish_reason: 'stop' | 'length' | 'tool_calls' | null;
+}
+
+export interface OpenAIUsage {
+  prompt_tokens: number;
+  completion_tokens: number;
+  total_tokens: number;
+}
+
+// --- Streaming ---
+
+export interface OpenAIStreamChunk {
+  id: string;
+  object: string;
+  created: number;
+  model: string;
+  choices: OpenAIStreamChoice[];
+  usage?: OpenAIUsage;
+}
+
+export interface OpenAIStreamChoice {
+  index: number;
+  delta: OpenAIStreamDelta;
+  finish_reason: 'stop' | 'length' | 'tool_calls' | null;
+}
+
+export interface OpenAIStreamDelta {
+  role?: string;
+  content?: string | null;
+  reasoning_content?: string;
+  tool_calls?: OpenAIStreamToolCall[];
+}
+
+export interface OpenAIStreamToolCall {
+  index: number;
+  id?: string;
+  type?: string;
+  function?: {
+    name?: string;
+    arguments?: string;
+  };
+}