memorix 1.1.7 → 1.1.8

package/src/multimodal/image-loader.ts

@@ -1,143 +1,143 @@
-/**
- * Image Loader — Vision LLM Integration
- *
- * Analyzes images via OpenAI Vision API (or compatible),
- * extracting descriptions, tags, and entities.
- */
-
-import { getLLMApiKey, getLLMBaseUrl, getLLMModel } from '../config.js';
-import { isLLMEnabled, getLLMConfig } from '../llm/provider.js';
-
-// Providers that use the OpenAI-compatible /chat/completions Vision endpoint
-const OPENAI_COMPATIBLE_PROVIDERS = new Set(['openai', 'openrouter', 'custom']);
-
-// ── Types ────────────────────────────────────────────────────────────
-
-export interface ImageInput {
-  /** Base64-encoded image data */
-  base64: string;
-  /** Image MIME type (default: image/png) */
-  mimeType?: string;
-  /** Original filename */
-  filename?: string;
-  /** Custom analysis prompt */
-  prompt?: string;
-}
-
-export interface ImageAnalysisResult {
-  /** Natural language description of the image */
-  description: string;
-  /** Relevant tags/categories */
-  tags: string[];
-  /** Key entities/concepts depicted */
-  entities: string[];
-}
-
-// ── Internal Vision LLM Call ─────────────────────────────────────────
-
-async function callVisionLLM(
-  systemPrompt: string,
-  imageBase64: string,
-  mimeType: string,
-): Promise<string> {
-  const apiKey = getLLMApiKey();
-  if (!apiKey) {
-    throw new Error('No LLM API key configured for image analysis.');
-  }
-
-  let baseUrl = getLLMBaseUrl('https://api.openai.com/v1').replace(/\/+$/, '');
-  if (!baseUrl.endsWith('/v1')) baseUrl += '/v1';
-  const model = getLLMModel('gpt-4o');
-
-  const response = await fetch(`${baseUrl}/chat/completions`, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${apiKey}`,
-    },
-    body: JSON.stringify({
-      model,
-      messages: [{
-        role: 'user',
-        content: [
-          { type: 'text', text: systemPrompt },
-          {
-            type: 'image_url',
-            image_url: { url: `data:${mimeType};base64,${imageBase64}` },
-          },
-        ],
-      }],
-      temperature: 0.1,
-      max_tokens: 1024,
-    }),
-    signal: AbortSignal.timeout(30_000),
-  });
-
-  if (!response.ok) {
-    const errorText = await response.text().catch(() => 'unknown error');
-    throw new Error(`Vision LLM error (${response.status}): ${errorText}`);
-  }
-
-  const data = await response.json() as {
-    choices: Array<{ message: { content: string } }>;
-  };
-
-  return data.choices[0]?.message?.content ?? '';
-}
-
-// ── Public API ───────────────────────────────────────────────────────
-
-const DEFAULT_PROMPT =
-  'Analyze this image. Return ONLY a JSON object with this exact format: ' +
-  '{"description": "detailed description", "tags": ["tag1", "tag2"], "entities": ["entity1", "entity2"]}';
-
-/**
- * Analyze an image using Vision LLM.
- *
- * @throws Error if LLM not configured.
- */
-export async function analyzeImage(input: ImageInput): Promise<ImageAnalysisResult> {
-  if (!isLLMEnabled()) {
-    throw new Error(
-      'LLM not configured for image analysis. ' +
-      'Set MEMORIX_LLM_API_KEY or OPENAI_API_KEY.',
-    );
-  }
-
-  const config = getLLMConfig()!;
-  if (!OPENAI_COMPATIBLE_PROVIDERS.has(config.provider)) {
-    throw new Error(
-      `Image analysis requires an OpenAI-compatible provider (openai, openrouter, or custom). ` +
-      `Current provider "${config.provider}" uses a different API shape. ` +
-      `Set MEMORIX_LLM_PROVIDER=openai or configure an OpenAI-compatible base URL.`,
-    );
-  }
-
-  const mimeType = input.mimeType ?? 'image/png';
-  const prompt = input.prompt ?? DEFAULT_PROMPT;
-
-  const response = await callVisionLLM(prompt, input.base64, mimeType);
-
-  // Try to parse structured JSON response
-  try {
-    // Extract JSON from response (may be wrapped in markdown code block)
-    const jsonMatch = response.match(/\{[\s\S]*\}/);
-    if (jsonMatch) {
-      const parsed = JSON.parse(jsonMatch[0]);
-      return {
-        description: parsed.description ?? response,
-        tags: Array.isArray(parsed.tags) ? parsed.tags : [],
-        entities: Array.isArray(parsed.entities) ? parsed.entities : [],
-      };
-    }
-  } catch {
-    // JSON parse failed — fall through to text extraction
-  }
-
-  // Fallback: treat entire response as description
-  return {
-    description: response,
-    tags: [],
-    entities: [],
-  };
-}
+/**
+ * Image Loader — Vision LLM Integration
+ *
+ * Analyzes images via OpenAI Vision API (or compatible),
+ * extracting descriptions, tags, and entities.
+ */
+
+import { getLLMApiKey, getLLMBaseUrl, getLLMModel } from '../config.js';
+import { isLLMEnabled, getLLMConfig } from '../llm/provider.js';
+
+// Providers that use the OpenAI-compatible /chat/completions Vision endpoint
+const OPENAI_COMPATIBLE_PROVIDERS = new Set(['openai', 'openrouter', 'custom']);
+
+// ── Types ────────────────────────────────────────────────────────────
+
+export interface ImageInput {
+  /** Base64-encoded image data */
+  base64: string;
+  /** Image MIME type (default: image/png) */
+  mimeType?: string;
+  /** Original filename */
+  filename?: string;
+  /** Custom analysis prompt */
+  prompt?: string;
+}
+
+export interface ImageAnalysisResult {
+  /** Natural language description of the image */
+  description: string;
+  /** Relevant tags/categories */
+  tags: string[];
+  /** Key entities/concepts depicted */
+  entities: string[];
+}
+
+// ── Internal Vision LLM Call ─────────────────────────────────────────
+
+async function callVisionLLM(
+  systemPrompt: string,
+  imageBase64: string,
+  mimeType: string,
+): Promise<string> {
+  const apiKey = getLLMApiKey();
+  if (!apiKey) {
+    throw new Error('No LLM API key configured for image analysis.');
+  }
+
+  let baseUrl = getLLMBaseUrl('https://api.openai.com/v1').replace(/\/+$/, '');
+  if (!baseUrl.endsWith('/v1')) baseUrl += '/v1';
+  const model = getLLMModel('gpt-4o');
+
+  const response = await fetch(`${baseUrl}/chat/completions`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: systemPrompt },
+          {
+            type: 'image_url',
+            image_url: { url: `data:${mimeType};base64,${imageBase64}` },
+          },
+        ],
+      }],
+      temperature: 0.1,
+      max_tokens: 1024,
+    }),
+    signal: AbortSignal.timeout(30_000),
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text().catch(() => 'unknown error');
+    throw new Error(`Vision LLM error (${response.status}): ${errorText}`);
+  }
+
+  const data = await response.json() as {
+    choices: Array<{ message: { content: string } }>;
+  };
+
+  return data.choices[0]?.message?.content ?? '';
+}
+
+// ── Public API ───────────────────────────────────────────────────────
+
+const DEFAULT_PROMPT =
+  'Analyze this image. Return ONLY a JSON object with this exact format: ' +
+  '{"description": "detailed description", "tags": ["tag1", "tag2"], "entities": ["entity1", "entity2"]}';
+
+/**
+ * Analyze an image using Vision LLM.
+ *
+ * @throws Error if LLM not configured.
+ */
+export async function analyzeImage(input: ImageInput): Promise<ImageAnalysisResult> {
+  if (!isLLMEnabled()) {
+    throw new Error(
+      'LLM not configured for image analysis. ' +
+      'Set MEMORIX_LLM_API_KEY or OPENAI_API_KEY.',
+    );
+  }
+
+  const config = getLLMConfig()!;
+  if (!OPENAI_COMPATIBLE_PROVIDERS.has(config.provider)) {
+    throw new Error(
+      `Image analysis requires an OpenAI-compatible provider (openai, openrouter, or custom). ` +
+      `Current provider "${config.provider}" uses a different API shape. ` +
+      `Set MEMORIX_LLM_PROVIDER=openai or configure an OpenAI-compatible base URL.`,
+    );
+  }
+
+  const mimeType = input.mimeType ?? 'image/png';
+  const prompt = input.prompt ?? DEFAULT_PROMPT;
+
+  const response = await callVisionLLM(prompt, input.base64, mimeType);
+
+  // Try to parse structured JSON response
+  try {
+    // Extract JSON from response (may be wrapped in markdown code block)
+    const jsonMatch = response.match(/\{[\s\S]*\}/);
+    if (jsonMatch) {
+      const parsed = JSON.parse(jsonMatch[0]);
+      return {
+        description: parsed.description ?? response,
+        tags: Array.isArray(parsed.tags) ? parsed.tags : [],
+        entities: Array.isArray(parsed.entities) ? parsed.entities : [],
+      };
+    }
+  } catch {
+    // JSON parse failed — fall through to text extraction
+  }
+
+  // Fallback: treat entire response as description
+  return {
+    description: response,
+    tags: [],
+    entities: [],
+  };
+}

package/src/orchestrate/adapters/claude-stream.ts

@@ -1,192 +1,192 @@
-/**
- * Claude Code stream-json parser.
- *
- * Parses NDJSON from `claude -p --output-format stream-json --verbose`
- * into AgentMessage events. Based on observed format from Multica reference
- * and Claude Code documentation.
- *
- * Message types:
- *   - "system"    → session_id, status
- *   - "assistant" → content blocks (text, thinking, tool_use) + usage
- *   - "user"      → tool_result content blocks
- *   - "result"    → final result text, is_error, session_id
- *   - "log"       → debug/info log entries
- */
-
-import type { AgentMessage, TokenUsage } from './types.js';
-
-// ── Claude SDK JSON shapes ──────────────────────────────────────
-
-interface ClaudeStreamLine {
-  type: string;
-  message?: unknown;
-  subtype?: string;
-  session_id?: string;
-  result?: string;
-  is_error?: boolean;
-  duration_ms?: number;
-  num_turns?: number;
-  log?: { level: string; message: string };
-  /** Per-model token usage — only present on 'result' messages */
-  modelUsage?: Record<string, {
-    inputTokens?: number;
-    outputTokens?: number;
-    cacheReadInputTokens?: number;
-    cacheCreationInputTokens?: number;
-    costUSD?: number;
-  }>;
-}
-
-interface ClaudeMessageContent {
-  role: string;
-  model?: string;
-  content?: ClaudeContentBlock[];
-  usage?: {
-    input_tokens?: number;
-    output_tokens?: number;
-    cache_read_input_tokens?: number;
-    cache_creation_input_tokens?: number;
-  };
-}
-
-interface ClaudeContentBlock {
-  type: string;
-  text?: string;
-  id?: string;
-  name?: string;
-  input?: unknown;
-  tool_use_id?: string;
-  content?: unknown;
-}
-
-// ── State ───────────────────────────────────────────────────────
-
-export interface ClaudeStreamState {
-  sessionId?: string;
-  usage: Record<string, TokenUsage>;
-  resultText?: string;
-  isError: boolean;
-}
-
-// ── Parser ──────────────────────────────────────────────────────
-
-/**
- * Parse a single NDJSON line from Claude stream-json output.
- * Returns zero or more AgentMessage events.
- */
-export function parseClaudeStreamLine(
-  raw: string,
-  state: ClaudeStreamState,
-): AgentMessage[] {
-  const line = raw.trim();
-  if (!line) return [];
-
-  let parsed: ClaudeStreamLine;
-  try {
-    parsed = JSON.parse(line);
-  } catch {
-    return [];
-  }
-
-  const messages: AgentMessage[] = [];
-
-  switch (parsed.type) {
-    case 'system':
-      if (parsed.session_id) {
-        state.sessionId = parsed.session_id;
-      }
-      break;
-
-    case 'assistant':
-      if (parsed.message) {
-        const content = parsed.message as ClaudeMessageContent;
-        // Accumulate usage
-        if (content.usage && content.model) {
-          const model = content.model;
-          const prev = state.usage[model] ?? { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, model };
-          prev.inputTokens += content.usage.input_tokens ?? 0;
-          prev.outputTokens += content.usage.output_tokens ?? 0;
-          prev.cacheReadTokens += content.usage.cache_read_input_tokens ?? 0;
-          prev.cacheWriteTokens += content.usage.cache_creation_input_tokens ?? 0;
-          state.usage[model] = prev;
-        }
-        // Parse content blocks
-        for (const block of content.content ?? []) {
-          switch (block.type) {
-            case 'text':
-              if (block.text) {
-                messages.push({ type: 'text', content: block.text });
-              }
-              break;
-            case 'thinking':
-              if (block.text) {
-                messages.push({ type: 'thinking', content: block.text });
-              }
-              break;
-            case 'tool_use':
-              messages.push({
-                type: 'tool_use',
-                tool: block.name ?? 'unknown',
-                callId: block.id,
-                input: typeof block.input === 'object' && block.input !== null
-                  ? block.input as Record<string, unknown>
-                  : undefined,
-              });
-              break;
-          }
-        }
-      }
-      break;
-
-    case 'user':
-      if (parsed.message) {
-        const content = parsed.message as ClaudeMessageContent;
-        for (const block of content.content ?? []) {
-          if (block.type === 'tool_result') {
-            const output = block.content != null ? String(block.content) : '';
-            messages.push({
-              type: 'tool_result',
-              callId: block.tool_use_id,
-              output: output.length > 4096 ? output.slice(0, 4096) : output,
-            });
-          }
-        }
-      }
-      break;
-
-    case 'result':
-      if (parsed.session_id) state.sessionId = parsed.session_id;
-      if (parsed.result) state.resultText = parsed.result;
-      if (parsed.is_error) {
-        state.isError = true;
-        messages.push({ type: 'error', content: parsed.result ?? 'Unknown error' });
-      }
-      // Extract real token usage from result.modelUsage (per-model, camelCase)
-      // Note: assistant message usage fields are always 0 during streaming
-      if (parsed.modelUsage) {
-        for (const [model, mu] of Object.entries(parsed.modelUsage)) {
-          state.usage[model] = {
-            inputTokens: mu.inputTokens ?? 0,
-            outputTokens: mu.outputTokens ?? 0,
-            cacheReadTokens: mu.cacheReadInputTokens ?? 0,
-            cacheWriteTokens: mu.cacheCreationInputTokens ?? 0,
-            model,
-          };
-        }
-      }
-      break;
-
-    case 'log':
-      // Skip debug noise — only surface warnings/errors
-      break;
-  }
-
-  return messages;
-}
-
-/**
- * Create a fresh parser state.
- */
-export function createStreamState(): ClaudeStreamState {
-  return { usage: {}, isError: false };
-}
+/**
+ * Claude Code stream-json parser.
+ *
+ * Parses NDJSON from `claude -p --output-format stream-json --verbose`
+ * into AgentMessage events. Based on observed format from Multica reference
+ * and Claude Code documentation.
+ *
+ * Message types:
+ *   - "system"    → session_id, status
+ *   - "assistant" → content blocks (text, thinking, tool_use) + usage
+ *   - "user"      → tool_result content blocks
+ *   - "result"    → final result text, is_error, session_id
+ *   - "log"       → debug/info log entries
+ */
+
+import type { AgentMessage, TokenUsage } from './types.js';
+
+// ── Claude SDK JSON shapes ──────────────────────────────────────
+
+interface ClaudeStreamLine {
+  type: string;
+  message?: unknown;
+  subtype?: string;
+  session_id?: string;
+  result?: string;
+  is_error?: boolean;
+  duration_ms?: number;
+  num_turns?: number;
+  log?: { level: string; message: string };
+  /** Per-model token usage — only present on 'result' messages */
+  modelUsage?: Record<string, {
+    inputTokens?: number;
+    outputTokens?: number;
+    cacheReadInputTokens?: number;
+    cacheCreationInputTokens?: number;
+    costUSD?: number;
+  }>;
+}
+
+interface ClaudeMessageContent {
+  role: string;
+  model?: string;
+  content?: ClaudeContentBlock[];
+  usage?: {
+    input_tokens?: number;
+    output_tokens?: number;
+    cache_read_input_tokens?: number;
+    cache_creation_input_tokens?: number;
+  };
+}
+
+interface ClaudeContentBlock {
+  type: string;
+  text?: string;
+  id?: string;
+  name?: string;
+  input?: unknown;
+  tool_use_id?: string;
+  content?: unknown;
+}
+
+// ── State ───────────────────────────────────────────────────────
+
+export interface ClaudeStreamState {
+  sessionId?: string;
+  usage: Record<string, TokenUsage>;
+  resultText?: string;
+  isError: boolean;
+}
+
+// ── Parser ──────────────────────────────────────────────────────
+
+/**
+ * Parse a single NDJSON line from Claude stream-json output.
+ * Returns zero or more AgentMessage events.
+ */
+export function parseClaudeStreamLine(
+  raw: string,
+  state: ClaudeStreamState,
+): AgentMessage[] {
+  const line = raw.trim();
+  if (!line) return [];
+
+  let parsed: ClaudeStreamLine;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    return [];
+  }
+
+  const messages: AgentMessage[] = [];
+
+  switch (parsed.type) {
+    case 'system':
+      if (parsed.session_id) {
+        state.sessionId = parsed.session_id;
+      }
+      break;
+
+    case 'assistant':
+      if (parsed.message) {
+        const content = parsed.message as ClaudeMessageContent;
+        // Accumulate usage
+        if (content.usage && content.model) {
+          const model = content.model;
+          const prev = state.usage[model] ?? { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, model };
+          prev.inputTokens += content.usage.input_tokens ?? 0;
+          prev.outputTokens += content.usage.output_tokens ?? 0;
+          prev.cacheReadTokens += content.usage.cache_read_input_tokens ?? 0;
+          prev.cacheWriteTokens += content.usage.cache_creation_input_tokens ?? 0;
+          state.usage[model] = prev;
+        }
+        // Parse content blocks
+        for (const block of content.content ?? []) {
+          switch (block.type) {
+            case 'text':
+              if (block.text) {
+                messages.push({ type: 'text', content: block.text });
+              }
+              break;
+            case 'thinking':
+              if (block.text) {
+                messages.push({ type: 'thinking', content: block.text });
+              }
+              break;
+            case 'tool_use':
+              messages.push({
+                type: 'tool_use',
+                tool: block.name ?? 'unknown',
+                callId: block.id,
+                input: typeof block.input === 'object' && block.input !== null
+                  ? block.input as Record<string, unknown>
+                  : undefined,
+              });
+              break;
+          }
+        }
+      }
+      break;
+
+    case 'user':
+      if (parsed.message) {
+        const content = parsed.message as ClaudeMessageContent;
+        for (const block of content.content ?? []) {
+          if (block.type === 'tool_result') {
+            const output = block.content != null ? String(block.content) : '';
+            messages.push({
+              type: 'tool_result',
+              callId: block.tool_use_id,
+              output: output.length > 4096 ? output.slice(0, 4096) : output,
+            });
+          }
+        }
+      }
+      break;
+
+    case 'result':
+      if (parsed.session_id) state.sessionId = parsed.session_id;
+      if (parsed.result) state.resultText = parsed.result;
+      if (parsed.is_error) {
+        state.isError = true;
+        messages.push({ type: 'error', content: parsed.result ?? 'Unknown error' });
+      }
+      // Extract real token usage from result.modelUsage (per-model, camelCase)
+      // Note: assistant message usage fields are always 0 during streaming
+      if (parsed.modelUsage) {
+        for (const [model, mu] of Object.entries(parsed.modelUsage)) {
+          state.usage[model] = {
+            inputTokens: mu.inputTokens ?? 0,
+            outputTokens: mu.outputTokens ?? 0,
+            cacheReadTokens: mu.cacheReadInputTokens ?? 0,
+            cacheWriteTokens: mu.cacheCreationInputTokens ?? 0,
+            model,
+          };
+        }
+      }
+      break;
+
+    case 'log':
+      // Skip debug noise — only surface warnings/errors
+      break;
+  }
+
+  return messages;
+}
+
+/**
+ * Create a fresh parser state.
+ */
+export function createStreamState(): ClaudeStreamState {
+  return { usage: {}, isError: false };
+}