create-walle 0.9.21 → 0.9.23

package/template/wall-e/eval/replay.js

@@ -1,395 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-const { classifyTaskType } = require('./harvester');
-const { getShadowConfig } = require('./shadow');
-const { createClient } = require('../llm/client');
-
-const INTER_CALL_DELAY_MS = 1000;
-const CALL_TIMEOUT_MS = 600000; // 10 min safety ceiling — no real timeout for training; local models can take minutes on complex prompts
-const MAX_HISTORY_CHARS = 32000; // Cap conversation history sent to shadow model
-
-// --- Shared helpers ---
-
-function openCtmDb() {
-  const dataDir = process.env.WALL_E_DATA_DIR || path.join(process.env.HOME, '.walle', 'data');
-  const ctmDbPath = path.join(dataDir, 'task-manager.db');
-  if (!fs.existsSync(ctmDbPath)) return null;
-  const Database = require('better-sqlite3');
-  try {
-    return new Database(ctmDbPath, { readonly: true, fileMustExist: true });
-  } catch {
-    return null;
-  }
-}
-
-function makeShadowClient(models) {
-  const shadowModel = models[0];
-  const client = createClient(shadowModel.provider || 'ollama', {
-    apiKey: shadowModel.apiKey,
-    baseUrl: shadowModel.baseUrl,
-  });
-  return { client, shadowModel };
-}
-
-function extractShadowContent(response) {
-  return typeof response.content === 'string'
-    ? response.content : (response.text || '');
-}
-
-/**
- * Build existing shadow prompt prefixes for dedup.
- * Uses first 100 chars of prompt as key — cheaper than full hash and
- * catches duplicates across sources (live shadow, CTM replay, coding agent).
- */
-function loadExistingPromptPrefixes(brain) {
-  return new Set(
-    brain.getShadowResults({ limit: 10000 })
-      .filter(r => r.prompt)
-      .map(r => r.prompt.slice(0, 100))
-  );
-}
-
-// --- Single-turn replay (simple prompt/response pairs) ---
-
-/**
- * Replay single-turn CTM prompt/response pairs through shadow models.
- * Best for: Q&A, short coding questions, chat.
- *
- * Rate-limited: max `limit` replays per run, 2s between calls.
- */
-async function runReplay(brain, options = {}) {
-  const { limit = 10 } = options;
-
-  const shadowConfig = getShadowConfig();
-  if (!shadowConfig.enabled) return { replayed: 0 };
-  const models = shadowConfig.models || [];
-  if (!models.length) return { replayed: 0 };
-
-  const state = brain.getHarvestState('ctm-replay');
-  const lastReplayAt = state?.last_processed_at || '2000-01-01';
-
-  const ctmDb = openCtmDb();
-  if (!ctmDb) return { replayed: 0 };
-
-  let rows;
-  try {
-    // Only pick single-turn sessions (sessions with exactly 1 user/assistant pair)
-    // Multi-turn sessions are handled by runMultiTurnReplay
-    rows = ctmDb.prepare(`
-      SELECT u.session_id, u.message_text AS prompt, u.executed_at,
-             a.message_text AS response
-      FROM prompt_executions u
-      JOIN prompt_executions a ON a.session_id = u.session_id
-        AND a.message_index = u.message_index + 1
-      WHERE u.role = 'user' AND a.role = 'assistant'
-        AND length(u.message_text) >= 20 AND length(a.message_text) >= 20
-        AND u.executed_at > ?
-        AND u.session_id IN (
-          SELECT session_id FROM prompt_executions
-          WHERE role = 'user'
-          GROUP BY session_id HAVING COUNT(*) = 1
-        )
-      ORDER BY u.executed_at ASC
-      LIMIT ?
-    `).all(lastReplayAt, limit);
-  } catch (err) {
-    try { ctmDb.close(); } catch {}
-    console.error('[replay] Error querying CTM DB:', err.message);
-    return { replayed: 0 };
-  }
-  ctmDb.close();
-
-  if (!rows.length) return { replayed: 0 };
-
-  const existing = loadExistingPromptPrefixes(brain);
-  const { client, shadowModel } = makeShadowClient(models);
-
-  let replayed = 0;
-  let lastTimestamp = lastReplayAt;
-
-  for (const row of rows) {
-    if (existing.has(row.prompt.slice(0, 100))) continue;
-
-    try {
-      const start = Date.now();
-      const controller = new AbortController();
-      const timeout = setTimeout(() => controller.abort(), CALL_TIMEOUT_MS);
-
-      const response = await client.chat({
-        model: shadowModel.model,
-        messages: [{ role: 'user', content: row.prompt }],
-        maxTokens: 4096,
-        signal: controller.signal,
-      });
-      clearTimeout(timeout);
-
-      const usage = response.usage || {};
-      const inputTokens = usage.input || usage.inputTokens || usage.input_tokens || 0;
-      const outputTokens = usage.output || usage.outputTokens || usage.output_tokens || 0;
-
-      brain.insertShadowResult({
-        sessionId: row.session_id,
-        taskType: classifyTaskType(row.prompt),
-        prompt: row.prompt,
-        primaryResponse: row.response,
-        primaryModel: 'claude',
-        primaryLatencyMs: null,
-        shadowResponse: extractShadowContent(response),
-        shadowModel: shadowModel.model,
-        shadowLatencyMs: Date.now() - start,
-        inputTokens,
-        outputTokens,
-        genTokPerSec: usage.genTokPerSec || null,
-      });
-
-      replayed++;
-      lastTimestamp = row.executed_at;
-      existing.add(row.prompt.slice(0, 100));
-
-      await new Promise(r => setTimeout(r, INTER_CALL_DELAY_MS));
-    } catch (err) {
-      console.error(`[replay] Error replaying: ${err.message}`);
-    }
-  }
-
-  if (replayed > 0) {
-    brain.updateHarvestState('ctm-replay', {
-      lastProcessedAt: lastTimestamp,
-      totalHarvested: replayed,
-    });
-  }
-
-  return { replayed, total: rows.length };
-}
-
-// --- Multi-turn replay (full conversation context) ---
-
-/**
- * Replay multi-turn CTM coding sessions through shadow models.
- *
- * For each session with 2+ user turns:
- * 1. Load all messages in order
- * 2. Build progressive conversation history
- * 3. At each assistant turn, send full history to shadow model
- * 4. Compare shadow response against Claude's actual response
- *
- * The shadow model sees the same context Claude had — prior user messages
- * AND Claude's actual responses (including tool call descriptions) — so
- * it's a fair comparison of what each model would produce given identical context.
- *
- * Rate-limited: max `sessionLimit` sessions per run, max `turnsPerSession`
- * turns replayed per session, 2s between calls.
- */
-async function runMultiTurnReplay(brain, options = {}) {
-  const { sessionLimit = 3, turnsPerSession = 5 } = options;
-
-  const shadowConfig = getShadowConfig();
-  if (!shadowConfig.enabled) return { replayed: 0, sessions: 0 };
-  const models = shadowConfig.models || [];
-  if (!models.length) return { replayed: 0, sessions: 0 };
-
-  const state = brain.getHarvestState('ctm-replay-mt');
-  const lastReplayAt = state?.last_processed_at || '2000-01-01';
-
-  const ctmDb = openCtmDb();
-  if (!ctmDb) return { replayed: 0, sessions: 0 };
-
-  let sessions;
-  try {
-    // Find multi-turn sessions (2+ user messages) newer than watermark
-    sessions = ctmDb.prepare(`
-      SELECT session_id, MIN(executed_at) AS started_at, COUNT(*) AS msg_count
-      FROM prompt_executions
-      WHERE executed_at > ?
-      GROUP BY session_id
-      HAVING SUM(CASE WHEN role = 'user' THEN 1 ELSE 0 END) >= 2
-      ORDER BY MIN(executed_at) ASC
-      LIMIT ?
-    `).all(lastReplayAt, sessionLimit);
-  } catch (err) {
-    try { ctmDb.close(); } catch {}
-    console.error('[replay-mt] Error finding sessions:', err.message);
-    return { replayed: 0, sessions: 0 };
-  }
-
-  if (!sessions.length) {
-    ctmDb.close();
-    return { replayed: 0, sessions: 0 };
-  }
-
-  // Pre-load all messages for selected sessions
-  const sessionMessages = new Map();
-  try {
-    for (const s of sessions) {
-      const msgs = ctmDb.prepare(`
-        SELECT message_text, role, message_index, tool_uses, executed_at
-        FROM prompt_executions
-        WHERE session_id = ?
-        ORDER BY message_index ASC
-      `).all(s.session_id);
-      sessionMessages.set(s.session_id, msgs);
-    }
-  } catch (err) {
-    try { ctmDb.close(); } catch {}
-    console.error('[replay-mt] Error loading messages:', err.message);
-    return { replayed: 0, sessions: 0 };
-  }
-  ctmDb.close();
-
-  const existing = loadExistingPromptPrefixes(brain);
-  const { client, shadowModel } = makeShadowClient(models);
-
-  let totalReplayed = 0;
-  let sessionsProcessed = 0;
-  let lastTimestamp = lastReplayAt;
-
-  for (const session of sessions) {
-    const msgs = sessionMessages.get(session.session_id);
-    if (!msgs || msgs.length < 4) continue; // need at least 2 user + 2 assistant
-
-    // Build progressive history and replay each assistant turn
-    const history = [];
-    let turnsReplayed = 0;
-
-    for (const msg of msgs) {
-      if (msg.role === 'user') {
-        history.push({ role: 'user', content: msg.message_text });
-        continue;
-      }
-
-      if (msg.role !== 'assistant') continue;
-      if (!history.length || history[history.length - 1].role !== 'user') {
-        // No preceding user message — just add to history and move on
-        history.push({ role: 'assistant', content: msg.message_text });
-        continue;
-      }
-
-      if (turnsReplayed >= turnsPerSession) {
-        // Still add to history for context but don't replay
-        history.push({ role: 'assistant', content: msg.message_text });
-        continue;
-      }
-
-      const userPrompt = history[history.length - 1].content;
-      if (!userPrompt || userPrompt.length < 20) {
-        history.push({ role: 'assistant', content: msg.message_text });
-        continue;
-      }
-      if (!msg.message_text || msg.message_text.length < 20) {
-        history.push({ role: 'assistant', content: msg.message_text });
-        continue;
-      }
-
-      // Dedup: use prompt prefix (multi-turn prompts at different depths
-      // are distinct enough that prefix matching works)
-      const dedupKey = userPrompt.slice(0, 100);
-      if (existing.has(dedupKey)) {
-        history.push({ role: 'assistant', content: msg.message_text });
-        continue;
-      }
-
-      // Build context-aware messages for the shadow model
-      // Trim early history if total exceeds cap
-      const contextMessages = trimHistory(history, MAX_HISTORY_CHARS);
-
-      try {
-        const start = Date.now();
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), CALL_TIMEOUT_MS);
-
-        const response = await client.chat({
-          model: shadowModel.model,
-          messages: contextMessages,
-          maxTokens: 4096,
-          signal: controller.signal,
-        });
-        clearTimeout(timeout);
-
-        // Classify based on first user message (sets session context)
-        const firstUserMsg = history.find(m => m.role === 'user');
-        const taskType = classifyTaskType(firstUserMsg?.content || userPrompt);
-
-        const usage = response.usage || {};
-        const inputTokens = usage.input || usage.inputTokens || usage.input_tokens || 0;
-        const outputTokens = usage.output || usage.outputTokens || usage.output_tokens || 0;
-
-        brain.insertShadowResult({
-          sessionId: session.session_id,
-          taskType,
-          prompt: userPrompt,
-          primaryResponse: msg.message_text,
-          primaryModel: 'claude',
-          primaryLatencyMs: null,
-          shadowResponse: extractShadowContent(response),
-          shadowModel: shadowModel.model,
-          shadowLatencyMs: Date.now() - start,
-          inputTokens,
-          outputTokens,
-          genTokPerSec: usage.genTokPerSec || null,
-        });
-
-        turnsReplayed++;
-        totalReplayed++;
-        existing.add(dedupKey);
-        lastTimestamp = session.started_at;
-
-        await new Promise(r => setTimeout(r, INTER_CALL_DELAY_MS));
-      } catch (err) {
-        console.error(`[replay-mt] Error replaying turn: ${err.message}`);
-      }
-
-      // Add actual Claude response to history for subsequent turns
-      history.push({ role: 'assistant', content: msg.message_text });
-    }
-
-    if (turnsReplayed > 0) sessionsProcessed++;
-  }
-
-  if (totalReplayed > 0) {
-    brain.updateHarvestState('ctm-replay-mt', {
-      lastProcessedAt: lastTimestamp,
-      totalHarvested: totalReplayed,
-    });
-  }
-
-  return { replayed: totalReplayed, sessions: sessionsProcessed };
-}
-
-/**
- * Trim conversation history to fit within a character budget.
- * Keeps the system structure intact: always preserves the first user message
- * (establishes task context) and the most recent messages.
- */
-function trimHistory(messages, maxChars) {
-  const totalChars = messages.reduce((sum, m) => sum + (m.content?.length || 0), 0);
-  if (totalChars <= maxChars) return [...messages];
-
-  // Always keep first message (task context) and last messages
-  if (messages.length <= 3) return [...messages];
-
-  const first = messages[0];
-  const result = [first];
-  let budget = maxChars - (first.content?.length || 0);
-
-  // Walk backwards from the end, adding messages until budget exhausted
-  const tail = [];
-  for (let i = messages.length - 1; i >= 1; i--) {
-    const len = messages[i].content?.length || 0;
-    if (budget - len < 0 && tail.length >= 2) break; // keep at least last 2
-    budget -= len;
-    tail.unshift(messages[i]);
-  }
-
-  // If we skipped middle messages, add a marker
-  if (tail.length < messages.length - 1) {
-    const skipped = messages.length - 1 - tail.length;
-    result.push({ role: 'user', content: `[... ${skipped} earlier messages omitted for context length ...]` });
-  }
-
-  result.push(...tail);
-  return result;
-}
-
-module.exports = { runReplay, runMultiTurnReplay, trimHistory };