@yeaft/webchat-agent 0.1.398 → 0.1.408

package/unify/config.js

@@ -0,0 +1,256 @@
+/**
+ * config.js — Yeaft configuration management
+ *
+ * Priority (high → low): CLI overrides > ENV vars > .env file > config.md frontmatter > defaults
+ *
+ * Note: "model" in Yeaft always means a model ID (e.g. "gpt-5", "claude-sonnet-4-20250514").
+ * Yeaft does not provide its own models — it routes to external LLM providers via adapters.
+ */
+
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { DEFAULT_YEAFT_DIR } from './init.js';
+import { resolveModel } from './models.js';
+
+/** Default configuration values. */
+const DEFAULTS = {
+  model: 'claude-sonnet-4-20250514',
+  fallbackModel: null,
+  language: 'en', // 'en' | 'zh'
+  apiKey: null,
+  openaiApiKey: null,
+  proxyUrl: 'http://localhost:6628',
+  baseUrl: null,
+  adapter: null, // auto-detect: 'anthropic' | 'openai' | 'proxy'
+  debug: false,
+  dir: DEFAULT_YEAFT_DIR,
+  maxContextTokens: 200000,
+  maxOutputTokens: 16384,
+};
+
+/**
+ * Parse YAML frontmatter from a markdown file.
+ * Simple parser — handles key: value pairs, no nested objects.
+ *
+ * @param {string} content — File content
+ * @returns {Record<string, string>} — Parsed frontmatter
+ */
+export function parseFrontmatter(content) {
+  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (!match) return {};
+
+  const result = {};
+  for (const line of match[1].split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const colonIdx = trimmed.indexOf(':');
+    if (colonIdx === -1) continue;
+    const key = trimmed.slice(0, colonIdx).trim();
+    let value = trimmed.slice(colonIdx + 1).trim();
+    // Parse booleans and numbers
+    if (value === 'true') value = true;
+    else if (value === 'false') value = false;
+    else if (value === 'null') value = null;
+    else if (/^\d+$/.test(value)) value = parseInt(value, 10);
+    result[key] = value;
+  }
+  return result;
+}
+
+/**
+ * Load configuration from config.md file.
+ *
+ * @param {string} dir — Yeaft data directory
+ * @returns {Record<string, unknown>} — Config from file
+ */
+function loadConfigFile(dir) {
+  const configPath = join(dir, 'config.md');
+  if (!existsSync(configPath)) return {};
+
+  try {
+    const content = readFileSync(configPath, 'utf8');
+    return parseFrontmatter(content);
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Load .env file from a directory. Sets process.env for any keys
+ * not already defined (env vars take precedence over .env file).
+ *
+ * ⚠️ Side-effect: mutates process.env globally. Values set by a previous
+ * call persist across subsequent loadConfig() calls within the same process.
+ * This is by design (matches dotenv behavior), but callers that need isolation
+ * (e.g. tests) must manually delete keys from process.env between calls.
+ *
+ * @param {string} dir — Directory containing .env file
+ */
+function loadEnvFile(dir) {
+  const envPath = join(dir, '.env');
+  if (!existsSync(envPath)) return;
+
+  try {
+    const content = readFileSync(envPath, 'utf8');
+    for (const line of content.split('\n')) {
+      let trimmed = line.trim();
+      // Skip empty lines and comments
+      if (!trimmed || trimmed.startsWith('#')) continue;
+      // Strip optional 'export ' prefix (common in .env files)
+      if (trimmed.startsWith('export ')) trimmed = trimmed.slice(7);
+      const eqIdx = trimmed.indexOf('=');
+      if (eqIdx === -1) continue;
+
+      const key = trimmed.slice(0, eqIdx).trim();
+      let value = trimmed.slice(eqIdx + 1).trim();
+      // Remove surrounding quotes if present
+      if ((value.startsWith('"') && value.endsWith('"')) ||
+          (value.startsWith("'") && value.endsWith("'"))) {
+        value = value.slice(1, -1);
+      }
+
+      // Only set if not already defined (shell env takes precedence)
+      if (process.env[key] === undefined) {
+        process.env[key] = value;
+      }
+    }
+  } catch {
+    // Silently ignore .env read errors
+  }
+}
+
+/**
+ * Helper: check if an env var is truthy.
+ * @param {string|undefined} val
+ * @returns {boolean}
+ */
+function isTruthy(val) {
+  return val === '1' || val === 'true' || val === 'yes';
+}
+
+/**
+ * Load full configuration.
+ *
+ * @param {Record<string, unknown>} [overrides] — CLI overrides
+ * @returns {object} — Merged configuration
+ */
+export function loadConfig(overrides = {}) {
+  const env = process.env;
+
+  // Determine data directory first (needed to load .env and config.md)
+  const dir = overrides.dir || env.YEAFT_DIR || DEFAULTS.dir;
+
+  // Load .env file (sets process.env for undefined keys — shell env takes precedence)
+  loadEnvFile(dir);
+
+  // Load from config.md
+  const fileConfig = loadConfigFile(dir);
+
+  // Build merged config: defaults < file < env < overrides
+  const config = {
+    model:
+      overrides.model ||
+      env.YEAFT_MODEL ||
+      fileConfig.model ||
+      DEFAULTS.model,
+
+    fallbackModel:
+      overrides.fallbackModel ||
+      env.YEAFT_FALLBACK_MODEL ||
+      fileConfig.fallbackModel ||
+      DEFAULTS.fallbackModel,
+
+    language:
+      overrides.language ||
+      env.YEAFT_LANGUAGE ||
+      fileConfig.language ||
+      DEFAULTS.language,
+
+    apiKey:
+      overrides.apiKey ||
+      env.YEAFT_API_KEY ||
+      fileConfig.apiKey ||
+      DEFAULTS.apiKey,
+
+    openaiApiKey:
+      overrides.openaiApiKey ||
+      env.YEAFT_OPENAI_API_KEY ||
+      fileConfig.openaiApiKey ||
+      DEFAULTS.openaiApiKey,
+
+    proxyUrl:
+      overrides.proxyUrl ||
+      env.YEAFT_PROXY_URL ||
+      fileConfig.proxyUrl ||
+      DEFAULTS.proxyUrl,
+
+    baseUrl:
+      overrides.baseUrl ||
+      env.YEAFT_BASE_URL ||
+      fileConfig.baseUrl ||
+      DEFAULTS.baseUrl,
+
+    adapter:
+      overrides.adapter ||
+      env.YEAFT_ADAPTER ||
+      fileConfig.adapter ||
+      DEFAULTS.adapter,
+
+    debug:
+      overrides.debug !== undefined
+        ? overrides.debug
+        : env.YEAFT_DEBUG !== undefined
+          ? isTruthy(env.YEAFT_DEBUG)
+          : fileConfig.debug !== undefined
+            ? fileConfig.debug
+            : DEFAULTS.debug,
+
+    dir,
+
+    maxContextTokens:
+      overrides.maxContextTokens ??
+      (env.YEAFT_MAX_CONTEXT ? parseInt(env.YEAFT_MAX_CONTEXT, 10) : null) ??
+      fileConfig.maxContextTokens ??
+      DEFAULTS.maxContextTokens,
+
+    maxOutputTokens:
+      overrides.maxOutputTokens ??
+      fileConfig.maxOutputTokens ??
+      DEFAULTS.maxOutputTokens,
+  };
+
+  // Auto-detect adapter using model registry + credential fallback
+  if (!config.adapter) {
+    const modelInfo = resolveModel(config.model);
+    if (modelInfo) {
+      // Known model → set adapter from registry
+      config.adapter = modelInfo.adapter === 'anthropic' ? 'anthropic' : 'openai';
+      // Use registry baseUrl if not explicitly overridden
+      if (!config.baseUrl) {
+        config.baseUrl = modelInfo.baseUrl;
+      }
+      // Use registry contextWindow if still at default
+      if (config.maxContextTokens === DEFAULTS.maxContextTokens) {
+        config.maxContextTokens = modelInfo.contextWindow;
+      }
+      // Use registry maxOutputTokens if still at default
+      if (config.maxOutputTokens === DEFAULTS.maxOutputTokens) {
+        config.maxOutputTokens = modelInfo.maxOutputTokens;
+      }
+    } else {
+      // Unknown model → fallback to credential-based detection
+      if (config.apiKey) {
+        config.adapter = 'anthropic';
+      } else if (config.openaiApiKey) {
+        config.adapter = 'openai';
+      } else if (config.proxyUrl) {
+        config.adapter = 'proxy';
+      }
+    }
+  }
+
+  // Store resolved model info for reference
+  config.modelInfo = resolveModel(config.model) || null;
+
+  return config;
+}

package/unify/debug-trace.js

@@ -0,0 +1,398 @@
+/**
+ * debug-trace.js — SQLite-backed debug trace for Yeaft
+ *
+ * Records every LLM turn, tool call, and event for debugging and analytics.
+ * When disabled, uses NullTrace (same interface, zero overhead).
+ *
+ * Reference: server/db/connection.js — Database(path), pragma WAL
+ */
+
+import Database from 'better-sqlite3';
+import { randomUUID } from 'crypto';
+import { statSync } from 'fs';
+
+/** Schema DDL — 3 tables + indexes */
+const SCHEMA = `
+  CREATE TABLE IF NOT EXISTS trace_turns (
+    id TEXT PRIMARY KEY,
+    trace_id TEXT NOT NULL,
+    message_id TEXT,
+    mode TEXT,
+    turn_number INTEGER,
+    model TEXT,
+    input_tokens INTEGER,
+    output_tokens INTEGER,
+    cache_read_tokens INTEGER DEFAULT 0,
+    cache_write_tokens INTEGER DEFAULT 0,
+    stop_reason TEXT,
+    latency_ms INTEGER,
+    response_text TEXT,
+    started_at INTEGER NOT NULL,
+    ended_at INTEGER
+  );
+
+  CREATE TABLE IF NOT EXISTS trace_tools (
+    id TEXT PRIMARY KEY,
+    turn_id TEXT NOT NULL,
+    tool_name TEXT NOT NULL,
+    tool_input TEXT,
+    tool_output TEXT,
+    duration_ms INTEGER,
+    is_error INTEGER DEFAULT 0,
+    created_at INTEGER NOT NULL,
+    FOREIGN KEY (turn_id) REFERENCES trace_turns(id)
+  );
+
+  CREATE TABLE IF NOT EXISTS trace_events (
+    id TEXT PRIMARY KEY,
+    trace_id TEXT NOT NULL,
+    event_type TEXT NOT NULL,
+    event_data TEXT,
+    created_at INTEGER NOT NULL
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_turns_trace_id ON trace_turns(trace_id);
+  CREATE INDEX IF NOT EXISTS idx_turns_message_id ON trace_turns(message_id);
+  CREATE INDEX IF NOT EXISTS idx_turns_started_at ON trace_turns(started_at);
+  CREATE INDEX IF NOT EXISTS idx_turns_model ON trace_turns(model);
+  CREATE INDEX IF NOT EXISTS idx_tools_turn_id ON trace_tools(turn_id);
+  CREATE INDEX IF NOT EXISTS idx_tools_name ON trace_tools(tool_name);
+  CREATE INDEX IF NOT EXISTS idx_events_trace_id ON trace_events(trace_id);
+  CREATE INDEX IF NOT EXISTS idx_events_type ON trace_events(event_type);
+`;
+
+/** Max tool_output size stored (10KB). Longer outputs are truncated. */
+const MAX_TOOL_OUTPUT = 10240;
+
+/**
+ * Truncate a string to a max length, appending "... [truncated]" if needed.
+ * @param {string|null|undefined} str
+ * @param {number} max
+ * @returns {string|null}
+ */
+function truncate(str, max) {
+  if (!str) return str ?? null;
+  if (str.length <= max) return str;
+  return str.slice(0, max) + '... [truncated]';
+}
+
+/**
+ * DebugTrace — SQLite-backed debug trace.
+ */
+export class DebugTrace {
+  /** @type {Database.Database} */
+  #db;
+
+  /** @type {string} */
+  #dbPath;
+
+  // Prepared statements (created lazily)
+  #stmts = {};
+
+  /**
+   * @param {string} dbPath — Path to the SQLite database file.
+   */
+  constructor(dbPath) {
+    this.#dbPath = dbPath;
+    this.#db = new Database(dbPath);
+    this.#db.pragma('journal_mode = WAL');
+    this.#db.pragma('foreign_keys = ON');
+    this.#db.exec(SCHEMA);
+  }
+
+  // ─── Write API ───────────────────────────────────────────────
+
+  /**
+   * Start a new turn.
+   * @param {{ traceId: string, messageId?: string, mode?: string, turnNumber?: number }} opts
+   * @returns {string} — turnId
+   */
+  startTurn({ traceId, messageId = null, mode = null, turnNumber = null }) {
+    const id = randomUUID();
+    const now = Date.now();
+    this.#prepare('insertTurn', `
+      INSERT INTO trace_turns (id, trace_id, message_id, mode, turn_number, started_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `).run(id, traceId, messageId, mode, turnNumber, now);
+    return id;
+  }
+
+  /**
+   * End a turn with model response info.
+   * @param {string} turnId
+   * @param {{ model?: string, inputTokens?: number, outputTokens?: number, cacheReadTokens?: number, cacheWriteTokens?: number, stopReason?: string, latencyMs?: number, responseText?: string }} info
+   */
+  endTurn(turnId, {
+    model = null,
+    inputTokens = null,
+    outputTokens = null,
+    cacheReadTokens = 0,
+    cacheWriteTokens = 0,
+    stopReason = null,
+    latencyMs = null,
+    responseText = null,
+  } = {}) {
+    const now = Date.now();
+    this.#prepare('endTurn', `
+      UPDATE trace_turns SET
+        model = ?, input_tokens = ?, output_tokens = ?,
+        cache_read_tokens = ?, cache_write_tokens = ?,
+        stop_reason = ?, latency_ms = ?, response_text = ?, ended_at = ?
+      WHERE id = ?
+    `).run(
+      model, inputTokens, outputTokens,
+      cacheReadTokens, cacheWriteTokens,
+      stopReason, latencyMs, truncate(responseText, MAX_TOOL_OUTPUT),
+      now, turnId,
+    );
+  }
+
+  /**
+   * Log a tool call within a turn.
+   * @param {string} turnId
+   * @param {{ toolName: string, toolInput?: string, toolOutput?: string, durationMs?: number, isError?: boolean }} info
+   * @returns {string} — tool record id
+   */
+  logTool(turnId, {
+    toolName,
+    toolInput = null,
+    toolOutput = null,
+    durationMs = null,
+    isError = false,
+  }) {
+    const id = randomUUID();
+    const now = Date.now();
+    this.#prepare('insertTool', `
+      INSERT INTO trace_tools (id, turn_id, tool_name, tool_input, tool_output, duration_ms, is_error, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      id, turnId, toolName,
+      truncate(toolInput, MAX_TOOL_OUTPUT),
+      truncate(toolOutput, MAX_TOOL_OUTPUT),
+      durationMs, isError ? 1 : 0, now,
+    );
+    return id;
+  }
+
+  /**
+   * Log a freeform event.
+   * @param {{ traceId: string, eventType: string, eventData?: unknown }} info
+   * @returns {string} — event id
+   */
+  logEvent({ traceId, eventType, eventData = null }) {
+    const id = randomUUID();
+    const now = Date.now();
+    const data = eventData != null ? JSON.stringify(eventData) : null;
+    this.#prepare('insertEvent', `
+      INSERT INTO trace_events (id, trace_id, event_type, event_data, created_at)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(id, traceId, eventType, data, now);
+    return id;
+  }
+
+  // ─── Read API ────────────────────────────────────────────────
+
+  /**
+   * Query all data for a specific message.
+   * @param {string} messageId
+   * @returns {{ turns: object[], tools: object[], events: object[] }}
+   */
+  queryByMessage(messageId) {
+    const turns = this.#prepare('turnsByMessage', `
+      SELECT * FROM trace_turns WHERE message_id = ? ORDER BY started_at
+    `).all(messageId);
+    return this.#expandTurns(turns);
+  }
+
+  /**
+   * Query all data for a trace.
+   * @param {string} traceId
+   * @returns {{ turns: object[], tools: object[], events: object[] }}
+   */
+  queryByTrace(traceId) {
+    const turns = this.#prepare('turnsByTrace', `
+      SELECT * FROM trace_turns WHERE trace_id = ? ORDER BY started_at
+    `).all(traceId);
+    const events = this.#prepare('eventsByTrace', `
+      SELECT * FROM trace_events WHERE trace_id = ? ORDER BY created_at
+    `).all(traceId);
+    const turnIds = turns.map(t => t.id);
+    const tools = turnIds.length > 0
+      ? this.#db.prepare(
+          `SELECT * FROM trace_tools WHERE turn_id IN (${turnIds.map(() => '?').join(',')}) ORDER BY created_at`
+        ).all(...turnIds)
+      : [];
+    return { turns, tools, events };
+  }
+
+  /**
+   * Query recent turns.
+   * @param {number} [limit=20]
+   * @returns {object[]}
+   */
+  queryRecent(limit = 20) {
+    return this.#prepare('recentTurns', `
+      SELECT * FROM trace_turns ORDER BY started_at DESC LIMIT ?
+    `).all(limit);
+  }
+
+  /**
+   * Query tool calls with optional filters.
+   * @param {{ name?: string, since?: number }} [filters={}]
+   * @returns {object[]}
+   */
+  queryTools({ name = null, since = null } = {}) {
+    if (name && since) {
+      return this.#prepare('toolsByNameSince', `
+        SELECT * FROM trace_tools WHERE tool_name = ? AND created_at >= ? ORDER BY created_at DESC
+      `).all(name, since);
+    }
+    if (name) {
+      return this.#prepare('toolsByName', `
+        SELECT * FROM trace_tools WHERE tool_name = ? ORDER BY created_at DESC
+      `).all(name);
+    }
+    if (since) {
+      return this.#prepare('toolsSince', `
+        SELECT * FROM trace_tools WHERE created_at >= ? ORDER BY created_at DESC
+      `).all(since);
+    }
+    return this.#prepare('allTools', `
+      SELECT * FROM trace_tools ORDER BY created_at DESC LIMIT 100
+    `).all();
+  }
+
+  /**
+   * Full-text search across response_text and tool_output.
+   * @param {string} keyword
+   * @returns {object[]}
+   */
+  search(keyword) {
+    const like = `%${keyword}%`;
+    return this.#prepare('search', `
+      SELECT DISTINCT t.* FROM trace_turns t
+      LEFT JOIN trace_tools tt ON tt.turn_id = t.id
+      WHERE t.response_text LIKE ? OR tt.tool_output LIKE ?
+      ORDER BY t.started_at DESC LIMIT 50
+    `).all(like, like);
+  }
+
+  /**
+   * Get trace statistics.
+   * @returns {{ turnCount: number, toolCount: number, eventCount: number, dbSizeBytes: number }}
+   */
+  stats() {
+    const turnCount = this.#db.prepare('SELECT COUNT(*) as c FROM trace_turns').get().c;
+    const toolCount = this.#db.prepare('SELECT COUNT(*) as c FROM trace_tools').get().c;
+    const eventCount = this.#db.prepare('SELECT COUNT(*) as c FROM trace_events').get().c;
+    let dbSizeBytes = 0;
+    try {
+      dbSizeBytes = statSync(this.#dbPath).size;
+    } catch { /* ignore */ }
+    return { turnCount, toolCount, eventCount, dbSizeBytes };
+  }
+
+  // ─── Maintenance ─────────────────────────────────────────────
+
+  /**
+   * Delete data older than retentionDays.
+   * @param {number} [retentionDays=30]
+   * @returns {{ deletedTurns: number, deletedTools: number, deletedEvents: number }}
+   */
+  cleanup(retentionDays = 30) {
+    const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+    const deletedTools = this.#db.prepare(`
+      DELETE FROM trace_tools WHERE turn_id IN (
+        SELECT id FROM trace_turns WHERE started_at < ?
+      )
+    `).run(cutoff).changes;
+    const deletedTurns = this.#db.prepare(`
+      DELETE FROM trace_turns WHERE started_at < ?
+    `).run(cutoff).changes;
+    const deletedEvents = this.#db.prepare(`
+      DELETE FROM trace_events WHERE created_at < ?
+    `).run(cutoff).changes;
+    return { deletedTurns, deletedTools, deletedEvents };
+  }
+
+  /** Delete all trace data. */
+  purge() {
+    this.#db.exec('DELETE FROM trace_tools');
+    this.#db.exec('DELETE FROM trace_turns');
+    this.#db.exec('DELETE FROM trace_events');
+  }
+
+  /** Close the database connection. */
+  close() {
+    this.#db.close();
+  }
+
+  // ─── Internal ────────────────────────────────────────────────
+
+  /**
+   * Get or create a prepared statement.
+   * @param {string} key
+   * @param {string} sql
+   * @returns {Database.Statement}
+   */
+  #prepare(key, sql) {
+    if (!this.#stmts[key]) {
+      this.#stmts[key] = this.#db.prepare(sql);
+    }
+    return this.#stmts[key];
+  }
+
+  /**
+   * Expand turns with their tools.
+   * @param {object[]} turns
+   * @returns {{ turns: object[], tools: object[], events: object[] }}
+   */
+  #expandTurns(turns) {
+    const turnIds = turns.map(t => t.id);
+    const tools = turnIds.length > 0
+      ? this.#db.prepare(
+          `SELECT * FROM trace_tools WHERE turn_id IN (${turnIds.map(() => '?').join(',')}) ORDER BY created_at`
+        ).all(...turnIds)
+      : [];
+    // Events need trace_ids from turns
+    const traceIds = [...new Set(turns.map(t => t.trace_id))];
+    const events = traceIds.length > 0
+      ? this.#db.prepare(
+          `SELECT * FROM trace_events WHERE trace_id IN (${traceIds.map(() => '?').join(',')}) ORDER BY created_at`
+        ).all(...traceIds)
+      : [];
+    return { turns, tools, events };
+  }
+}
+
+/**
+ * NullTrace — No-op implementation with the same interface.
+ * Used when debug is disabled. Zero overhead.
+ */
+export class NullTrace {
+  startTurn() { return 'null'; }
+  endTurn() {}
+  logTool() { return 'null'; }
+  logEvent() { return 'null'; }
+  queryByMessage() { return { turns: [], tools: [], events: [] }; }
+  queryByTrace() { return { turns: [], tools: [], events: [] }; }
+  queryRecent() { return []; }
+  queryTools() { return []; }
+  search() { return []; }
+  stats() { return { turnCount: 0, toolCount: 0, eventCount: 0, dbSizeBytes: 0 }; }
+  cleanup() { return { deletedTurns: 0, deletedTools: 0, deletedEvents: 0 }; }
+  purge() {}
+  close() {}
+}
+
+/**
+ * Create a DebugTrace or NullTrace based on config.
+ * @param {{ enabled: boolean, dbPath?: string }} opts
+ * @returns {DebugTrace | NullTrace}
+ */
+export function createTrace({ enabled, dbPath }) {
+  if (!enabled || !dbPath) {
+    return new NullTrace();
+  }
+  return new DebugTrace(dbPath);
+}