@yeaft/webchat-agent 0.1.399 → 0.1.409

package/unify/config.js

@@ -0,0 +1,269 @@
+/**
+ * config.js — Yeaft configuration management
+ *
+ * Priority (high → low): CLI overrides > ENV vars > .env file > config.md frontmatter > defaults
+ *
+ * Note: "model" in Yeaft always means a model ID (e.g. "gpt-5", "claude-sonnet-4-20250514").
+ * Yeaft does not provide its own models — it routes to external LLM providers via adapters.
+ */
+
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { DEFAULT_YEAFT_DIR } from './init.js';
+import { resolveModel } from './models.js';
+
+/** Default configuration values. */
+const DEFAULTS = {
+  model: 'claude-sonnet-4-20250514',
+  fallbackModel: null,
+  language: 'en', // 'en' | 'zh'
+  apiKey: null,
+  openaiApiKey: null,
+  proxyUrl: 'http://localhost:6628',
+  baseUrl: null,
+  adapter: null, // auto-detect: 'anthropic' | 'openai' | 'proxy'
+  debug: false,
+  dir: DEFAULT_YEAFT_DIR,
+  maxContextTokens: 200000,
+  maxOutputTokens: 16384,
+  messageTokenBudget: 8192, // Phase 2: context * 4%, triggers consolidation
+  maxContinueTurns: 3,      // Phase 2: auto-continue on max_tokens
+};
+
+/**
+ * Parse YAML frontmatter from a markdown file.
+ * Simple parser — handles key: value pairs, no nested objects.
+ *
+ * @param {string} content — File content
+ * @returns {Record<string, string>} — Parsed frontmatter
+ */
+export function parseFrontmatter(content) {
+  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (!match) return {};
+
+  const result = {};
+  for (const line of match[1].split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const colonIdx = trimmed.indexOf(':');
+    if (colonIdx === -1) continue;
+    const key = trimmed.slice(0, colonIdx).trim();
+    let value = trimmed.slice(colonIdx + 1).trim();
+    // Parse booleans and numbers
+    if (value === 'true') value = true;
+    else if (value === 'false') value = false;
+    else if (value === 'null') value = null;
+    else if (/^\d+$/.test(value)) value = parseInt(value, 10);
+    result[key] = value;
+  }
+  return result;
+}
+
+/**
+ * Load configuration from config.md file.
+ *
+ * @param {string} dir — Yeaft data directory
+ * @returns {Record<string, unknown>} — Config from file
+ */
+function loadConfigFile(dir) {
+  const configPath = join(dir, 'config.md');
+  if (!existsSync(configPath)) return {};
+
+  try {
+    const content = readFileSync(configPath, 'utf8');
+    return parseFrontmatter(content);
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Load .env file from a directory. Sets process.env for any keys
+ * not already defined (env vars take precedence over .env file).
+ *
+ * ⚠️ Side-effect: mutates process.env globally. Values set by a previous
+ * call persist across subsequent loadConfig() calls within the same process.
+ * This is by design (matches dotenv behavior), but callers that need isolation
+ * (e.g. tests) must manually delete keys from process.env between calls.
+ *
+ * @param {string} dir — Directory containing .env file
+ */
+function loadEnvFile(dir) {
+  const envPath = join(dir, '.env');
+  if (!existsSync(envPath)) return;
+
+  try {
+    const content = readFileSync(envPath, 'utf8');
+    for (const line of content.split('\n')) {
+      let trimmed = line.trim();
+      // Skip empty lines and comments
+      if (!trimmed || trimmed.startsWith('#')) continue;
+      // Strip optional 'export ' prefix (common in .env files)
+      if (trimmed.startsWith('export ')) trimmed = trimmed.slice(7);
+      const eqIdx = trimmed.indexOf('=');
+      if (eqIdx === -1) continue;
+
+      const key = trimmed.slice(0, eqIdx).trim();
+      let value = trimmed.slice(eqIdx + 1).trim();
+      // Remove surrounding quotes if present
+      if ((value.startsWith('"') && value.endsWith('"')) ||
+          (value.startsWith("'") && value.endsWith("'"))) {
+        value = value.slice(1, -1);
+      }
+
+      // Only set if not already defined (shell env takes precedence)
+      if (process.env[key] === undefined) {
+        process.env[key] = value;
+      }
+    }
+  } catch {
+    // Silently ignore .env read errors
+  }
+}
+
+/**
+ * Helper: check if an env var is truthy.
+ * @param {string|undefined} val
+ * @returns {boolean}
+ */
+function isTruthy(val) {
+  return val === '1' || val === 'true' || val === 'yes';
+}
+
+/**
+ * Load full configuration.
+ *
+ * @param {Record<string, unknown>} [overrides] — CLI overrides
+ * @returns {object} — Merged configuration
+ */
+export function loadConfig(overrides = {}) {
+  const env = process.env;
+
+  // Determine data directory first (needed to load .env and config.md)
+  const dir = overrides.dir || env.YEAFT_DIR || DEFAULTS.dir;
+
+  // Load .env file (sets process.env for undefined keys — shell env takes precedence)
+  loadEnvFile(dir);
+
+  // Load from config.md
+  const fileConfig = loadConfigFile(dir);
+
+  // Build merged config: defaults < file < env < overrides
+  const config = {
+    model:
+      overrides.model ||
+      env.YEAFT_MODEL ||
+      fileConfig.model ||
+      DEFAULTS.model,
+
+    fallbackModel:
+      overrides.fallbackModel ||
+      env.YEAFT_FALLBACK_MODEL ||
+      fileConfig.fallbackModel ||
+      DEFAULTS.fallbackModel,
+
+    language:
+      overrides.language ||
+      env.YEAFT_LANGUAGE ||
+      fileConfig.language ||
+      DEFAULTS.language,
+
+    apiKey:
+      overrides.apiKey ||
+      env.YEAFT_API_KEY ||
+      fileConfig.apiKey ||
+      DEFAULTS.apiKey,
+
+    openaiApiKey:
+      overrides.openaiApiKey ||
+      env.YEAFT_OPENAI_API_KEY ||
+      fileConfig.openaiApiKey ||
+      DEFAULTS.openaiApiKey,
+
+    proxyUrl:
+      overrides.proxyUrl ||
+      env.YEAFT_PROXY_URL ||
+      fileConfig.proxyUrl ||
+      DEFAULTS.proxyUrl,
+
+    baseUrl:
+      overrides.baseUrl ||
+      env.YEAFT_BASE_URL ||
+      fileConfig.baseUrl ||
+      DEFAULTS.baseUrl,
+
+    adapter:
+      overrides.adapter ||
+      env.YEAFT_ADAPTER ||
+      fileConfig.adapter ||
+      DEFAULTS.adapter,
+
+    debug:
+      overrides.debug !== undefined
+        ? overrides.debug
+        : env.YEAFT_DEBUG !== undefined
+          ? isTruthy(env.YEAFT_DEBUG)
+          : fileConfig.debug !== undefined
+            ? fileConfig.debug
+            : DEFAULTS.debug,
+
+    dir,
+
+    maxContextTokens:
+      overrides.maxContextTokens ??
+      (env.YEAFT_MAX_CONTEXT ? parseInt(env.YEAFT_MAX_CONTEXT, 10) : null) ??
+      fileConfig.maxContextTokens ??
+      DEFAULTS.maxContextTokens,
+
+    maxOutputTokens:
+      overrides.maxOutputTokens ??
+      fileConfig.maxOutputTokens ??
+      DEFAULTS.maxOutputTokens,
+
+    messageTokenBudget:
+      overrides.messageTokenBudget ??
+      (env.YEAFT_MESSAGE_TOKEN_BUDGET ? parseInt(env.YEAFT_MESSAGE_TOKEN_BUDGET, 10) : null) ??
+      fileConfig.messageTokenBudget ??
+      DEFAULTS.messageTokenBudget,
+
+    maxContinueTurns:
+      overrides.maxContinueTurns ??
+      fileConfig.maxContinueTurns ??
+      DEFAULTS.maxContinueTurns,
+  };
+
+  // Auto-detect adapter using model registry + credential fallback
+  if (!config.adapter) {
+    const modelInfo = resolveModel(config.model);
+    if (modelInfo) {
+      // Known model → set adapter from registry
+      config.adapter = modelInfo.adapter === 'anthropic' ? 'anthropic' : 'openai';
+      // Use registry baseUrl if not explicitly overridden
+      if (!config.baseUrl) {
+        config.baseUrl = modelInfo.baseUrl;
+      }
+      // Use registry contextWindow if still at default
+      if (config.maxContextTokens === DEFAULTS.maxContextTokens) {
+        config.maxContextTokens = modelInfo.contextWindow;
+      }
+      // Use registry maxOutputTokens if still at default
+      if (config.maxOutputTokens === DEFAULTS.maxOutputTokens) {
+        config.maxOutputTokens = modelInfo.maxOutputTokens;
+      }
+    } else {
+      // Unknown model → fallback to credential-based detection
+      if (config.apiKey) {
+        config.adapter = 'anthropic';
+      } else if (config.openaiApiKey) {
+        config.adapter = 'openai';
+      } else if (config.proxyUrl) {
+        config.adapter = 'proxy';
+      }
+    }
+  }
+
+  // Store resolved model info for reference
+  config.modelInfo = resolveModel(config.model) || null;
+
+  return config;
+}

package/unify/conversation/persist.js

@@ -0,0 +1,436 @@
+/**
+ * persist.js — Conversation message persistence
+ *
+ * Each message is stored as a .md file with YAML frontmatter in
+ * ~/.yeaft/conversation/messages/. Design: zero JSON, all Markdown.
+ *
+ * Message format:
+ *   ---
+ *   id: m0355
+ *   role: user
+ *   time: 2026-04-09T14:35:00Z
+ *   mode: chat
+ *   model: claude-sonnet-4-20250514
+ *   tokens_est: 230
+ *   ---
+ *   Message content here...
+ *
+ * Reference: yeaft-unify-core-systems.md §4.1, yeaft-unify-brainstorm-v5.1.md
+ */
+
+import { existsSync, mkdirSync, writeFileSync, readFileSync, readdirSync, renameSync, unlinkSync } from 'fs';
+import { join, basename } from 'path';
+
+// ─── Token estimation ────────────────────────────────────────
+
+/** Rough token estimation: ~4 chars per token. */
+export function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+// ─── Frontmatter helpers ─────────────────────────────────────
+
+/**
+ * Serialize message metadata to YAML frontmatter + body.
+ * @param {object} msg
+ * @returns {string}
+ */
+function serializeMessage(msg) {
+  const fm = [
+    '---',
+    `id: ${msg.id}`,
+    `role: ${msg.role}`,
+    `time: ${msg.time || new Date().toISOString()}`,
+  ];
+
+  if (msg.mode) fm.push(`mode: ${msg.mode}`);
+  if (msg.model) fm.push(`model: ${msg.model}`);
+  if (msg.turnNumber != null) fm.push(`turnNumber: ${msg.turnNumber}`);
+  if (msg.toolCallId) fm.push(`toolCallId: ${msg.toolCallId}`);
+  if (msg.isError) fm.push(`isError: true`);
+
+  // Token estimate
+  const content = msg.content || '';
+  const tokensEst = msg.tokens_est || estimateTokens(content);
+  fm.push(`tokens_est: ${tokensEst}`);
+
+  // Tool calls as YAML array (simplified)
+  if (msg.toolCalls && msg.toolCalls.length > 0) {
+    fm.push(`toolCalls:`);
+    for (const tc of msg.toolCalls) {
+      fm.push(`  - id: ${tc.id}`);
+      fm.push(`    name: ${tc.name}`);
+    }
+  }
+
+  fm.push('---');
+  fm.push('');
+  fm.push(content);
+
+  return fm.join('\n');
+}
+
+/**
+ * Parse a message .md file into a message object.
+ * @param {string} raw — Raw file content
+ * @returns {object|null}
+ */
+export function parseMessage(raw) {
+  if (!raw || !raw.startsWith('---')) return null;
+
+  const endIdx = raw.indexOf('\n---', 3);
+  if (endIdx === -1) return null;
+
+  const frontmatter = raw.slice(4, endIdx).trim();
+  const body = raw.slice(endIdx + 4).trim();
+
+  const msg = { content: body };
+
+  for (const line of frontmatter.split('\n')) {
+    const colonIdx = line.indexOf(':');
+    if (colonIdx === -1) continue;
+
+    const key = line.slice(0, colonIdx).trim();
+    const value = line.slice(colonIdx + 1).trim();
+
+    switch (key) {
+      case 'id': msg.id = value; break;
+      case 'role': msg.role = value; break;
+      case 'time': msg.time = value; break;
+      case 'mode': msg.mode = value; break;
+      case 'model': msg.model = value; break;
+      case 'turnNumber': msg.turnNumber = parseInt(value, 10); break;
+      case 'toolCallId': msg.toolCallId = value; break;
+      case 'isError': msg.isError = value === 'true'; break;
+      case 'tokens_est': msg.tokens_est = parseInt(value, 10); break;
+      // toolCalls are multi-line YAML — handled separately below
+    }
+  }
+
+  // Parse toolCalls if present (simplified multi-line YAML)
+  if (frontmatter.includes('toolCalls:')) {
+    const toolCalls = [];
+    const tcMatch = frontmatter.match(/toolCalls:\n((?:\s+-\s+[\s\S]*?)(?=\n\w|$))/);
+    if (tcMatch) {
+      const tcBlock = tcMatch[1];
+      const entries = tcBlock.split(/\n\s+-\s+/).filter(Boolean);
+      for (const entry of entries) {
+        const tc = {};
+        for (const line of entry.split('\n')) {
+          const trimmed = line.trim();
+          const ci = trimmed.indexOf(':');
+          if (ci === -1) continue;
+          const k = trimmed.slice(0, ci).trim();
+          const v = trimmed.slice(ci + 1).trim();
+          if (k === 'id') tc.id = v;
+          if (k === 'name') tc.name = v;
+        }
+        if (tc.id && tc.name) toolCalls.push(tc);
+      }
+    }
+    if (toolCalls.length > 0) msg.toolCalls = toolCalls;
+  }
+
+  return msg;
+}
+
+// ─── ConversationStore ───────────────────────────────────────
+
+/**
+ * ConversationStore — persist and load messages to/from disk.
+ *
+ * Directory layout:
+ *   conversation/
+ *     index.md       — message index with frontmatter
+ *     compact.md     — cumulative compact summary
+ *     messages/      — hot messages (mNNNN.md)
+ *     cold/          — archived messages (moved from messages/)
+ *     blobs/         — attachments (never moved)
+ */
+export class ConversationStore {
+  #dir;         // root dir (e.g. ~/.yeaft)
+  #convDir;     // ~/.yeaft/conversation
+  #msgDir;      // ~/.yeaft/conversation/messages
+  #coldDir;     // ~/.yeaft/conversation/cold
+  #indexPath;   // ~/.yeaft/conversation/index.md
+  #compactPath; // ~/.yeaft/conversation/compact.md
+  #nextSeq;     // next message sequence number
+
+  /**
+   * @param {string} dir — Yeaft root directory (e.g. ~/.yeaft)
+   */
+  constructor(dir) {
+    this.#dir = dir;
+    this.#convDir = join(dir, 'conversation');
+    this.#msgDir = join(dir, 'conversation', 'messages');
+    this.#coldDir = join(dir, 'conversation', 'cold');
+    this.#indexPath = join(dir, 'conversation', 'index.md');
+    this.#compactPath = join(dir, 'conversation', 'compact.md');
+    this.#nextSeq = null;
+
+    // Ensure directories exist
+    for (const d of [this.#convDir, this.#msgDir, this.#coldDir]) {
+      if (!existsSync(d)) mkdirSync(d, { recursive: true });
+    }
+  }
+
+  // ─── Write API ──────────────────────────────────────────
+
+  /**
+   * Append a single message to the conversation.
+   *
+   * @param {object} msg — { role, content, mode?, model?, turnNumber?, toolCalls?, toolCallId?, isError? }
+   * @returns {object} — the persisted message with id assigned
+   */
+  append(msg) {
+    const seq = this.#getNextSeq();
+    const id = `m${String(seq).padStart(4, '0')}`;
+    const fullMsg = {
+      ...msg,
+      id,
+      time: msg.time || new Date().toISOString(),
+      tokens_est: msg.tokens_est || estimateTokens(msg.content || ''),
+    };
+
+    const filePath = join(this.#msgDir, `${id}.md`);
+    writeFileSync(filePath, serializeMessage(fullMsg), 'utf8');
+
+    this.#nextSeq = seq + 1;
+
+    return fullMsg;
+  }
+
+  /**
+   * Append multiple messages at once.
+   *
+   * @param {object[]} messages
+   * @returns {object[]} — persisted messages with ids
+   */
+  appendBatch(messages) {
+    return messages.map(m => this.append(m));
+  }
+
+  /**
+   * Move a message from hot (messages/) to cold (cold/).
+   *
+   * @param {string} id — message id (e.g. "m0355")
+   */
+  moveToCold(id) {
+    const src = join(this.#msgDir, `${id}.md`);
+    const dst = join(this.#coldDir, `${id}.md`);
+    if (existsSync(src)) {
+      renameSync(src, dst);
+    }
+  }
+
+  /**
+   * Move multiple messages to cold.
+   *
+   * @param {string[]} ids
+   */
+  moveToColdBatch(ids) {
+    for (const id of ids) {
+      this.moveToCold(id);
+    }
+  }
+
+  /**
+   * Update the compact summary (cumulative).
+   *
+   * @param {string} summary — new summary to append
+   */
+  updateCompactSummary(summary) {
+    let existing = '';
+    if (existsSync(this.#compactPath)) {
+      existing = readFileSync(this.#compactPath, 'utf8');
+    }
+
+    const date = new Date().toISOString().split('T')[0];
+    const entry = `\n## ${date}\n\n${summary}\n`;
+    writeFileSync(this.#compactPath, existing + entry, 'utf8');
+  }
+
+  /**
+   * Read the compact summary.
+   *
+   * @returns {string}
+   */
+  readCompactSummary() {
+    if (!existsSync(this.#compactPath)) return '';
+    return readFileSync(this.#compactPath, 'utf8');
+  }
+
+  /**
+   * Update the conversation index.md with current state.
+   *
+   * @param {{ totalMessages?: number, lastMessageId?: string }} info
+   */
+  updateIndex(info = {}) {
+    const total = info.totalMessages ?? this.countHot() + this.countCold();
+    const lastId = info.lastMessageId ?? null;
+    const lastAccessed = new Date().toISOString();
+
+    const content = [
+      '---',
+      `lastMessageId: ${lastId || 'null'}`,
+      `totalMessages: ${total}`,
+      `hotMessages: ${this.countHot()}`,
+      `coldMessages: ${this.countCold()}`,
+      `lastAccessed: ${lastAccessed}`,
+      '---',
+      '',
+      '# Conversation Index',
+      '',
+      'This file tracks the conversation state for the "one eternal conversation" model.',
+    ].join('\n');
+
+    writeFileSync(this.#indexPath, content, 'utf8');
+  }
+
+  /**
+   * Clear all messages (hot + cold + compact).
+   */
+  clear() {
+    for (const dir of [this.#msgDir, this.#coldDir]) {
+      if (existsSync(dir)) {
+        for (const file of readdirSync(dir)) {
+          if (file.endsWith('.md')) {
+            unlinkSync(join(dir, file));
+          }
+        }
+      }
+    }
+    // Reset compact
+    if (existsSync(this.#compactPath)) {
+      writeFileSync(this.#compactPath, '', 'utf8');
+    }
+    this.#nextSeq = 1;
+    this.updateIndex({ totalMessages: 0, lastMessageId: null });
+  }
+
+  // ─── Read API ───────────────────────────────────────────
+
+  /**
+   * Load recent hot messages, sorted by id (chronological).
+   *
+   * @param {number} [limit=50] — max messages to load
+   * @returns {object[]} — parsed message objects
+   */
+  loadRecent(limit = 50) {
+    return this.#loadFromDir(this.#msgDir, limit);
+  }
+
+  /**
+   * Load all hot messages.
+   *
+   * @returns {object[]}
+   */
+  loadAll() {
+    return this.#loadFromDir(this.#msgDir, Infinity);
+  }
+
+  /**
+   * Count hot messages.
+   *
+   * @returns {number}
+   */
+  countHot() {
+    if (!existsSync(this.#msgDir)) return 0;
+    return readdirSync(this.#msgDir).filter(f => f.endsWith('.md')).length;
+  }
+
+  /**
+   * Count cold messages.
+   *
+   * @returns {number}
+   */
+  countCold() {
+    if (!existsSync(this.#coldDir)) return 0;
+    return readdirSync(this.#coldDir).filter(f => f.endsWith('.md')).length;
+  }
+
+  /**
+   * Get total estimated tokens for hot messages.
+   *
+   * @returns {number}
+   */
+  hotTokens() {
+    const messages = this.loadAll();
+    return messages.reduce((sum, m) => sum + (m.tokens_est || estimateTokens(m.content || '')), 0);
+  }
+
+  /**
+   * Read the conversation index.
+   *
+   * @returns {object}
+   */
+  readIndex() {
+    if (!existsSync(this.#indexPath)) {
+      return { lastMessageId: null, totalMessages: 0, hotMessages: 0, coldMessages: 0 };
+    }
+    const raw = readFileSync(this.#indexPath, 'utf8');
+    const parsed = parseMessage(raw);
+    if (!parsed) {
+      return { lastMessageId: null, totalMessages: 0, hotMessages: 0, coldMessages: 0 };
+    }
+    // Re-parse from frontmatter fields
+    return {
+      lastMessageId: parsed.id || null,
+      totalMessages: parsed.tokens_est || 0, // reuse field parsing
+    };
+  }
+
+  // ─── Internal ───────────────────────────────────────────
+
+  /**
+   * Load messages from a directory, sorted by filename, limited.
+   * @param {string} dir
+   * @param {number} limit
+   * @returns {object[]}
+   */
+  #loadFromDir(dir, limit) {
+    if (!existsSync(dir)) return [];
+
+    const files = readdirSync(dir)
+      .filter(f => f.endsWith('.md'))
+      .sort(); // m0001.md < m0002.md — chronological
+
+    // Take the most recent `limit` files
+    const selected = limit < Infinity
+      ? files.slice(-limit)
+      : files;
+
+    const messages = [];
+    for (const file of selected) {
+      const raw = readFileSync(join(dir, file), 'utf8');
+      const parsed = parseMessage(raw);
+      if (parsed) messages.push(parsed);
+    }
+
+    return messages;
+  }
+
+  /**
+   * Determine the next sequence number by scanning existing files.
+   * @returns {number}
+   */
+  #getNextSeq() {
+    if (this.#nextSeq != null) return this.#nextSeq;
+
+    let maxSeq = 0;
+    for (const dir of [this.#msgDir, this.#coldDir]) {
+      if (!existsSync(dir)) continue;
+      for (const file of readdirSync(dir)) {
+        const match = file.match(/^m(\d+)\.md$/);
+        if (match) {
+          const seq = parseInt(match[1], 10);
+          if (seq > maxSeq) maxSeq = seq;
+        }
+      }
+    }
+
+    this.#nextSeq = maxSeq + 1;
+    return this.#nextSeq;
+  }
+}