@yeaft/webchat-agent 0.1.409 → 0.1.411

package/unify/memory/scan.js

@@ -0,0 +1,273 @@
+/**
+ * scan.js — Memory header scanning and scope/tag matching
+ *
+ * Fast in-memory scanning of entry frontmatter for:
+ * - Scope tree traversal
+ * - Tag overlap scoring
+ * - Kind-based filtering
+ * - Stale entry detection (for Dream)
+ *
+ * Reference: yeaft-unify-core-systems.md §3.3, yeaft-unify-design.md §5.1
+ */
+
+import { KINDS, KIND_PRIORITY, IMPORTANCE_WEIGHT, getAncestorScopes } from './types.js';
+
+// ─── Scan Results ──────────────────────────────────────────
+
+/**
+ * @typedef {Object} ScanResult
+ * @property {object[]} entries — all parsed entries
+ * @property {Map<string, number>} scopeCount — scope → entry count
+ * @property {Map<string, number>} kindCount — kind → entry count
+ * @property {Map<string, Set<string>>} tagIndex — tag → set of entry names
+ * @property {number} totalEntries — total count
+ */
+
+/**
+ * Scan all entries from a MemoryStore and build indexes.
+ *
+ * @param {import('./store.js').MemoryStore} memoryStore
+ * @returns {ScanResult}
+ */
+export function scanEntries(memoryStore) {
+  const entries = memoryStore.listEntries();
+
+  const scopeCount = new Map();
+  const kindCount = new Map();
+  const tagIndex = new Map();
+
+  for (const entry of entries) {
+    // Scope count
+    const scope = entry.scope || 'global';
+    scopeCount.set(scope, (scopeCount.get(scope) || 0) + 1);
+
+    // Kind count
+    const kind = entry.kind || 'fact';
+    kindCount.set(kind, (kindCount.get(kind) || 0) + 1);
+
+    // Tag index
+    const tags = entry.tags || [];
+    for (const tag of tags) {
+      const lowerTag = tag.toLowerCase();
+      if (!tagIndex.has(lowerTag)) tagIndex.set(lowerTag, new Set());
+      tagIndex.get(lowerTag).add(entry.name);
+    }
+  }
+
+  return {
+    entries,
+    scopeCount,
+    kindCount,
+    tagIndex,
+    totalEntries: entries.length,
+  };
+}
+
+// ─── Scoring Functions ─────────────────────────────────────
+
+/**
+ * Score an entry for relevance to a query context.
+ *
+ * Scoring factors:
+ *   - Scope match: exact=5, parent/child=3, global=1
+ *   - Tag overlap: 2 per matching tag
+ *   - Kind priority: see KIND_PRIORITY
+ *   - Importance weight: see IMPORTANCE_WEIGHT
+ *   - Frequency bonus: log2(frequency)
+ *   - Recency bonus: entries updated in last 7 days get +2
+ *
+ * @param {object} entry — memory entry
+ * @param {{ scope?: string, tags?: string[], preferKinds?: string[] }} context
+ * @returns {number} — relevance score
+ */
+export function scoreEntry(entry, context = {}) {
+  let score = 0;
+
+  // Scope match
+  if (context.scope && entry.scope) {
+    if (entry.scope === context.scope) {
+      score += 5; // exact match
+    } else {
+      const ancestors = getAncestorScopes(context.scope);
+      if (ancestors.includes(entry.scope)) {
+        score += 3; // ancestor match
+      } else if (entry.scope.startsWith(context.scope + '/')) {
+        score += 3; // descendant match
+      } else if (entry.scope === 'global') {
+        score += 1; // global fallback
+      }
+    }
+  }
+
+  // Tag overlap
+  if (context.tags && context.tags.length > 0 && entry.tags) {
+    const entryTags = new Set(entry.tags.map(t => t.toLowerCase()));
+    for (const tag of context.tags) {
+      if (entryTags.has(tag.toLowerCase())) {
+        score += 2;
+      }
+    }
+  }
+
+  // Kind priority
+  const kindPriority = KIND_PRIORITY[entry.kind] || 0;
+  score += kindPriority * 0.5;
+
+  // Preferred kinds bonus
+  if (context.preferKinds && context.preferKinds.includes(entry.kind)) {
+    score += 2;
+  }
+
+  // Importance weight
+  const impWeight = IMPORTANCE_WEIGHT[entry.importance] || IMPORTANCE_WEIGHT.normal;
+  score += impWeight * 0.5;
+
+  // Frequency bonus (logarithmic)
+  const freq = entry.frequency || 1;
+  score += Math.log2(Math.max(freq, 1));
+
+  // Recency bonus
+  if (entry.updated_at) {
+    const daysSince = (Date.now() - new Date(entry.updated_at).getTime()) / (1000 * 60 * 60 * 24);
+    if (daysSince <= 7) score += 2;
+    else if (daysSince <= 30) score += 1;
+  }
+
+  return score;
+}
+
+// ─── Stale Detection (for Dream) ────────────────────────────
+
+/**
+ * Find entries that are potentially stale.
+ *
+ * Stale criteria:
+ * - context entries older than 30 days
+ * - entries never recalled (frequency = 1) and older than 60 days
+ * - relation entries older than 90 days
+ *
+ * @param {object[]} entries
+ * @returns {object[]} — stale entries
+ */
+export function findStaleEntries(entries) {
+  const now = Date.now();
+  const stale = [];
+
+  for (const entry of entries) {
+    const updatedAt = entry.updated_at ? new Date(entry.updated_at).getTime() : 0;
+    const daysSince = (now - updatedAt) / (1000 * 60 * 60 * 24);
+
+    let isStale = false;
+
+    // Context entries become stale fast
+    if (entry.kind === 'context' && daysSince > 30) {
+      isStale = true;
+    }
+
+    // Entries never recalled and old
+    if ((entry.frequency || 1) <= 1 && daysSince > 60) {
+      isStale = true;
+    }
+
+    // Relations are volatile
+    if (entry.kind === 'relation' && daysSince > 90) {
+      isStale = true;
+    }
+
+    if (isStale) {
+      stale.push({ ...entry, _daysSinceUpdate: Math.round(daysSince) });
+    }
+  }
+
+  return stale;
+}
+
+// ─── Duplicate Detection (for Dream Merge) ──────────────────
+
+/**
+ * Find groups of entries that are potentially duplicates.
+ * Entries are grouped if they share ≥2 tags AND the same kind.
+ *
+ * @param {object[]} entries
+ * @returns {object[][]} — groups of potentially duplicate entries
+ */
+export function findDuplicateGroups(entries) {
+  const groups = [];
+  const visited = new Set();
+
+  for (let i = 0; i < entries.length; i++) {
+    if (visited.has(i)) continue;
+
+    const group = [entries[i]];
+    const eTags = new Set((entries[i].tags || []).map(t => t.toLowerCase()));
+
+    for (let j = i + 1; j < entries.length; j++) {
+      if (visited.has(j)) continue;
+      if (entries[i].kind !== entries[j].kind) continue;
+
+      const jTags = new Set((entries[j].tags || []).map(t => t.toLowerCase()));
+      let overlap = 0;
+      for (const tag of eTags) {
+        if (jTags.has(tag)) overlap++;
+      }
+
+      if (overlap >= 2) {
+        group.push(entries[j]);
+        visited.add(j);
+      }
+    }
+
+    if (group.length > 1) {
+      visited.add(i);
+      groups.push(group);
+    }
+  }
+
+  return groups;
+}
+
+// ─── Stats Summary ──────────────────────────────────────────
+
+/**
+ * Generate a text summary of memory state (for Dream prompts).
+ *
+ * @param {ScanResult} scan
+ * @returns {string}
+ */
+export function summarizeScan(scan) {
+  const lines = [];
+
+  lines.push(`Total entries: ${scan.totalEntries}`);
+
+  // Kind breakdown
+  const kindLines = [];
+  for (const kind of KINDS) {
+    const count = scan.kindCount.get(kind) || 0;
+    if (count > 0) kindLines.push(`${kind}: ${count}`);
+  }
+  if (kindLines.length > 0) {
+    lines.push(`Kinds: ${kindLines.join(', ')}`);
+  }
+
+  // Scope breakdown (top 10)
+  const scopeEntries = [...scan.scopeCount.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 10);
+  if (scopeEntries.length > 0) {
+    lines.push('Top scopes:');
+    for (const [scope, count] of scopeEntries) {
+      lines.push(`  ${scope}: ${count}`);
+    }
+  }
+
+  // Tag cloud (top 20)
+  const tagEntries = [...scan.tagIndex.entries()]
+    .map(([tag, names]) => [tag, names.size])
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 20);
+  if (tagEntries.length > 0) {
+    lines.push(`Top tags: ${tagEntries.map(([t, c]) => `${t}(${c})`).join(', ')}`);
+  }
+
+  return lines.join('\n');
+}

package/unify/memory/types.js

@@ -0,0 +1,139 @@
+/**
+ * types.js — Memory type definitions and constants
+ *
+ * Defines the 3D memory model:
+ *   Kind  = WHAT — 6 types: fact, preference, skill, lesson, context, relation
+ *   Scope = WHERE — dynamic tree path: global / work/project / tech/typescript
+ *   Tags  = HOW — free keywords for retrieval
+ *
+ * Reference: yeaft-unify-core-systems.md §2.2, yeaft-unify-brainstorm-v3.md
+ */
+
+// ─── Kind ────────────────────────────────────────────────────
+
+/** All valid memory kinds. */
+export const KINDS = ['fact', 'preference', 'skill', 'lesson', 'context', 'relation'];
+
+/** Kind descriptions for prompt context. */
+export const KIND_DESCRIPTIONS = {
+  fact: 'Objective facts (project structure, tech stack, verified information)',
+  preference: 'User preferences (coding style, tools, communication style)',
+  skill: 'How to do something (patterns, techniques, workflows, commands)',
+  lesson: 'Lessons learned (bugs, pitfalls, effective alternatives)',
+  context: 'Temporal context (current OKR, project progress, deadlines)',
+  relation: 'People and relationships (teammates, roles, responsibilities)',
+};
+
+/** Kind priority for dream consolidation (higher = more important). */
+export const KIND_PRIORITY = {
+  fact: 6,
+  preference: 5,
+  skill: 4,
+  lesson: 3,
+  context: 2,
+  relation: 1,
+};
+
+// ─── Scope ──────────────────────────────────────────────────
+
+/**
+ * Parse a scope path into segments.
+ * @param {string} scope — e.g. "work/claude-web-chat/auth"
+ * @returns {string[]} — e.g. ["work", "claude-web-chat", "auth"]
+ */
+export function parseScopePath(scope) {
+  if (!scope) return ['global'];
+  return scope.split('/').filter(Boolean);
+}
+
+/**
+ * Get all ancestor scopes (including the scope itself and 'global').
+ * @param {string} scope — e.g. "work/claude-web-chat/auth"
+ * @returns {string[]} — e.g. ["global", "work", "work/claude-web-chat", "work/claude-web-chat/auth"]
+ */
+export function getAncestorScopes(scope) {
+  if (!scope || scope === 'global') return ['global'];
+
+  const segments = parseScopePath(scope);
+  const ancestors = ['global'];
+
+  for (let i = 0; i < segments.length; i++) {
+    ancestors.push(segments.slice(0, i + 1).join('/'));
+  }
+
+  return ancestors;
+}
+
+/**
+ * Check if two scopes are related (one is ancestor/descendant of the other).
+ * @param {string} a
+ * @param {string} b
+ * @returns {boolean}
+ */
+export function areScopesRelated(a, b) {
+  if (!a || !b || a === 'global' || b === 'global') return true;
+  return a.startsWith(b + '/') || b.startsWith(a + '/') || a === b;
+}
+
+// ─── Importance ─────────────────────────────────────────────
+
+/** Valid importance levels. */
+export const IMPORTANCE_LEVELS = ['high', 'normal', 'low'];
+
+/** Importance weight for scoring. */
+export const IMPORTANCE_WEIGHT = {
+  high: 3,
+  normal: 2,
+  low: 1,
+};
+
+// ─── Entry Schema ──────────────────────────────────────────
+
+/**
+ * @typedef {Object} MemoryEntry
+ * @property {string} name — unique slug name
+ * @property {string} kind — one of KINDS
+ * @property {string} scope — tree path (e.g. "global", "tech/typescript")
+ * @property {string[]} tags — free keywords
+ * @property {string} importance — "high" | "normal" | "low"
+ * @property {number} frequency — how often this entry is recalled
+ * @property {string} content — the actual memory content
+ * @property {string[]} [related] — related entry names
+ * @property {string} [created_at] — ISO timestamp
+ * @property {string} [updated_at] — ISO timestamp
+ */
+
+/**
+ * Validate a memory entry object.
+ * @param {object} entry
+ * @returns {{ valid: boolean, errors: string[] }}
+ */
+export function validateEntry(entry) {
+  const errors = [];
+
+  if (!entry || typeof entry !== 'object') {
+    return { valid: false, errors: ['Entry must be an object'] };
+  }
+
+  if (!entry.name || typeof entry.name !== 'string') {
+    errors.push('Entry must have a string "name"');
+  }
+
+  if (entry.kind && !KINDS.includes(entry.kind)) {
+    errors.push(`Invalid kind "${entry.kind}". Must be one of: ${KINDS.join(', ')}`);
+  }
+
+  if (entry.importance && !IMPORTANCE_LEVELS.includes(entry.importance)) {
+    errors.push(`Invalid importance "${entry.importance}". Must be one of: ${IMPORTANCE_LEVELS.join(', ')}`);
+  }
+
+  if (!entry.content || typeof entry.content !== 'string') {
+    errors.push('Entry must have string "content"');
+  }
+
+  if (entry.tags && !Array.isArray(entry.tags)) {
+    errors.push('"tags" must be an array');
+  }
+
+  return { valid: errors.length === 0, errors };
+}

package/unify/prompts.js

@@ -61,6 +61,7 @@ export function buildSystemPrompt({
   toolNames = [],
   memory,
   compactSummary,
+  skillContent,
 } = {}) {
   // Fallback to English for unknown languages
   const lang = PROMPTS[language] || PROMPTS.en;
@@ -81,6 +82,11 @@ export function buildSystemPrompt({
     parts.push(lang.tools(toolNames.join(', ')));
   }
 
+  // ─── Skills Section ─────────────────────────────────────
+  if (skillContent) {
+    parts.push(skillContent);
+  }
+
   // ─── Memory Section ─────────────────────────────────────
   if (memory && (memory.profile || (memory.entries && memory.entries.length > 0))) {
     const memoryParts = [lang.memoryHeader];

package/unify/session.js

@@ -0,0 +1,191 @@
+/**
+ * session.js — Session orchestrator for Yeaft Unify
+ *
+ * Single entry point: loadSession(options?) → Session
+ *
+ * Wires all subsystems together:
+ *   initYeaftDir → loadConfig → createTrace → createLLMAdapter →
+ *   ConversationStore → MemoryStore → SkillManager → MCPManager →
+ *   ToolRegistry → Engine → Session
+ *
+ * The ~/.yeaft/ directory is the agent's persistent workspace.
+ * loadSession() loads (or initializes) this workspace and returns
+ * a fully wired Session ready for queries.
+ */
+
+import { initYeaftDir } from './init.js';
+import { loadConfig, loadMCPConfig } from './config.js';
+import { createTrace } from './debug-trace.js';
+import { createLLMAdapter } from './llm/adapter.js';
+import { ConversationStore } from './conversation/persist.js';
+import { MemoryStore } from './memory/store.js';
+import { SkillManager, createSkillManager } from './skills.js';
+import { MCPManager } from './mcp.js';
+import { createEmptyRegistry } from './tools/registry.js';
+import { Engine } from './engine.js';
+import { join } from 'path';
+
+// Built-in tools
+import mcpTools from './tools/mcp-tools.js';
+import skillTool from './tools/skill.js';
+import enterWorktree from './tools/enter-worktree.js';
+import exitWorktree from './tools/exit-worktree.js';
+
+/**
+ * @typedef {Object} SessionOptions
+ * @property {string} [dir] — Yeaft data directory override (default: ~/.yeaft)
+ * @property {string} [model] — Model override
+ * @property {string} [language] — Language override ('en' | 'zh')
+ * @property {boolean} [debug] — Debug mode override
+ * @property {boolean} [skipMCP] — Skip MCP server connections (faster startup)
+ * @property {boolean} [skipSkills] — Skip skill loading
+ * @property {object[]} [extraTools] — Additional ToolDef objects to register
+ * @property {object} [configOverrides] — Additional config overrides
+ */
+
+/**
+ * @typedef {Object} Session
+ * @property {Engine} engine — The wired engine, ready for .query()
+ * @property {object} config — Resolved configuration
+ * @property {ConversationStore} conversationStore — Conversation persistence
+ * @property {MemoryStore} memoryStore — Memory persistence
+ * @property {SkillManager} skillManager — Skill manager
+ * @property {MCPManager} mcpManager — MCP manager
+ * @property {import('./tools/registry.js').ToolRegistry} toolRegistry — Tool registry
+ * @property {import('./debug-trace.js').DebugTrace|import('./debug-trace.js').NullTrace} trace
+ * @property {string} yeaftDir — Resolved data directory path
+ * @property {{ skills: number, mcpServers: string[], mcpFailed: object[], tools: number }} status
+ * @property {() => Promise<void>} shutdown — Graceful shutdown
+ */
+
+/**
+ * Load (or initialize) a Yeaft session.
+ *
+ * This is the main entry point for using Yeaft programmatically.
+ * It creates the directory structure if needed, loads config, connects
+ * to services, registers tools, and returns a ready-to-use Session.
+ *
+ * @param {SessionOptions} [options={}]
+ * @returns {Promise<Session>}
+ */
+export async function loadSession(options = {}) {
+  const {
+    dir,
+    model,
+    language,
+    debug,
+    skipMCP = false,
+    skipSkills = false,
+    extraTools = [],
+    configOverrides = {},
+  } = options;
+
+  // ─── 1. Load config (determines yeaftDir) ──────────────
+  const overrides = { ...configOverrides };
+  if (dir) overrides.dir = dir;
+  if (model) overrides.model = model;
+  if (language) overrides.language = language;
+  if (debug !== undefined) overrides.debug = debug;
+
+  const config = loadConfig(overrides);
+  const yeaftDir = config.dir;
+
+  // ─── 2. Ensure directory structure ─────────────────────
+  initYeaftDir(yeaftDir);
+
+  // ─── 3. Create debug trace ─────────────────────────────
+  const trace = createTrace({
+    enabled: config.debug,
+    dbPath: join(yeaftDir, 'debug.db'),
+  });
+
+  // ─── 4. Create LLM adapter ────────────────────────────
+  const adapter = await createLLMAdapter(config);
+
+  // ─── 5. Create stores ──────────────────────────────────
+  const conversationStore = new ConversationStore(yeaftDir);
+  const memoryStore = new MemoryStore(yeaftDir);
+
+  // ─── 6. Load skills ────────────────────────────────────
+  let skillManager;
+  if (skipSkills) {
+    skillManager = new SkillManager(yeaftDir);
+    // Don't call .load() — empty skill manager
+  } else {
+    skillManager = createSkillManager(yeaftDir);
+  }
+
+  // ─── 7. Connect MCP servers ────────────────────────────
+  const mcpConfig = loadMCPConfig(yeaftDir);
+  const mcpManager = new MCPManager();
+  let mcpStatus = { connected: [], failed: [] };
+
+  if (!skipMCP && mcpConfig.servers.length > 0) {
+    mcpStatus = await mcpManager.connectAll(mcpConfig.servers);
+  }
+
+  // ─── 8. Build tool registry ────────────────────────────
+  const toolRegistry = createEmptyRegistry();
+
+  // Register built-in tools
+  for (const tool of mcpTools) {
+    toolRegistry.register(tool);
+  }
+  toolRegistry.register(skillTool);
+  toolRegistry.register(enterWorktree);
+  toolRegistry.register(exitWorktree);
+
+  // Register any extra tools from caller
+  for (const tool of extraTools) {
+    toolRegistry.register(tool);
+  }
+
+  // ─── 9. Create engine (wires everything) ───────────────
+  const engine = new Engine({
+    adapter,
+    trace,
+    config,
+    conversationStore,
+    memoryStore,
+    toolRegistry,
+    skillManager,
+    mcpManager,
+    yeaftDir,
+  });
+
+  // ─── 10. Build session ─────────────────────────────────
+  const status = {
+    skills: skillManager.size,
+    mcpServers: mcpStatus.connected,
+    mcpFailed: mcpStatus.failed,
+    tools: toolRegistry.size,
+  };
+
+  /** Graceful shutdown: disconnect MCP, close trace DB. */
+  async function shutdown() {
+    try {
+      await mcpManager.disconnectAll();
+    } catch {
+      // Best-effort cleanup
+    }
+    try {
+      trace.close();
+    } catch {
+      // Trace might not have close() (NullTrace)
+    }
+  }
+
+  return {
+    engine,
+    config,
+    conversationStore,
+    memoryStore,
+    skillManager,
+    mcpManager,
+    toolRegistry,
+    trace,
+    yeaftDir,
+    status,
+    shutdown,
+  };
+}