claude-brain 0.27.2 → 0.28.0

package/src/training/data-store.ts

@@ -0,0 +1,298 @@
+/**
+ * Training Data Store — Phase 1A (SLM Upgrade)
+ * Logs classification decisions to SQLite for model training.
+ * Async, non-blocking — never impacts main request path.
+ *
+ * Table: training_data in ~/.claude-brain/data/memory.db
+ */
+
+import { Database } from 'bun:sqlite'
+import { join } from 'node:path'
+import { existsSync, mkdirSync } from 'node:fs'
+import { getClaudeBrainHome } from '@/config/home'
+
+export type TrainingTask = 'intent' | 'entity' | 'query' | 'knowledge' | 'compress' | 'pattern'
+
+export interface TrainingEntry {
+  task: TrainingTask
+  input: string
+  output: string   // JSON-encoded: label, entities array, summary, etc.
+  metadata?: string // JSON-encoded: confidence, scores, timing
+}
+
+export interface ModelFeedbackEntry {
+  task: string
+  input: string
+  modelPrediction: string
+  modelConfidence: number
+  regexPrediction: string
+  actualLabel?: string
+}
+
+let db: Database | null = null
+let insertStmt: ReturnType<Database['prepare']> | null = null
+let feedbackInsertStmt: ReturnType<Database['prepare']> | null = null
+
+function getDb(): Database | null {
+  if (db) return db
+  try {
+    const dataDir = join(getClaudeBrainHome(), 'data')
+    if (!existsSync(dataDir)) {
+      mkdirSync(dataDir, { recursive: true })
+    }
+    const dbPath = join(dataDir, 'memory.db')
+    db = new Database(dbPath)
+    db.run('PRAGMA journal_mode = WAL')
+    ensureTable(db)
+    insertStmt = db.prepare(
+      'INSERT INTO training_data (task, input, output, metadata) VALUES (?, ?, ?, ?)'
+    )
+    feedbackInsertStmt = db.prepare(
+      'INSERT INTO model_feedback (task, input, model_prediction, model_confidence, regex_prediction, actual_label) VALUES (?, ?, ?, ?, ?, ?)'
+    )
+    return db
+  } catch {
+    return null
+  }
+}
+
+function ensureTable(database: Database): void {
+  database.run(`
+    CREATE TABLE IF NOT EXISTS training_data (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      task TEXT NOT NULL,
+      input TEXT NOT NULL,
+      output TEXT NOT NULL,
+      metadata TEXT,
+      verified INTEGER DEFAULT 0,
+      created_at TEXT DEFAULT (datetime('now'))
+    )
+  `)
+  // Indexes for efficient querying
+  database.run('CREATE INDEX IF NOT EXISTS idx_training_task ON training_data(task)')
+  database.run('CREATE INDEX IF NOT EXISTS idx_training_verified ON training_data(verified)')
+
+  // Phase 6A: Model feedback table for continuous learning loop
+  database.run(`
+    CREATE TABLE IF NOT EXISTS model_feedback (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      task TEXT NOT NULL,
+      input TEXT NOT NULL,
+      model_prediction TEXT NOT NULL,
+      model_confidence REAL NOT NULL,
+      regex_prediction TEXT NOT NULL,
+      actual_label TEXT,
+      created_at TEXT DEFAULT (datetime('now'))
+    )
+  `)
+  database.run('CREATE INDEX IF NOT EXISTS idx_feedback_task ON model_feedback(task)')
+}
+
+/**
+ * Log a training example. Fire-and-forget — errors are silently swallowed.
+ */
+export function logTrainingData(entry: TrainingEntry): void {
+  setImmediate(() => {
+    try {
+      const database = getDb()
+      if (!database || !insertStmt) return
+      insertStmt.run(entry.task, entry.input, entry.output, entry.metadata || null)
+    } catch {
+      // Never block or crash the main path
+    }
+  })
+}
+
+/**
+ * Export training data as JSONL lines for a specific task.
+ */
+export function exportTrainingData(
+  task: TrainingTask,
+  options?: { verifiedOnly?: boolean; limit?: number }
+): string[] {
+  const database = getDb()
+  if (!database) return []
+
+  let sql = 'SELECT input, output, metadata, verified, created_at FROM training_data WHERE task = ?'
+  const params: any[] = [task]
+
+  if (options?.verifiedOnly) {
+    sql += ' AND verified = 1'
+  }
+
+  sql += ' ORDER BY created_at DESC'
+
+  if (options?.limit) {
+    sql += ' LIMIT ?'
+    params.push(options.limit)
+  }
+
+  const rows = database.prepare(sql).all(...params) as any[]
+  return rows.map(row => JSON.stringify({
+    input: row.input,
+    output: JSON.parse(row.output),
+    metadata: row.metadata ? JSON.parse(row.metadata) : null,
+    verified: row.verified === 1,
+    created_at: row.created_at,
+  }))
+}
+
+/**
+ * Get count of training examples per task.
+ */
+export function getTrainingStats(): Record<TrainingTask, { total: number; verified: number }> {
+  const database = getDb()
+  const tasks: TrainingTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
+  const stats = {} as Record<TrainingTask, { total: number; verified: number }>
+
+  for (const task of tasks) {
+    if (!database) {
+      stats[task] = { total: 0, verified: 0 }
+      continue
+    }
+    const total = (database.prepare('SELECT COUNT(*) as c FROM training_data WHERE task = ?').get(task) as any)?.c || 0
+    const verified = (database.prepare('SELECT COUNT(*) as c FROM training_data WHERE task = ? AND verified = 1').get(task) as any)?.c || 0
+    stats[task] = { total, verified }
+  }
+
+  return stats
+}
+
+// ── Phase 6A: Model Feedback Functions ──────────────────────────────
+
+/**
+ * Log a model vs regex comparison. Fire-and-forget — errors are silently swallowed.
+ */
+export function logModelFeedback(entry: ModelFeedbackEntry): void {
+  setImmediate(() => {
+    try {
+      const database = getDb()
+      if (!database || !feedbackInsertStmt) return
+      feedbackInsertStmt.run(
+        entry.task,
+        entry.input,
+        entry.modelPrediction,
+        entry.modelConfidence,
+        entry.regexPrediction,
+        entry.actualLabel || null
+      )
+    } catch {
+      // Never block or crash the main path
+    }
+  })
+}
+
+/**
+ * Get per-task feedback stats: total, agreements, disagreements, disagreement rate, reviewed count.
+ */
+export function getModelFeedbackStats(): Record<string, {
+  total: number
+  agreements: number
+  disagreements: number
+  disagreementRate: number
+  reviewed: number
+}> {
+  const database = getDb()
+  const tasks: TrainingTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
+  const stats = {} as Record<string, {
+    total: number
+    agreements: number
+    disagreements: number
+    disagreementRate: number
+    reviewed: number
+  }>
+
+  for (const task of tasks) {
+    if (!database) {
+      stats[task] = { total: 0, agreements: 0, disagreements: 0, disagreementRate: 0, reviewed: 0 }
+      continue
+    }
+    const total = (database.prepare(
+      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ?'
+    ).get(task) as any)?.c || 0
+
+    const agreements = (database.prepare(
+      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ? AND model_prediction = regex_prediction'
+    ).get(task) as any)?.c || 0
+
+    const disagreements = total - agreements
+
+    const reviewed = (database.prepare(
+      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ? AND actual_label IS NOT NULL'
+    ).get(task) as any)?.c || 0
+
+    stats[task] = {
+      total,
+      agreements,
+      disagreements,
+      disagreementRate: total > 0 ? disagreements / total : 0,
+      reviewed,
+    }
+  }
+
+  return stats
+}
+
+/**
+ * Export feedback as JSONL lines for a specific task.
+ */
+export function exportModelFeedback(
+  task: string,
+  options?: { limit?: number }
+): string[] {
+  const database = getDb()
+  if (!database) return []
+
+  let sql = 'SELECT input, model_prediction, model_confidence, regex_prediction, actual_label, created_at FROM model_feedback WHERE task = ?'
+  const params: any[] = [task]
+
+  sql += ' ORDER BY created_at DESC'
+
+  if (options?.limit) {
+    sql += ' LIMIT ?'
+    params.push(options.limit)
+  }
+
+  const rows = database.prepare(sql).all(...params) as any[]
+  return rows.map(row => JSON.stringify({
+    input: row.input,
+    modelPrediction: row.model_prediction,
+    modelConfidence: row.model_confidence,
+    regexPrediction: row.regex_prediction,
+    actualLabel: row.actual_label,
+    createdAt: row.created_at,
+  }))
+}
+
+/**
+ * Get the most recent disagreements for human review.
+ */
+export function getDisagreements(
+  task: string,
+  limit: number = 50
+): Array<{
+  input: string
+  modelPrediction: string
+  modelConfidence: number
+  regexPrediction: string
+  createdAt: string
+}> {
+  const database = getDb()
+  if (!database) return []
+
+  const rows = database.prepare(`
+    SELECT input, model_prediction, model_confidence, regex_prediction, created_at
+    FROM model_feedback
+    WHERE task = ? AND model_prediction != regex_prediction
+    ORDER BY created_at DESC
+    LIMIT ?
+  `).all(task, limit) as any[]
+
+  return rows.map(row => ({
+    input: row.input,
+    modelPrediction: row.model_prediction,
+    modelConfidence: row.model_confidence,
+    regexPrediction: row.regex_prediction,
+    createdAt: row.created_at,
+  }))
+}

package/src/training/retrain-pipeline.ts

@@ -0,0 +1,394 @@
+/**
+ * Retrain Pipeline — SLM Upgrade Phase 6B
+ * Automated retraining when enough feedback accumulates.
+ *
+ * Flow:
+ *   1. Check if retraining is needed (feedback count + disagreement rate)
+ *   2. Export merged training data (existing + feedback corrections) as JSONL
+ *   3. Shell out to Python: prepare_data → train → evaluate → export_onnx
+ *   4. Compare new accuracy to old, promote model if improved
+ *   5. Update retrain-state.json with timestamp + stats
+ */
+
+import { execSync } from 'node:child_process'
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { homedir } from 'node:os'
+import { getClaudeBrainHome, getHomePaths } from '@/config/home'
+import {
+  exportTrainingData,
+  exportModelFeedback,
+  getModelFeedbackStats,
+  type TrainingTask,
+} from '@/training/data-store'
+import type { ModelManifest, ModelTask } from '@/intelligence/model-manager'
+
+// ── Types ────────────────────────────────────────────────────────────
+
+export interface RetrainConfig {
+  minFeedbackCount: number
+  maxDisagreementRate: number
+  pythonPath: string
+  trainingDir: string
+  force: boolean
+}
+
+export interface RetrainCheck {
+  needed: boolean
+  reason: string
+  feedbackCount: number
+  disagreementRate: number
+  lastRetrainDate: string | null
+}
+
+export interface RetrainResult {
+  task: string
+  success: boolean
+  error?: string
+  oldAccuracy?: number
+  newAccuracy?: number
+  trainingDataCount: number
+  duration: number
+}
+
+interface RetrainStateEntry {
+  lastRetrain: string
+  lastAccuracy: number
+  feedbackAtRetrain: number
+}
+
+type RetrainState = Record<string, RetrainStateEntry>
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+function resolveTrainingDir(trainingDir: string): string {
+  if (trainingDir.startsWith('~')) {
+    return join(homedir(), trainingDir.slice(1))
+  }
+  return trainingDir
+}
+
+function getRetrainStatePath(): string {
+  return join(getClaudeBrainHome(), 'data', 'retrain-state.json')
+}
+
+function loadRetrainState(): RetrainState {
+  const statePath = getRetrainStatePath()
+  if (!existsSync(statePath)) return {}
+  try {
+    return JSON.parse(readFileSync(statePath, 'utf-8'))
+  } catch {
+    return {}
+  }
+}
+
+function saveRetrainState(state: RetrainState): void {
+  const statePath = getRetrainStatePath()
+  const dir = join(getClaudeBrainHome(), 'data')
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true })
+  }
+  writeFileSync(statePath, JSON.stringify(state, null, 2))
+}
+
+function loadManifest(): ModelManifest | null {
+  const manifestPath = join(getHomePaths().models, 'manifest.json')
+  if (!existsSync(manifestPath)) return null
+  try {
+    return JSON.parse(readFileSync(manifestPath, 'utf-8'))
+  } catch {
+    return null
+  }
+}
+
+function saveManifest(manifest: ModelManifest): void {
+  const modelsDir = getHomePaths().models
+  if (!existsSync(modelsDir)) {
+    mkdirSync(modelsDir, { recursive: true })
+  }
+  writeFileSync(join(modelsDir, 'manifest.json'), JSON.stringify(manifest, null, 2))
+}
+
+// ── Public API ───────────────────────────────────────────────────────
+
+/**
+ * Check if retraining is needed for a task based on feedback accumulation.
+ */
+export function shouldRetrain(task: ModelTask, config: RetrainConfig): RetrainCheck {
+  const state = loadRetrainState()
+  const taskState = state[task]
+  const lastRetrainDate = taskState?.lastRetrain ?? null
+  const feedbackAtLastRetrain = taskState?.feedbackAtRetrain ?? 0
+
+  const feedbackStats = getModelFeedbackStats()
+  const taskStats = feedbackStats[task]
+
+  if (!taskStats) {
+    return {
+      needed: false,
+      reason: 'No feedback data available',
+      feedbackCount: 0,
+      disagreementRate: 0,
+      lastRetrainDate,
+    }
+  }
+
+  const newFeedbackCount = taskStats.total - feedbackAtLastRetrain
+  const disagreementRate = taskStats.disagreementRate
+
+  // Check if enough new feedback has accumulated
+  if (newFeedbackCount < config.minFeedbackCount) {
+    return {
+      needed: false,
+      reason: `Only ${newFeedbackCount} new feedback entries (need ${config.minFeedbackCount})`,
+      feedbackCount: taskStats.total,
+      disagreementRate,
+      lastRetrainDate,
+    }
+  }
+
+  // Check if disagreement rate exceeds threshold
+  if (disagreementRate > config.maxDisagreementRate) {
+    return {
+      needed: true,
+      reason: `Disagreement rate ${(disagreementRate * 100).toFixed(1)}% exceeds threshold ${(config.maxDisagreementRate * 100).toFixed(1)}%`,
+      feedbackCount: taskStats.total,
+      disagreementRate,
+      lastRetrainDate,
+    }
+  }
+
+  // Enough feedback accumulated even if disagreement rate is below threshold
+  return {
+    needed: true,
+    reason: `${newFeedbackCount} new feedback entries since last retrain`,
+    feedbackCount: taskStats.total,
+    disagreementRate,
+    lastRetrainDate,
+  }
+}
+
+/**
+ * Export merged training data (training_data + feedback corrections) as JSONL.
+ */
+export function exportMergedTrainingData(
+  task: ModelTask,
+  trainingDir: string
+): { path: string; count: number } {
+  const resolvedDir = resolveTrainingDir(trainingDir)
+  const dataDir = join(resolvedDir, 'data', task)
+  if (!existsSync(dataDir)) {
+    mkdirSync(dataDir, { recursive: true })
+  }
+
+  const outputPath = join(dataDir, 'train.jsonl')
+
+  // Get existing training data
+  const trainingLines = exportTrainingData(task as TrainingTask)
+
+  // Get feedback data (disagreements are useful training signal)
+  const feedbackLines = exportModelFeedback(task)
+
+  // Merge: training data first, then feedback
+  const allLines = [...trainingLines, ...feedbackLines]
+
+  writeFileSync(outputPath, allLines.join('\n') + '\n')
+
+  return { path: outputPath, count: allLines.length }
+}
+
+/**
+ * Run the full retrain pipeline for a single task.
+ */
+export async function retrainTask(
+  task: ModelTask,
+  config: RetrainConfig
+): Promise<RetrainResult> {
+  const startTime = Date.now()
+  const resolvedDir = resolveTrainingDir(config.trainingDir)
+
+  // Step 1: Export merged training data
+  console.log(`  [${task}] Exporting merged training data...`)
+  let dataCount: number
+  try {
+    const { count } = exportMergedTrainingData(task, config.trainingDir)
+    dataCount = count
+    console.log(`  [${task}] Exported ${count} training examples`)
+  } catch (err) {
+    return {
+      task,
+      success: false,
+      error: `Failed to export training data: ${err instanceof Error ? err.message : String(err)}`,
+      trainingDataCount: 0,
+      duration: Date.now() - startTime,
+    }
+  }
+
+  if (dataCount === 0) {
+    return {
+      task,
+      success: false,
+      error: 'No training data available',
+      trainingDataCount: 0,
+      duration: Date.now() - startTime,
+    }
+  }
+
+  // Step 2: Run Python training pipeline
+  const pythonSteps = [
+    { script: 'scripts/prepare_data.py', args: `--task ${task}`, label: 'Preparing data' },
+    { script: 'scripts/train.py', args: `--task ${task}`, label: 'Training model' },
+    { script: 'scripts/evaluate.py', args: `--task ${task} --save`, label: 'Evaluating model' },
+    { script: 'scripts/export_onnx.py', args: `--task ${task} --output-dir models/`, label: 'Exporting ONNX' },
+  ]
+
+  for (const step of pythonSteps) {
+    console.log(`  [${task}] ${step.label}...`)
+    const cmd = `${config.pythonPath} ${step.script} ${step.args}`
+    try {
+      execSync(cmd, {
+        cwd: resolvedDir,
+        stdio: 'pipe',
+        timeout: 600_000, // 10 minute timeout per step
+      })
+    } catch (err) {
+      const stderr = err instanceof Error && 'stderr' in err
+        ? (err as any).stderr?.toString().slice(0, 500)
+        : String(err)
+      return {
+        task,
+        success: false,
+        error: `Python step "${step.label}" failed: ${stderr}`,
+        trainingDataCount: dataCount,
+        duration: Date.now() - startTime,
+      }
+    }
+  }
+
+  // Step 3: Read evaluation results
+  const benchmarkPath = join(resolvedDir, 'benchmarks', `${task}.json`)
+  let newAccuracy: number | undefined
+  try {
+    if (existsSync(benchmarkPath)) {
+      const benchData = JSON.parse(readFileSync(benchmarkPath, 'utf-8'))
+      newAccuracy = benchData.accuracy ?? benchData.overall_accuracy
+    }
+  } catch {
+    console.log(`  [${task}] Warning: Could not read benchmark results`)
+  }
+
+  // Step 4: Compare with existing model accuracy
+  const manifest = loadManifest()
+  const oldAccuracy = manifest?.models[task]?.accuracy
+
+  if (newAccuracy != null && oldAccuracy != null && newAccuracy < oldAccuracy && !config.force) {
+    console.log(`  [${task}] New accuracy ${(newAccuracy * 100).toFixed(1)}% < old ${(oldAccuracy * 100).toFixed(1)}% — skipping model replacement`)
+    // Still record the retrain attempt
+    const state = loadRetrainState()
+    const feedbackStats = getModelFeedbackStats()
+    state[task] = {
+      lastRetrain: new Date().toISOString(),
+      lastAccuracy: oldAccuracy,
+      feedbackAtRetrain: feedbackStats[task]?.total ?? 0,
+    }
+    saveRetrainState(state)
+
+    return {
+      task,
+      success: false,
+      error: `New accuracy (${(newAccuracy * 100).toFixed(1)}%) lower than current (${(oldAccuracy * 100).toFixed(1)}%). Use --force to override.`,
+      oldAccuracy,
+      newAccuracy,
+      trainingDataCount: dataCount,
+      duration: Date.now() - startTime,
+    }
+  }
+
+  // Step 5: Copy new ONNX model to ~/.claude-brain/models/
+  const modelsDir = getHomePaths().models
+  if (!existsSync(modelsDir)) {
+    mkdirSync(modelsDir, { recursive: true })
+  }
+
+  const onnxSource = join(resolvedDir, 'models', `${task}.onnx`)
+  if (existsSync(onnxSource)) {
+    const onnxDest = join(modelsDir, `${task}.onnx`)
+    const onnxData = readFileSync(onnxSource)
+    writeFileSync(onnxDest, onnxData)
+    console.log(`  [${task}] Model copied to ${onnxDest}`)
+
+    // Update manifest
+    const currentManifest = loadManifest() ?? { models: {} }
+    const existing = currentManifest.models[task]
+    currentManifest.models[task] = {
+      version: bumpVersion(existing?.version),
+      file: `${task}.onnx`,
+      accuracy: newAccuracy,
+      labels: existing?.labels,
+      maxSeqLen: existing?.maxSeqLen,
+    }
+    saveManifest(currentManifest)
+    console.log(`  [${task}] Manifest updated`)
+  } else {
+    console.log(`  [${task}] Warning: ONNX file not found at ${onnxSource}`)
+  }
+
+  // Step 6: Update retrain state
+  const state = loadRetrainState()
+  const feedbackStats = getModelFeedbackStats()
+  state[task] = {
+    lastRetrain: new Date().toISOString(),
+    lastAccuracy: newAccuracy ?? oldAccuracy ?? 0,
+    feedbackAtRetrain: feedbackStats[task]?.total ?? 0,
+  }
+  saveRetrainState(state)
+
+  console.log(`  [${task}] Retrain complete in ${((Date.now() - startTime) / 1000).toFixed(1)}s`)
+
+  return {
+    task,
+    success: true,
+    oldAccuracy,
+    newAccuracy,
+    trainingDataCount: dataCount,
+    duration: Date.now() - startTime,
+  }
+}
+
+/**
+ * Run retrain for all tasks that need it.
+ */
+export async function retrainAll(
+  config: RetrainConfig
+): Promise<Map<string, RetrainResult>> {
+  const ALL_TASKS: ModelTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
+  const results = new Map<string, RetrainResult>()
+
+  for (const task of ALL_TASKS) {
+    const check = shouldRetrain(task, config)
+
+    if (!check.needed && !config.force) {
+      console.log(`  [${task}] Skipping — ${check.reason}`)
+      continue
+    }
+
+    if (config.force && !check.needed) {
+      console.log(`  [${task}] Forcing retrain (${check.reason})`)
+    }
+
+    const result = await retrainTask(task, config)
+    results.set(task, result)
+  }
+
+  return results
+}
+
+// ── Internal Helpers ─────────────────────────────────────────────────
+
+function bumpVersion(current?: string): string {
+  if (!current) return '1.0.0'
+  const parts = current.split('.').map(Number)
+  // Bump patch version
+  parts[2] = (parts[2] ?? 0) + 1
+  return parts.join('.')
+}