claude-brain 0.30.2 → 0.30.3

package/src/training/data-store.ts

@@ -1,298 +1,320 @@
-/**
- * Training Data Store — Phase 1A (SLM Upgrade)
- * Logs classification decisions to SQLite for model training.
- * Async, non-blocking — never impacts main request path.
- *
- * Table: training_data in ~/.claude-brain/data/memory.db
- */
-
-import { Database } from 'bun:sqlite'
-import { join } from 'node:path'
-import { existsSync, mkdirSync } from 'node:fs'
-import { getClaudeBrainHome } from '@/config/home'
-
-export type TrainingTask = 'intent' | 'entity' | 'query' | 'knowledge' | 'compress' | 'pattern'
-
-export interface TrainingEntry {
-  task: TrainingTask
-  input: string
-  output: string   // JSON-encoded: label, entities array, summary, etc.
-  metadata?: string // JSON-encoded: confidence, scores, timing
-}
-
-export interface ModelFeedbackEntry {
-  task: string
-  input: string
-  modelPrediction: string
-  modelConfidence: number
-  regexPrediction: string
-  actualLabel?: string
-}
-
-let db: Database | null = null
-let insertStmt: ReturnType<Database['prepare']> | null = null
-let feedbackInsertStmt: ReturnType<Database['prepare']> | null = null
-
-function getDb(): Database | null {
-  if (db) return db
-  try {
-    const dataDir = join(getClaudeBrainHome(), 'data')
-    if (!existsSync(dataDir)) {
-      mkdirSync(dataDir, { recursive: true })
-    }
-    const dbPath = join(dataDir, 'memory.db')
-    db = new Database(dbPath)
-    db.run('PRAGMA journal_mode = WAL')
-    ensureTable(db)
-    insertStmt = db.prepare(
-      'INSERT INTO training_data (task, input, output, metadata) VALUES (?, ?, ?, ?)'
-    )
-    feedbackInsertStmt = db.prepare(
-      'INSERT INTO model_feedback (task, input, model_prediction, model_confidence, regex_prediction, actual_label) VALUES (?, ?, ?, ?, ?, ?)'
-    )
-    return db
-  } catch {
-    return null
-  }
-}
-
-function ensureTable(database: Database): void {
-  database.run(`
-    CREATE TABLE IF NOT EXISTS training_data (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      task TEXT NOT NULL,
-      input TEXT NOT NULL,
-      output TEXT NOT NULL,
-      metadata TEXT,
-      verified INTEGER DEFAULT 0,
-      created_at TEXT DEFAULT (datetime('now'))
-    )
-  `)
-  // Indexes for efficient querying
-  database.run('CREATE INDEX IF NOT EXISTS idx_training_task ON training_data(task)')
-  database.run('CREATE INDEX IF NOT EXISTS idx_training_verified ON training_data(verified)')
-
-  // Phase 6A: Model feedback table for continuous learning loop
-  database.run(`
-    CREATE TABLE IF NOT EXISTS model_feedback (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      task TEXT NOT NULL,
-      input TEXT NOT NULL,
-      model_prediction TEXT NOT NULL,
-      model_confidence REAL NOT NULL,
-      regex_prediction TEXT NOT NULL,
-      actual_label TEXT,
-      created_at TEXT DEFAULT (datetime('now'))
-    )
-  `)
-  database.run('CREATE INDEX IF NOT EXISTS idx_feedback_task ON model_feedback(task)')
-}
-
-/**
- * Log a training example. Fire-and-forget — errors are silently swallowed.
- */
-export function logTrainingData(entry: TrainingEntry): void {
-  setImmediate(() => {
-    try {
-      const database = getDb()
-      if (!database || !insertStmt) return
-      insertStmt.run(entry.task, entry.input, entry.output, entry.metadata || null)
-    } catch {
-      // Never block or crash the main path
-    }
-  })
-}
-
-/**
- * Export training data as JSONL lines for a specific task.
- */
-export function exportTrainingData(
-  task: TrainingTask,
-  options?: { verifiedOnly?: boolean; limit?: number }
-): string[] {
-  const database = getDb()
-  if (!database) return []
-
-  let sql = 'SELECT input, output, metadata, verified, created_at FROM training_data WHERE task = ?'
-  const params: any[] = [task]
-
-  if (options?.verifiedOnly) {
-    sql += ' AND verified = 1'
-  }
-
-  sql += ' ORDER BY created_at DESC'
-
-  if (options?.limit) {
-    sql += ' LIMIT ?'
-    params.push(options.limit)
-  }
-
-  const rows = database.prepare(sql).all(...params) as any[]
-  return rows.map(row => JSON.stringify({
-    input: row.input,
-    output: JSON.parse(row.output),
-    metadata: row.metadata ? JSON.parse(row.metadata) : null,
-    verified: row.verified === 1,
-    created_at: row.created_at,
-  }))
-}
-
-/**
- * Get count of training examples per task.
- */
-export function getTrainingStats(): Record<TrainingTask, { total: number; verified: number }> {
-  const database = getDb()
-  const tasks: TrainingTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
-  const stats = {} as Record<TrainingTask, { total: number; verified: number }>
-
-  for (const task of tasks) {
-    if (!database) {
-      stats[task] = { total: 0, verified: 0 }
-      continue
-    }
-    const total = (database.prepare('SELECT COUNT(*) as c FROM training_data WHERE task = ?').get(task) as any)?.c || 0
-    const verified = (database.prepare('SELECT COUNT(*) as c FROM training_data WHERE task = ? AND verified = 1').get(task) as any)?.c || 0
-    stats[task] = { total, verified }
-  }
-
-  return stats
-}
-
-// ── Phase 6A: Model Feedback Functions ──────────────────────────────
-
-/**
- * Log a model vs regex comparison. Fire-and-forget — errors are silently swallowed.
- */
-export function logModelFeedback(entry: ModelFeedbackEntry): void {
-  setImmediate(() => {
-    try {
-      const database = getDb()
-      if (!database || !feedbackInsertStmt) return
-      feedbackInsertStmt.run(
-        entry.task,
-        entry.input,
-        entry.modelPrediction,
-        entry.modelConfidence,
-        entry.regexPrediction,
-        entry.actualLabel || null
-      )
-    } catch {
-      // Never block or crash the main path
-    }
-  })
-}
-
-/**
- * Get per-task feedback stats: total, agreements, disagreements, disagreement rate, reviewed count.
- */
-export function getModelFeedbackStats(): Record<string, {
-  total: number
-  agreements: number
-  disagreements: number
-  disagreementRate: number
-  reviewed: number
-}> {
-  const database = getDb()
-  const tasks: TrainingTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
-  const stats = {} as Record<string, {
-    total: number
-    agreements: number
-    disagreements: number
-    disagreementRate: number
-    reviewed: number
-  }>
-
-  for (const task of tasks) {
-    if (!database) {
-      stats[task] = { total: 0, agreements: 0, disagreements: 0, disagreementRate: 0, reviewed: 0 }
-      continue
-    }
-    const total = (database.prepare(
-      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ?'
-    ).get(task) as any)?.c || 0
-
-    const agreements = (database.prepare(
-      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ? AND model_prediction = regex_prediction'
-    ).get(task) as any)?.c || 0
-
-    const disagreements = total - agreements
-
-    const reviewed = (database.prepare(
-      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ? AND actual_label IS NOT NULL'
-    ).get(task) as any)?.c || 0
-
-    stats[task] = {
-      total,
-      agreements,
-      disagreements,
-      disagreementRate: total > 0 ? disagreements / total : 0,
-      reviewed,
-    }
-  }
-
-  return stats
-}
-
-/**
- * Export feedback as JSONL lines for a specific task.
- */
-export function exportModelFeedback(
-  task: string,
-  options?: { limit?: number }
-): string[] {
-  const database = getDb()
-  if (!database) return []
-
-  let sql = 'SELECT input, model_prediction, model_confidence, regex_prediction, actual_label, created_at FROM model_feedback WHERE task = ?'
-  const params: any[] = [task]
-
-  sql += ' ORDER BY created_at DESC'
-
-  if (options?.limit) {
-    sql += ' LIMIT ?'
-    params.push(options.limit)
-  }
-
-  const rows = database.prepare(sql).all(...params) as any[]
-  return rows.map(row => JSON.stringify({
-    input: row.input,
-    modelPrediction: row.model_prediction,
-    modelConfidence: row.model_confidence,
-    regexPrediction: row.regex_prediction,
-    actualLabel: row.actual_label,
-    createdAt: row.created_at,
-  }))
-}
-
-/**
- * Get the most recent disagreements for human review.
- */
-export function getDisagreements(
-  task: string,
-  limit: number = 50
-): Array<{
-  input: string
-  modelPrediction: string
-  modelConfidence: number
-  regexPrediction: string
-  createdAt: string
-}> {
-  const database = getDb()
-  if (!database) return []
-
-  const rows = database.prepare(`
-    SELECT input, model_prediction, model_confidence, regex_prediction, created_at
-    FROM model_feedback
-    WHERE task = ? AND model_prediction != regex_prediction
-    ORDER BY created_at DESC
-    LIMIT ?
-  `).all(task, limit) as any[]
-
-  return rows.map(row => ({
-    input: row.input,
-    modelPrediction: row.model_prediction,
-    modelConfidence: row.model_confidence,
-    regexPrediction: row.regex_prediction,
-    createdAt: row.created_at,
-  }))
-}
+/**
+ * Training Data Store — Phase 1A (SLM Upgrade)
+ * Logs classification decisions to SQLite for model training.
+ * Async, non-blocking — never impacts main request path.
+ *
+ * Table: training_data in ~/.claude-brain/data/memory.db
+ */
+
+import { Database } from 'bun:sqlite'
+import { join } from 'node:path'
+import { existsSync, mkdirSync } from 'node:fs'
+import { getClaudeBrainHome } from '@/config/home'
+
+/** Row from training_data table */
+interface TrainingDataRow {
+  input: string
+  output: string
+  metadata: string | null
+  verified: number
+  created_at: string
+}
+
+/** Count query result */
+interface CountResult { c: number }
+
+/** Row from model_feedback table */
+interface FeedbackRow {
+  input: string
+  model_prediction: string
+  model_confidence: number
+  regex_prediction: string
+  actual_label: string | null
+  created_at: string
+}
+
+export type TrainingTask = 'intent' | 'entity' | 'query' | 'knowledge' | 'compress' | 'pattern'
+
+export interface TrainingEntry {
+  task: TrainingTask
+  input: string
+  output: string   // JSON-encoded: label, entities array, summary, etc.
+  metadata?: string // JSON-encoded: confidence, scores, timing
+}
+
+export interface ModelFeedbackEntry {
+  task: string
+  input: string
+  modelPrediction: string
+  modelConfidence: number
+  regexPrediction: string
+  actualLabel?: string
+}
+
+let db: Database | null = null
+let insertStmt: ReturnType<Database['prepare']> | null = null
+let feedbackInsertStmt: ReturnType<Database['prepare']> | null = null
+
+function getDb(): Database | null {
+  if (db) return db
+  try {
+    const dataDir = join(getClaudeBrainHome(), 'data')
+    if (!existsSync(dataDir)) {
+      mkdirSync(dataDir, { recursive: true })
+    }
+    const dbPath = join(dataDir, 'memory.db')
+    db = new Database(dbPath)
+    db.run('PRAGMA journal_mode = WAL')
+    ensureTable(db)
+    insertStmt = db.prepare(
+      'INSERT INTO training_data (task, input, output, metadata) VALUES (?, ?, ?, ?)'
+    )
+    feedbackInsertStmt = db.prepare(
+      'INSERT INTO model_feedback (task, input, model_prediction, model_confidence, regex_prediction, actual_label) VALUES (?, ?, ?, ?, ?, ?)'
+    )
+    return db
+  } catch {
+    return null
+  }
+}
+
+function ensureTable(database: Database): void {
+  database.run(`
+    CREATE TABLE IF NOT EXISTS training_data (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      task TEXT NOT NULL,
+      input TEXT NOT NULL,
+      output TEXT NOT NULL,
+      metadata TEXT,
+      verified INTEGER DEFAULT 0,
+      created_at TEXT DEFAULT (datetime('now'))
+    )
+  `)
+  // Indexes for efficient querying
+  database.run('CREATE INDEX IF NOT EXISTS idx_training_task ON training_data(task)')
+  database.run('CREATE INDEX IF NOT EXISTS idx_training_verified ON training_data(verified)')
+
+  // Phase 6A: Model feedback table for continuous learning loop
+  database.run(`
+    CREATE TABLE IF NOT EXISTS model_feedback (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      task TEXT NOT NULL,
+      input TEXT NOT NULL,
+      model_prediction TEXT NOT NULL,
+      model_confidence REAL NOT NULL,
+      regex_prediction TEXT NOT NULL,
+      actual_label TEXT,
+      created_at TEXT DEFAULT (datetime('now'))
+    )
+  `)
+  database.run('CREATE INDEX IF NOT EXISTS idx_feedback_task ON model_feedback(task)')
+}
+
+/**
+ * Log a training example. Fire-and-forget — errors are silently swallowed.
+ */
+export function logTrainingData(entry: TrainingEntry): void {
+  setImmediate(() => {
+    try {
+      const database = getDb()
+      if (!database || !insertStmt) return
+      insertStmt.run(entry.task, entry.input, entry.output, entry.metadata || null)
+    } catch {
+      // Never block or crash the main path
+    }
+  })
+}
+
+/**
+ * Export training data as JSONL lines for a specific task.
+ */
+export function exportTrainingData(
+  task: TrainingTask,
+  options?: { verifiedOnly?: boolean; limit?: number }
+): string[] {
+  const database = getDb()
+  if (!database) return []
+
+  let sql = 'SELECT input, output, metadata, verified, created_at FROM training_data WHERE task = ?'
+  const params: (string | number)[] = [task]
+
+  if (options?.verifiedOnly) {
+    sql += ' AND verified = 1'
+  }
+
+  sql += ' ORDER BY created_at DESC'
+
+  if (options?.limit) {
+    sql += ' LIMIT ?'
+    params.push(options.limit)
+  }
+
+  const rows = database.prepare(sql).all(...params) as TrainingDataRow[]
+  return rows.map(row => JSON.stringify({
+    input: row.input,
+    output: JSON.parse(row.output),
+    metadata: row.metadata ? JSON.parse(row.metadata) : null,
+    verified: row.verified === 1,
+    created_at: row.created_at,
+  }))
+}
+
+/**
+ * Get count of training examples per task.
+ */
+export function getTrainingStats(): Record<TrainingTask, { total: number; verified: number }> {
+  const database = getDb()
+  const tasks: TrainingTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
+  const stats = {} as Record<TrainingTask, { total: number; verified: number }>
+
+  for (const task of tasks) {
+    if (!database) {
+      stats[task] = { total: 0, verified: 0 }
+      continue
+    }
+    const total = (database.prepare('SELECT COUNT(*) as c FROM training_data WHERE task = ?').get(task) as CountResult | null)?.c || 0
+    const verified = (database.prepare('SELECT COUNT(*) as c FROM training_data WHERE task = ? AND verified = 1').get(task) as CountResult | null)?.c || 0
+    stats[task] = { total, verified }
+  }
+
+  return stats
+}
+
+// ── Phase 6A: Model Feedback Functions ──────────────────────────────
+
+/**
+ * Log a model vs regex comparison. Fire-and-forget — errors are silently swallowed.
+ */
+export function logModelFeedback(entry: ModelFeedbackEntry): void {
+  setImmediate(() => {
+    try {
+      const database = getDb()
+      if (!database || !feedbackInsertStmt) return
+      feedbackInsertStmt.run(
+        entry.task,
+        entry.input,
+        entry.modelPrediction,
+        entry.modelConfidence,
+        entry.regexPrediction,
+        entry.actualLabel || null
+      )
+    } catch {
+      // Never block or crash the main path
+    }
+  })
+}
+
+/**
+ * Get per-task feedback stats: total, agreements, disagreements, disagreement rate, reviewed count.
+ */
+export function getModelFeedbackStats(): Record<string, {
+  total: number
+  agreements: number
+  disagreements: number
+  disagreementRate: number
+  reviewed: number
+}> {
+  const database = getDb()
+  const tasks: TrainingTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
+  const stats = {} as Record<string, {
+    total: number
+    agreements: number
+    disagreements: number
+    disagreementRate: number
+    reviewed: number
+  }>
+
+  for (const task of tasks) {
+    if (!database) {
+      stats[task] = { total: 0, agreements: 0, disagreements: 0, disagreementRate: 0, reviewed: 0 }
+      continue
+    }
+    const total = (database.prepare(
+      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ?'
+    ).get(task) as CountResult | null)?.c || 0
+
+    const agreements = (database.prepare(
+      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ? AND model_prediction = regex_prediction'
+    ).get(task) as CountResult | null)?.c || 0
+
+    const disagreements = total - agreements
+
+    const reviewed = (database.prepare(
+      'SELECT COUNT(*) as c FROM model_feedback WHERE task = ? AND actual_label IS NOT NULL'
+    ).get(task) as CountResult | null)?.c || 0
+
+    stats[task] = {
+      total,
+      agreements,
+      disagreements,
+      disagreementRate: total > 0 ? disagreements / total : 0,
+      reviewed,
+    }
+  }
+
+  return stats
+}
+
+/**
+ * Export feedback as JSONL lines for a specific task.
+ */
+export function exportModelFeedback(
+  task: string,
+  options?: { limit?: number }
+): string[] {
+  const database = getDb()
+  if (!database) return []
+
+  let sql = 'SELECT input, model_prediction, model_confidence, regex_prediction, actual_label, created_at FROM model_feedback WHERE task = ?'
+  const params: (string | number)[] = [task]
+
+  sql += ' ORDER BY created_at DESC'
+
+  if (options?.limit) {
+    sql += ' LIMIT ?'
+    params.push(options.limit)
+  }
+
+  const rows = database.prepare(sql).all(...params) as FeedbackRow[]
+  return rows.map(row => JSON.stringify({
+    input: row.input,
+    modelPrediction: row.model_prediction,
+    modelConfidence: row.model_confidence,
+    regexPrediction: row.regex_prediction,
+    actualLabel: row.actual_label,
+    createdAt: row.created_at,
+  }))
+}
+
+/**
+ * Get the most recent disagreements for human review.
+ */
+export function getDisagreements(
+  task: string,
+  limit: number = 50
+): Array<{
+  input: string
+  modelPrediction: string
+  modelConfidence: number
+  regexPrediction: string
+  createdAt: string
+}> {
+  const database = getDb()
+  if (!database) return []
+
+  const rows = database.prepare(`
+    SELECT input, model_prediction, model_confidence, regex_prediction, created_at
+    FROM model_feedback
+    WHERE task = ? AND model_prediction != regex_prediction
+    ORDER BY created_at DESC
+    LIMIT ?
+  `).all(task, limit) as FeedbackRow[]
+
+  return rows.map(row => ({
+    input: row.input,
+    modelPrediction: row.model_prediction,
+    modelConfidence: row.model_confidence,
+    regexPrediction: row.regex_prediction,
+    createdAt: row.created_at,
+  }))
+}