claude-brain 0.29.1 → 0.30.0

package/VERSION

@@ -1 +1 @@
-0.29.1
+0.30.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-brain",
-  "version": "0.29.1",
+  "version": "0.30.0",
   "description": "Local development assistant bridging Obsidian vaults with Claude Code via MCP",
   "type": "module",
   "main": "src/index.ts",
@@ -9,6 +9,7 @@
   },
   "files": [
     "src/**/*.ts",
+    "src/intelligence/hf-manifest.json",
     "src/hooks/claude-code-mastery.md",
     "scripts/postinstall.mjs",
     "packs/",

package/src/cli/commands/models.ts

@@ -16,10 +16,13 @@ import { join } from 'node:path'
 import { homedir } from 'node:os'
 import { parseArgs } from 'citty'
 import { renderLogo, theme, heading, dimText, successText, warningText, errorText, box, summaryPanel } from '@/cli/ui/index.js'
+import { progressBar } from '@/cli/ui/components.js'
 import { getHomePaths, getClaudeBrainHome } from '@/config/home'
 import { getTrainingStats, type TrainingTask } from '@/training/data-store'
 import type { ModelManifest, ModelManifestEntry, ModelTask } from '@/intelligence/model-manager'
 import { shouldRetrain, retrainTask, retrainAll, type RetrainConfig } from '@/training/retrain-pipeline'
+import { downloadFromHuggingFace, type HfManifest } from '@/intelligence/hf-downloader'
+import hfManifestData from '@/intelligence/hf-manifest.json'
 
 const ALL_TASKS: ModelTask[] = ['intent', 'entity', 'query', 'knowledge', 'compress', 'pattern']
 
@@ -38,7 +41,7 @@ export async function runModels() {
     subcommand: { type: 'positional', required: false, description: 'Subcommand: list, status, download, enable, disable, benchmark, stats, retrain' },
     taskArg: { type: 'positional', required: false, description: 'Task name or "all" (for enable/disable/benchmark/retrain)' },
     task: { type: 'string', description: 'Target task (for download --task)' },
-    source: { type: 'string', description: 'Source: local (default) or release' },
+    source: { type: 'string', description: 'Source: local (default) or hf (Hugging Face Hub)' },
     force: { type: 'boolean', description: 'Force retrain even if checks say not needed' },
   })
 
@@ -256,7 +259,7 @@ function loadManifest(): ModelManifest | null {
 
 // ─── download ─────────────────────────────────────────────────────
 
-function downloadModels(taskFilter: string, source: string) {
+async function downloadModels(taskFilter: string, source: string) {
   console.log()
   console.log(renderLogo())
   console.log()
@@ -280,17 +283,9 @@ function downloadModels(taskFilter: string, source: string) {
     console.log(successText(`Created models directory: ${paths.models}`))
   }
 
-  // Release source — not yet implemented
-  if (source === 'release') {
-    console.log(
-      box(
-        'Downloading from release artifacts is not yet available.\n' +
-        'Use --source local to install from ~/slm-training/models/ instead.',
-        'Coming Soon'
-      )
-    )
-    console.log()
-    return
+  // Hugging Face Hub source
+  if (source === 'hf' || source === 'release') {
+    return downloadFromHF(tasks, paths.models)
   }
 
   // Local source — copy from ~/slm-training/models/
@@ -409,6 +404,109 @@ function downloadModels(taskFilter: string, source: string) {
   console.log()
 }
 
+// ─── download from HF ─────────────────────────────────────────────
+
+const isTTY = process.stdout.isTTY === true
+
+async function downloadFromHF(tasks: ModelTask[], modelsDir: string) {
+  const manifest = hfManifestData as HfManifest
+
+  // Compute total download size
+  let totalSize = 0
+  for (const task of tasks) {
+    const entry = manifest.models[task]
+    if (entry) totalSize += entry.size
+  }
+
+  console.log(`  ${dimText('Source:')} Hugging Face Hub (${manifest.hfRepo})`)
+  console.log(`  ${dimText('Target:')} ${modelsDir}`)
+  console.log(`  ${dimText('Models:')} ${tasks.join(', ')}`)
+  console.log(`  ${dimText('Total:')}  ~${formatBytes(totalSize)}`)
+  console.log()
+
+  const results = await downloadFromHuggingFace(manifest, {
+    destDir: modelsDir,
+    tasks,
+    onProgress(task, downloaded, total) {
+      if (isTTY && total > 0) {
+        const pct = (downloaded / total) * 100
+        const bar = progressBar(pct, 25)
+        process.stdout.write(`\r  ${task.padEnd(12)} ${bar}  ${formatBytes(downloaded)} / ${formatBytes(total)}`)
+      }
+    },
+    onComplete(task, bytes) {
+      if (isTTY) process.stdout.write('\r' + ' '.repeat(80) + '\r')
+      console.log(`  ${successText(task.padEnd(12))} ${formatBytes(bytes)} ${dimText('SHA256 verified')}`)
+    },
+    onError(task, error) {
+      if (isTTY) process.stdout.write('\r' + ' '.repeat(80) + '\r')
+      console.log(`  ${errorText(task.padEnd(12))} ${error}`)
+    },
+  })
+
+  console.log()
+
+  const succeeded = results.filter(r => r.success)
+  if (succeeded.length === 0) {
+    console.log(warningText('No models were downloaded.'))
+    console.log()
+    return
+  }
+
+  // Build manifest from downloaded results + existing
+  const manifestPath = join(modelsDir, 'manifest.json')
+  const manifestModels: Partial<Record<ModelTask, ModelManifestEntry>> = {}
+
+  // Preserve existing manifest entries
+  if (existsSync(manifestPath)) {
+    try {
+      const existing: ModelManifest = JSON.parse(readFileSync(manifestPath, 'utf-8'))
+      if (existing.models) Object.assign(manifestModels, existing.models)
+    } catch { /* overwrite corrupt */ }
+  }
+
+  const installedTasks: ModelTask[] = []
+  let totalBytes = 0
+
+  for (const result of succeeded) {
+    const task = result.task as ModelTask
+    const entry = manifest.models[task]
+    if (!entry) continue
+
+    manifestModels[task] = {
+      version: entry.version,
+      file: entry.file,
+      sha256: entry.sha256,
+      params: entry.params,
+      accuracy: entry.accuracy ?? undefined,
+      labels: entry.labels,
+      maxSeqLen: entry.maxSeqLen,
+    }
+    installedTasks.push(task)
+    totalBytes += result.bytes
+  }
+
+  // Write manifest
+  const localManifest: ModelManifest = { models: manifestModels }
+  writeFileSync(manifestPath, JSON.stringify(localManifest, null, 2))
+
+  // Auto-enable models in config
+  const config = loadConfigFile()
+  if (!config.slm) config.slm = {}
+  config.slm.enabled = true
+  if (!config.slm.tasks) config.slm.tasks = {}
+  for (const task of installedTasks) {
+    config.slm.tasks[task] = 'model'
+  }
+  saveConfigFile(config)
+  updateConfigYml(installedTasks, 'model')
+
+  console.log(successText(`Downloaded ${succeeded.length} model${succeeded.length !== 1 ? 's' : ''} (${formatBytes(totalBytes)})`))
+  console.log(successText(`Auto-enabled ${installedTasks.join(', ')} in config`))
+  console.log(dimText(`Manifest written to ${manifestPath}`))
+  console.log()
+}
+
 // ─── enable ───────────────────────────────────────────────────────
 
 function enableTask(taskArg: string) {
@@ -783,7 +881,7 @@ function printModelsHelp() {
   const subcommands = [
     ['list', 'Show installed models and their status'],
     ['status', 'Show inference routing and ONNX runtime status'],
-    ['download', 'Download pre-trained models (--task <task>|all)'],
+    ['download', 'Download models (--source local|hf, --task <task>|all)'],
     ['enable <task|all>', 'Enable model inference for task(s)'],
     ['disable <task|all>', 'Disable model inference for task(s)'],
     ['benchmark <task>', 'Run accuracy benchmark on test data'],
@@ -805,6 +903,9 @@ function printModelsHelp() {
   console.log(theme.bold('Examples:'))
   console.log(`  ${dimText('claude-brain models list')}`)
   console.log(`  ${dimText('claude-brain models status')}`)
+  console.log(`  ${dimText('claude-brain models download --source hf')}`)
+  console.log(`  ${dimText('claude-brain models download --source hf --task intent')}`)
+  console.log(`  ${dimText('claude-brain models download --source local')}`)
   console.log(`  ${dimText('claude-brain models enable all')}`)
   console.log(`  ${dimText('claude-brain models enable intent')}`)
   console.log(`  ${dimText('claude-brain models disable pattern')}`)

package/src/config/loader.ts

@@ -148,7 +148,7 @@ export async function loadConfig(basePath: string = process.cwd()): Promise<Conf
   const fileConfig = loadFromFile(basePath)
   const envConfig = loadFromEnv()
 
-  const merged = mergeConfigs(defaultConfig as Partial<Config>, envConfig, fileConfig)
+  const merged = mergeConfigs(defaultConfig as Partial<Config>, fileConfig, envConfig)
 
   const result = ConfigSchema.safeParse(merged)
 

package/src/config/schema.ts

@@ -362,6 +362,8 @@ export const ConfigSchema = z.object({
     enabled: z.boolean().default(false),
     /** Directory containing ONNX model files */
     modelsDir: z.string().default('~/.claude-brain/models'),
+    /** Hugging Face repo for downloading pre-trained models */
+    hfRepo: z.string().default('demgun101/claude-brain-models'),
     /** Minimum model confidence to use model prediction (below → regex fallback) */
     confidenceThreshold: z.number().min(0).max(1).default(0.7),
     /** Per-task mode: 'model' uses model only, 'regex' uses regex only, 'both' runs both and logs comparison */

package/src/hooks/passive-classifier.ts

@@ -211,7 +211,7 @@ export class PassiveClassifier {
         const packages = match[1]?.trim()
         if (packages) {
           return {
-            type: 'progress',
+            type: 'decision',
             confidence: 0.85,
             content: `Installed package(s): ${packages}`,
             project: this.extractProjectFromCwd(input.cwd),

package/src/intelligence/hf-downloader.ts

@@ -0,0 +1,222 @@
+/**
+ * Hugging Face Hub Downloader — streams ONNX models from HF with SHA256 verification.
+ * Atomic writes via temp file + rename. Inline retry with exponential backoff.
+ */
+
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, renameSync, unlinkSync } from 'node:fs'
+import { join } from 'node:path'
+import type { ModelTask } from '@/intelligence/model-manager'
+
+export interface HfManifestEntry {
+  file: string
+  metaFile: string
+  sha256: string
+  metaSha256: string
+  size: number
+  version: string
+  params: string
+  accuracy: number | null
+  labels: string[]
+  maxSeqLen: number
+}
+
+export interface HfManifest {
+  hfRepo: string
+  hfBranch: string
+  models: Record<string, HfManifestEntry>
+}
+
+export interface DownloadOptions {
+  /** Target directory to write models into */
+  destDir: string
+  /** Tasks to download (subset of manifest keys) */
+  tasks: ModelTask[]
+  /** HF repo override (default from manifest) */
+  hfRepo?: string
+  /** HF branch override (default from manifest) */
+  hfBranch?: string
+  /** Progress callback: task name, bytes downloaded so far, total bytes */
+  onProgress?: (task: string, downloaded: number, total: number) => void
+  /** Called when a task completes */
+  onComplete?: (task: string, bytes: number) => void
+  /** Called on error */
+  onError?: (task: string, error: string) => void
+}
+
+export interface DownloadResult {
+  task: string
+  success: boolean
+  bytes: number
+  error?: string
+}
+
+const MAX_RETRIES = 3
+const RETRY_DELAYS = [2000, 4000, 8000]
+const DOWNLOAD_TIMEOUT_MS = 300_000 // 5 minutes per file
+
+/**
+ * Build HF resolve URL for a file.
+ * Pattern: https://huggingface.co/{repo}/resolve/{branch}/{filename}
+ */
+function hfUrl(repo: string, branch: string, filename: string): string {
+  return `https://huggingface.co/${repo}/resolve/${branch}/${filename}`
+}
+
+/**
+ * Stream-download a single file with SHA256 verification and atomic write.
+ * Returns the number of bytes written.
+ */
+async function downloadFile(
+  url: string,
+  destPath: string,
+  expectedSha256: string,
+  expectedSize: number,
+  onProgress?: (downloaded: number, total: number) => void,
+): Promise<number> {
+  const tempPath = `${destPath}.download`
+
+  // Clean up any leftover temp file
+  if (existsSync(tempPath)) {
+    unlinkSync(tempPath)
+  }
+
+  const response = await fetch(url, {
+    signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
+  })
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}: ${response.statusText}`)
+  }
+
+  if (!response.body) {
+    throw new Error('Response body is null')
+  }
+
+  const contentLength = parseInt(response.headers.get('content-length') || '0', 10) || expectedSize
+  const hash = createHash('sha256')
+  const writer = Bun.file(tempPath).writer()
+  let downloaded = 0
+
+  const reader = response.body.getReader()
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+
+      writer.write(value)
+      hash.update(value)
+      downloaded += value.byteLength
+
+      if (onProgress) {
+        onProgress(downloaded, contentLength)
+      }
+    }
+  } finally {
+    await writer.end()
+  }
+
+  // Verify SHA256
+  const actualSha256 = hash.digest('hex')
+  if (actualSha256 !== expectedSha256) {
+    // Clean up bad file
+    if (existsSync(tempPath)) unlinkSync(tempPath)
+    throw new Error(
+      `SHA256 mismatch: expected ${expectedSha256.slice(0, 12)}..., got ${actualSha256.slice(0, 12)}...`
+    )
+  }
+
+  // Atomic rename
+  renameSync(tempPath, destPath)
+  return downloaded
+}
+
+/**
+ * Download a single file with retry logic.
+ */
+async function downloadWithRetry(
+  url: string,
+  destPath: string,
+  expectedSha256: string,
+  expectedSize: number,
+  onProgress?: (downloaded: number, total: number) => void,
+): Promise<number> {
+  let lastError: Error | null = null
+
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      return await downloadFile(url, destPath, expectedSha256, expectedSize, onProgress)
+    } catch (err) {
+      lastError = err instanceof Error ? err : new Error(String(err))
+
+      // Clean up temp file on failure
+      const tempPath = `${destPath}.download`
+      if (existsSync(tempPath)) {
+        try { unlinkSync(tempPath) } catch { /* ignore */ }
+      }
+
+      if (attempt < MAX_RETRIES) {
+        await new Promise(resolve => setTimeout(resolve, RETRY_DELAYS[attempt]))
+      }
+    }
+  }
+
+  throw lastError!
+}
+
+/**
+ * Main entry point: download models from Hugging Face Hub.
+ */
+export async function downloadFromHuggingFace(
+  manifest: HfManifest,
+  options: DownloadOptions,
+): Promise<DownloadResult[]> {
+  const repo = options.hfRepo || manifest.hfRepo
+  const branch = options.hfBranch || manifest.hfBranch
+  const results: DownloadResult[] = []
+
+  // Ensure dest directory exists
+  if (!existsSync(options.destDir)) {
+    mkdirSync(options.destDir, { recursive: true })
+  }
+
+  for (const task of options.tasks) {
+    const entry = manifest.models[task]
+    if (!entry) {
+      const err = `No manifest entry for task "${task}"`
+      options.onError?.(task, err)
+      results.push({ task, success: false, bytes: 0, error: err })
+      continue
+    }
+
+    try {
+      // Download ONNX model
+      const modelUrl = hfUrl(repo, branch, entry.file)
+      const modelDest = join(options.destDir, entry.file)
+
+      const bytes = await downloadWithRetry(
+        modelUrl,
+        modelDest,
+        entry.sha256,
+        entry.size,
+        (downloaded, total) => options.onProgress?.(task, downloaded, total),
+      )
+
+      // Download metadata JSON
+      const metaUrl = hfUrl(repo, branch, entry.metaFile)
+      const metaDest = join(options.destDir, entry.metaFile)
+
+      await downloadWithRetry(metaUrl, metaDest, entry.metaSha256, 0)
+
+      options.onComplete?.(task, bytes)
+      results.push({ task, success: true, bytes })
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err)
+      options.onError?.(task, msg)
+      results.push({ task, success: false, bytes: 0, error: msg })
+    }
+  }
+
+  return results
+}

package/src/intelligence/hf-manifest.json

@@ -0,0 +1,78 @@
+{
+  "hfRepo": "demgun101/claude-brain-models",
+  "hfBranch": "main",
+  "models": {
+    "intent": {
+      "file": "intent.onnx",
+      "metaFile": "intent.json",
+      "sha256": "f276ec091712f53ceeccfdca70d468d0b5aa6da7ee5c4bd7867a7464a9732dd2",
+      "metaSha256": "d5702fd45e5685187e74810da75e41be9d12d9fd82b7ccd7244a9f0d33076c65",
+      "size": 120073283,
+      "version": "0.1.0",
+      "params": "nanoGPT-intent",
+      "accuracy": 0.9904,
+      "labels": ["session_start", "context_needed", "decision_made", "store_this", "pattern_found", "mistake_learned", "progress_update", "question", "comparison", "exploration", "list_all", "update_memory", "delete_memory", "detail_request", "timeline", "no_action"],
+      "maxSeqLen": 128
+    },
+    "entity": {
+      "file": "entity.onnx",
+      "metaFile": "entity.json",
+      "sha256": "d15706b78112e8dda3caa7e054237106b67200f90a1e6e0e4336526468dea8b5",
+      "metaSha256": "1de0636bc3bd9de23c0ce767a148d6becad7b6add6a0a717449a7ac185296d55",
+      "size": 120058476,
+      "version": "0.1.0",
+      "params": "nanoGPT-entity",
+      "accuracy": 0.9795,
+      "labels": ["O", "B-TECH", "I-TECH", "B-PROJECT", "I-PROJECT", "B-CONCEPT", "I-CONCEPT"],
+      "maxSeqLen": 128
+    },
+    "query": {
+      "file": "query.onnx",
+      "metaFile": "query.json",
+      "sha256": "802446105ad873b4a172dc875c07e245882f48691f72ae0abb0bda5934fa084a",
+      "metaSha256": "32393875ff58a87da83480673868e4ac14c3fd1a59e16836d7c1f2fa02a7f25e",
+      "size": 120057880,
+      "version": "0.1.0",
+      "params": "nanoGPT-query",
+      "accuracy": 0.985,
+      "labels": ["factual", "procedural", "comparative", "temporal", "exploratory", "decision"],
+      "maxSeqLen": 128
+    },
+    "knowledge": {
+      "file": "knowledge.onnx",
+      "metaFile": "knowledge.json",
+      "sha256": "a786eefc5ef5c6f2fc132c6de7f0972891057f5a993c9c2d9264207d36165035",
+      "metaSha256": "6aa77194cbd8c10a2451958ec5a3e43328df97ee8aea46abf53e3e432f3f3c4d",
+      "size": 120056340,
+      "version": "0.1.0",
+      "params": "nanoGPT-knowledge",
+      "accuracy": 0.998,
+      "labels": ["fact", "preference", "constraint", "goal", "definition"],
+      "maxSeqLen": 128
+    },
+    "compress": {
+      "file": "compress.onnx",
+      "metaFile": "compress.json",
+      "sha256": "2d950a0e0a2cdc5dc90b7c44803c5fb81c76d824bd208c64a7104c5845e1c237",
+      "metaSha256": "e91500c665ec47083bb6b3ff8c83d529f263792d83f7f169401267ceb1e8d031",
+      "size": 357902441,
+      "version": "0.1.0",
+      "params": "nanoGPT-compress",
+      "accuracy": null,
+      "labels": [],
+      "maxSeqLen": 256
+    },
+    "pattern": {
+      "file": "pattern.onnx",
+      "metaFile": "pattern.json",
+      "sha256": "3b44371eaef11fb8ccc4c1636d6cfbdcdf62ae8bc6ffc808ff0ff45b24824fde",
+      "metaSha256": "1126f3fa9a115b4d26063e006e7d33b9e82b7740194ac86b5dae9f25d5dfd1a2",
+      "size": 254526620,
+      "version": "0.1.0",
+      "params": "nanoGPT-pattern",
+      "accuracy": 0.8667,
+      "labels": ["solution", "anti-pattern", "best-practice", "common-issue"],
+      "maxSeqLen": 128
+    }
+  }
+}

package/src/intelligence/model-manager.ts

@@ -213,9 +213,32 @@ export class ModelManager {
   }
 
   /**
-   * Unload all models (for cleanup/testing)
+   * Unload a single model, releasing its ONNX session to free native memory.
    */
-  unloadAll(): void {
+  async unloadModel(task: ModelTask): Promise<void> {
+    const loaded = this.loadedModels.get(task)
+    if (!loaded) return
+
+    try {
+      await loaded.session?.release?.()
+    } catch (err) {
+      this.logger.warn({ error: err, task }, 'Failed to release ONNX session')
+    }
+    this.loadedModels.delete(task)
+    this.logger.debug({ task }, 'Model unloaded')
+  }
+
+  /**
+   * Unload all models, releasing ONNX sessions to free native memory.
+   */
+  async unloadAll(): Promise<void> {
+    for (const [task, loaded] of this.loadedModels) {
+      try {
+        await loaded.session?.release?.()
+      } catch (err) {
+        this.logger.warn({ error: err, task }, 'Failed to release ONNX session during unloadAll')
+      }
+    }
     this.loadedModels.clear()
     this.logger.debug('All models unloaded')
   }

package/src/server/services.ts

@@ -697,9 +697,9 @@ export async function shutdownServices(): Promise<void> {
     }
   }
 
-  // Unload SLM models
+  // Unload SLM models (releases ONNX sessions)
   if (services.modelManager) {
-    services.modelManager.unloadAll()
+    await services.modelManager.unloadAll()
     serviceLogger.info('SLM models unloaded')
   }