@nano-step/nano-brain 2026.1.14

package/src/types.ts

@@ -0,0 +1,215 @@
+export interface SearchResult {
+  id: string;
+  path: string;
+  collection: string;
+  title: string;
+  snippet: string;
+  score: number;
+  startLine: number;
+  endLine: number;
+  docid: string;
+  agent?: string;
+}
+
+export interface Document {
+  id: number;
+  collection: string;
+  path: string;
+  title: string;
+  hash: string;
+  agent?: string;
+  createdAt: string;
+  modifiedAt: string;
+  active: boolean;
+  projectHash?: string;
+}
+
+export interface MemoryChunk {
+  hash: string;
+  seq: number;
+  pos: number;
+  text: string;
+  startLine: number;
+  endLine: number;
+}
+
+export interface BreakPoint {
+  pos: number;
+  score: number;
+  type: string;
+  lineNo: number;
+}
+
+export interface CodeFenceRegion {
+  start: number;
+  end: number;
+}
+
+export interface Collection {
+  name: string;
+  path: string;
+  pattern: string;
+  context?: Record<string, string>;
+}
+
+export interface CollectionConfig {
+  globalContext?: string
+  collections: Record<string, {
+    path: string
+    pattern?: string
+    context?: Record<string, string>
+    update?: string
+  }>
+  storage?: {
+    maxSize?: string
+    retention?: string
+    minFreeDisk?: string
+  }
+  codebase?: CodebaseConfig
+  workspaces?: Record<string, WorkspaceConfig>
+  embedding?: EmbeddingConfig
+  watcher?: WatcherConfig
+}
+
+export interface CodebaseConfig {
+  enabled: boolean
+  root?: string
+  exclude?: string[]
+  extensions?: string[]
+  maxFileSize?: string
+  maxSize?: string
+  batchSize?: number
+}
+
+export interface WorkspaceConfig {
+  codebase?: CodebaseConfig
+}
+
+export interface EmbeddingConfig {
+  provider?: 'ollama' | 'local'
+  url?: string
+  model?: string
+}
+
+export interface WatcherConfig {
+  debounceMs?: number
+  pollIntervalMs?: number
+  sessionPollMs?: number
+  embedIntervalMs?: number
+}
+
+export interface CodebaseIndexResult {
+  filesScanned: number
+  filesIndexed: number
+  filesSkippedUnchanged: number
+  filesSkippedTooLarge: number
+  filesSkippedBudget: number
+  chunksCreated: number
+  storageUsedBytes: number
+  maxSizeBytes: number
+}
+
+export interface StorageConfig {
+  maxSize: number;
+  retention: number;
+  minFreeDisk: number;
+}
+
+export interface EmbeddingResult {
+  embedding: number[];
+  model: string;
+  dimensions: number;
+}
+
+export interface RerankResult {
+  results: Array<{
+    file: string;
+    score: number;
+    index: number;
+  }>;
+  model: string;
+}
+
+export interface RerankDocument {
+  text: string;
+  file: string;
+  index: number;
+}
+
+export interface HarvestedSession {
+  sessionId: string;
+  slug: string;
+  title: string;
+  agent: string;
+  date: string;
+  project: string;
+  projectHash: string;
+  messages: Array<{
+    role: 'user' | 'assistant';
+    agent?: string;
+    text: string;
+  }>;
+}
+
+export interface IndexHealth {
+  documentCount: number
+  embeddedCount: number
+  pendingEmbeddings: number
+  collections: Array<{
+    name: string
+    documentCount: number
+    path: string
+  }>
+  databaseSize: number
+  modelStatus: {
+    embedding: string
+    reranker: string
+    expander: string
+  }
+  workspaceStats?: Array<{ projectHash: string; count: number }>
+  codebase?: {
+    enabled: boolean
+    documents: number
+    chunks: number
+    extensions: string[]
+    excludeCount: number
+    storageUsed: number
+    maxSize: number
+  }
+}
+
+export interface Store {
+  close(): void;
+  
+  insertDocument(doc: Omit<Document, 'id'>): number;
+  findDocument(pathOrDocid: string): Document | null;
+  getDocumentBody(hash: string, fromLine?: number, maxLines?: number): string | null;
+  deactivateDocument(collection: string, path: string): void;
+  bulkDeactivateExcept(collection: string, activePaths: string[]): number;
+  
+  insertContent(hash: string, body: string): void;
+  
+  insertEmbedding(hash: string, seq: number, pos: number, embedding: number[], model: string): void;
+  ensureVecTable(dimensions: number): void;
+  
+  searchFTS(query: string, limit?: number, collection?: string, projectHash?: string): SearchResult[];
+  searchVec(query: string, embedding: number[], limit?: number, collection?: string, projectHash?: string): SearchResult[];
+  
+  getCachedResult(hash: string): string | null;
+  setCachedResult(hash: string, result: string): void;
+  
+  getIndexHealth(): IndexHealth;
+  getHashesNeedingEmbedding(projectHash?: string): Array<{ hash: string; body: string; path: string }>;
+  getNextHashNeedingEmbedding(projectHash?: string): { hash: string; body: string; path: string } | null;
+  getWorkspaceStats(): Array<{ projectHash: string; count: number }>;
+  
+  deleteDocumentsByPath(filePath: string): number;
+  cleanOrphanedEmbeddings(): number;
+  getCollectionStorageSize(collection: string): number;
+  
+  modelStatus: {
+    embedding: string;
+    reranker: string;
+    expander: string;
+  };
+}

package/src/watcher.ts

@@ -0,0 +1,389 @@
+import { watch, type FSWatcher } from 'chokidar';
+import type { Store, Collection, StorageConfig, CodebaseConfig } from './types.js'
+import { scanCollectionFiles } from './collections.js';
+import { indexDocument, computeHash, extractProjectHashFromPath } from './store.js';
+import { harvestSessions } from './harvester.js';
+import { checkDiskSpace, evictExpiredSessions, evictBySize } from './storage.js';
+import { indexCodebase, mergeExcludePatterns, resolveExtensions, embedPendingCodebase } from './codebase.js'
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+export interface WatcherOptions {
+  store: Store
+  collections: Collection[]
+  embedder?: { embed(text: string): Promise<{ embedding: number[] }> } | null
+  onUpdate?: (path: string) => void
+  debounceMs?: number
+  pollIntervalMs?: number
+  sessionPollMs?: number
+  embedIntervalMs?: number
+  sessionStorageDir?: string
+  outputDir?: string
+  storageConfig?: StorageConfig
+  dbPath?: string
+  codebaseConfig?: CodebaseConfig
+  workspaceRoot?: string
+  projectHash?: string
+}
+
+export interface Watcher {
+  stop(): void;
+  isDirty(): boolean;
+  triggerReindex(): Promise<void>;
+  getStats(): WatcherStats;
+}
+
+export interface WatcherStats {
+  filesWatched: number;
+  lastReindexAt: number | null;
+  pendingChanges: number;
+  isReindexing: boolean;
+}
+
+export function startWatcher(options: WatcherOptions): Watcher {
+  const {
+    store,
+    collections,
+    embedder,
+    onUpdate,
+    debounceMs = 2000,
+    pollIntervalMs = 300000,
+    sessionPollMs = 120000,
+    embedIntervalMs = 60000,
+    sessionStorageDir = path.join(os.homedir(), '.local/share/opencode/storage'),
+    outputDir = path.join(os.homedir(), '.nano-brain/sessions'),
+    storageConfig,
+    dbPath,
+    codebaseConfig,
+    workspaceRoot = process.cwd(),
+    projectHash = 'global',
+  } = options
+
+  const codebaseExtensions = codebaseConfig?.enabled
+    ? new Set(resolveExtensions(codebaseConfig, workspaceRoot))
+    : new Set<string>()
+
+  let dirty = false;
+  const pendingPaths = new Set<string>();
+  let lastReindexAt: number | null = null;
+  let isReindexing = false;
+  let stopped = false;
+  let debounceTimer: NodeJS.Timeout | null = null;
+  let pollInterval: NodeJS.Timeout | null = null;
+  let sessionPollInterval: NodeJS.Timeout | null = null;
+  let watcher: FSWatcher | null = null;
+  let harvestCycleCount = 0;
+  const watchedPaths = new Set<string>();
+  let embeddingInterval: NodeJS.Timeout | null = null;
+  let isEmbedding = false;
+
+  const handleFileChange = (filePath: string) => {
+    if (stopped) return
+    
+    dirty = true
+    pendingPaths.add(filePath)
+    if (debounceTimer) {
+      clearTimeout(debounceTimer)
+    }
+    debounceTimer = setTimeout(() => {
+      if (onUpdate) {
+        for (const p of pendingPaths) {
+          onUpdate(p)
+        }
+      }
+    }, debounceMs)
+  }
+
+  const isCodebaseFile = (filePath: string): boolean => {
+    if (!codebaseConfig?.enabled) return false
+    const ext = path.extname(filePath).toLowerCase()
+    return codebaseExtensions.has(ext)
+  }
+
+  const triggerReindex = async (): Promise<void> => {
+    if (isReindexing || stopped) return
+    
+    isReindexing = true
+    
+    try {
+      for (const collection of collections) {
+        const files = await scanCollectionFiles(collection)
+        const activePaths: string[] = []
+        for (const filePath of files) {
+          if (!fs.existsSync(filePath)) continue
+          
+          const content = fs.readFileSync(filePath, 'utf-8')
+          const hash = computeHash(content)
+          
+          const existingDoc = store.findDocument(filePath)
+          if (!existingDoc || existingDoc.hash !== hash) {
+            const title = extractTitle(content)
+            const effectiveProjectHash = collection.name === 'sessions'
+              ? extractProjectHashFromPath(filePath, outputDir) ?? projectHash
+              : projectHash;
+            indexDocument(store, collection.name, filePath, content, title, effectiveProjectHash)
+          }
+          
+          activePaths.push(filePath)
+        }
+        
+        store.bulkDeactivateExcept(collection.name, activePaths)
+      }
+      
+      if (codebaseConfig?.enabled) {
+        await indexCodebase(store, workspaceRoot, codebaseConfig, projectHash, embedder)
+      }
+      if (embedder) {
+        await embedPendingCodebase(store, embedder, 10, projectHash)
+      }
+      
+      dirty = false
+      pendingPaths.clear()
+      lastReindexAt = Date.now()
+    } finally {
+      isReindexing = false
+    }
+  }
+
+  const startupIntegrityCheck = async () => {
+    const health = store.getIndexHealth();
+    let mismatches = 0;
+    
+    for (const collectionInfo of health.collections) {
+      const collection = collections.find(c => c.name === collectionInfo.name);
+      if (!collection) continue;
+      
+      const files = await scanCollectionFiles(collection);
+      
+      for (const filePath of files) {
+        if (!fs.existsSync(filePath)) continue;
+        
+        const existingDoc = store.findDocument(filePath);
+        if (!existingDoc) continue;
+        
+        const content = fs.readFileSync(filePath, 'utf-8');
+        const hash = computeHash(content);
+        
+        if (existingDoc.hash !== hash) {
+          mismatches++;
+          dirty = true;
+          pendingPaths.add(filePath);
+        }
+      }
+    }
+    
+    if (mismatches > 0) {
+      console.log(`Integrity check: ${mismatches} file(s) need re-indexing`);
+    }
+  };
+
+  const setupWatcher = () => {
+    const pathsToWatch: string[] = []
+    const ignoredPatterns: (string | RegExp)[] = [/(^|[\/])\../]
+    for (const collection of collections) {
+      const expandedPath = collection.path.replace(/^~/, os.homedir())
+      if (fs.existsSync(expandedPath)) {
+        pathsToWatch.push(expandedPath)
+        watchedPaths.add(expandedPath)
+      }
+    }
+    if (codebaseConfig?.enabled && fs.existsSync(workspaceRoot)) {
+      pathsToWatch.push(workspaceRoot)
+      watchedPaths.add(workspaceRoot)
+      const excludePatterns = mergeExcludePatterns(codebaseConfig, workspaceRoot)
+      for (const pattern of excludePatterns) {
+        // Convert glob patterns to regex for chokidar directory-level matching
+        // e.g. 'node_modules' -> /[\/]node_modules([\/]|$)/
+        // e.g. '*.min.js' -> /\.min\.js$/
+        if (pattern.startsWith('*')) {
+          const escaped = pattern.slice(1).replace(/\./g, '\\.').replace(/\*/g, '.*')
+          ignoredPatterns.push(new RegExp(`${escaped}$`))
+        } else {
+          const escaped = pattern.replace(/\./g, '\\.').replace(/\*/g, '.*')
+          ignoredPatterns.push(new RegExp(`[\\/]${escaped}([\\/]|$)`))
+        }
+      }
+    }
+    if (pathsToWatch.length === 0) return
+    watcher = watch(pathsToWatch, {
+      ignored: ignoredPatterns,
+      persistent: true,
+      ignoreInitial: true,
+      awaitWriteFinish: {
+        stabilityThreshold: 100,
+        pollInterval: 100,
+      },
+    })
+    watcher.on('error', (err: unknown) => {
+      console.error(`[watcher] Error: ${err instanceof Error ? err.message : String(err)}`)
+    })
+    watcher.on('add', (filePath) => {
+      if (filePath.endsWith('.md') || isCodebaseFile(filePath)) {
+        handleFileChange(filePath)
+      }
+    })
+    watcher.on('change', (filePath) => {
+      if (filePath.endsWith('.md') || isCodebaseFile(filePath)) {
+        handleFileChange(filePath)
+      }
+    })
+    watcher.on('unlink', (filePath) => {
+      if (filePath.endsWith('.md') || isCodebaseFile(filePath)) {
+        handleFileChange(filePath)
+      }
+    })
+  }
+
+  const setupPolling = () => {
+    pollInterval = setInterval(async () => {
+      if (dirty && !isReindexing) {
+        await triggerReindex();
+      }
+    }, pollIntervalMs);
+    
+    sessionPollInterval = setInterval(async () => {
+      if (stopped) return;
+      if (storageConfig) {
+        const diskCheck = checkDiskSpace(outputDir, storageConfig.minFreeDisk);
+        if (!diskCheck.ok) {
+          console.warn(`[storage] Disk space critically low (<${Math.round(storageConfig.minFreeDisk / 1024 / 1024)}MB free), skipping writes`);
+          return;
+        }
+      }
+      
+      try {
+        const sessions = await harvestSessions({
+          sessionDir: sessionStorageDir,
+          outputDir,
+        });
+        
+        if (sessions.length > 0) {
+          await triggerReindex();
+        }
+        
+        if (storageConfig && dbPath) {
+          const expiredCount = evictExpiredSessions(outputDir, storageConfig.retention, store);
+          if (expiredCount > 0) {
+            console.log(`[storage] Evicted ${expiredCount} expired session(s)`);
+          }
+          
+          const sizeEvictedCount = evictBySize(outputDir, dbPath, storageConfig.maxSize, store);
+          if (sizeEvictedCount > 0) {
+            console.log(`[storage] Evicted ${sizeEvictedCount} session(s) due to size limit`);
+          }
+        }
+        
+        harvestCycleCount++;
+        if (harvestCycleCount % 10 === 0) {
+          const orphansDeleted = store.cleanOrphanedEmbeddings();
+          if (orphansDeleted > 0) {
+            console.log(`[storage] Cleaned ${orphansDeleted} orphaned embedding(s)`);
+          }
+        }
+      } catch (err) {
+        console.warn('Session harvest failed:', err);
+      }
+    }, sessionPollMs);
+
+    if (embedder) {
+      embeddingInterval = setInterval(async () => {
+        if (stopped || isEmbedding) return;
+        isEmbedding = true;
+        try {
+          const count = await embedPendingCodebase(store, embedder, 10, projectHash);
+          if (count > 0) {
+            console.log(`[embed] Embedded ${count} document(s)`);
+          }
+        } catch (err) {
+          console.warn('[embed] Embedding cycle failed:', err);
+        } finally {
+          isEmbedding = false;
+        }
+      }, embedIntervalMs);
+    }
+  };
+
+  setupWatcher();
+  setupPolling();
+  startupIntegrityCheck().catch(err => {
+    console.warn('Startup integrity check failed:', err);
+  });
+
+  if (embedder) {
+    setTimeout(async () => {
+      isEmbedding = true;
+      try {
+        const count = await embedPendingCodebase(store, embedder, 10, projectHash);
+        if (count > 0) {
+          console.log(`[embed] Initial embedding: ${count} document(s)`);
+        }
+      } catch (err) {
+        console.warn('[embed] Initial embedding failed:', err);
+      } finally {
+        isEmbedding = false;
+      }
+    }, 5000);
+  }
+
+  return {
+    stop() {
+      stopped = true;
+      
+      if (debounceTimer) {
+        clearTimeout(debounceTimer);
+        debounceTimer = null;
+      }
+      
+      if (pollInterval) {
+        clearInterval(pollInterval);
+        pollInterval = null;
+      }
+      
+      if (sessionPollInterval) {
+        clearInterval(sessionPollInterval);
+        sessionPollInterval = null;
+      }
+
+      if (embeddingInterval) {
+        clearInterval(embeddingInterval);
+        embeddingInterval = null;
+      }
+      
+      if (watcher) {
+        watcher.close();
+        watcher = null;
+      }
+    },
+    
+    isDirty() {
+      return dirty;
+    },
+    
+    async triggerReindex() {
+      await triggerReindex();
+    },
+    
+    getStats(): WatcherStats {
+      return {
+        filesWatched: watchedPaths.size,
+        lastReindexAt,
+        pendingChanges: pendingPaths.size,
+        isReindexing,
+      };
+    },
+  };
+}
+
+function extractTitle(content: string): string {
+  const lines = content.split('\n');
+  
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith('# ')) {
+      return trimmed.substring(2).trim();
+    }
+  }
+  
+  return 'Untitled';
+}