@nano-step/nano-brain 2026.1.14

package/src/codebase.ts

@@ -0,0 +1,425 @@
+import type { Store, CodebaseConfig, CodebaseIndexResult } from './types.js'
+import { computeHash } from './store.js'
+import { chunkSourceCode } from './chunker.js'
+import { parseSize } from './storage.js'
+import * as fs from 'fs'
+import * as path from 'path'
+import fg from 'fast-glob'
+
+const DEFAULT_MAX_FILE_SIZE = 5 * 1024 * 1024
+const DEFAULT_CODEBASE_MAX_SIZE = 2 * 1024 * 1024 * 1024
+
+const BUILTIN_EXCLUDE_PATTERNS = [
+  // Version control
+  '**/.git/**',
+  '**/.svn/**',
+  '**/.hg/**',
+
+  // JS/TS — dependencies
+  '**/node_modules/**',
+  '**/.pnpm-store/**',
+  '**/.yarn/**',
+  '**/bower_components/**',
+
+  // JS/TS — build outputs
+  '**/dist/**',
+  '**/build/**',
+  '**/out/**',
+  '**/output/**',
+  '**/.next/**',
+  '**/.nuxt/**',
+  '**/.svelte-kit/**',
+  '**/.astro/**',
+  '**/.remix/**',
+  '**/.turbo/**',
+  '**/.vercel/**',
+  '**/.output/**',
+  '**/.cache/**',
+  '**/.parcel-cache/**',
+  '**/.vite/**',
+  '**/storybook-static/**',
+
+  // JS/TS — generated files
+  '**/*.min.js',
+  '**/*.min.css',
+  '**/*.map',
+  '**/*.lock',
+  '**/*.tsbuildinfo',
+  '**/.eslintcache',
+
+  // Python
+  '**/__pycache__/**',
+  '**/.venv/**',
+  '**/venv/**',
+  '**/env/**',
+  '**/.env/**',
+  '**/.conda/**',
+  '**/*.egg-info/**',
+  '**/.mypy_cache/**',
+  '**/.ruff_cache/**',
+  '**/.pytest_cache/**',
+  '**/htmlcov/**',
+  '**/.tox/**',
+
+  // Go
+  '**/vendor/**',
+
+  // Rust
+  '**/target/**',
+
+  // Java/Kotlin/JVM
+  '**/.gradle/**',
+  '**/.mvn/**',
+  '**/gradle/wrapper/**',
+  '**/*.class',
+  '**/*.jar',
+  '**/*.war',
+
+  // Ruby
+  '**/gems/**',
+  '**/.bundle/**',
+
+  // PHP
+  '**/storage/framework/**',
+  '**/bootstrap/cache/**',
+
+  // Mobile — iOS
+  '**/Pods/**',
+  '**/*.xcworkspace/**',
+  '**/DerivedData/**',
+
+  // Mobile — Android
+  '**/.gradle/**',
+  '**/generated/**',
+
+  // DevOps / infra
+  '**/.terraform/**',
+  '**/.terraform.lock.hcl',
+  '**/terraform.tfstate*',
+
+  // Editors & IDEs
+  '**/.idea/**',
+  '**/.vscode/extensions/**',
+
+  // Logs & tmp
+  '**/logs/**',
+  '**/log/**',
+  '**/tmp/**',
+  '**/temp/**',
+  '**/*.log',
+
+  // Test coverage
+  '**/coverage/**',
+  '**/.nyc_output/**',
+  '**/lcov-report/**',
+
+  // Misc large binary/generated
+  '**/*.sum',
+  '**/*.snap',
+  '**/docker-data/**',
+]
+
+const PROJECT_TYPE_MARKERS: Record<string, string[]> = {
+  'package.json': ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'],
+  'pyproject.toml': ['.py', '.pyi'],
+  'setup.py': ['.py', '.pyi'],
+  'requirements.txt': ['.py', '.pyi'],
+  'go.mod': ['.go'],
+  'Cargo.toml': ['.rs'],
+  'pom.xml': ['.java', '.kt', '.kts'],
+  'build.gradle': ['.java', '.kt', '.kts'],
+  'build.gradle.kts': ['.java', '.kt', '.kts'],
+  'Gemfile': ['.rb', '.erb'],
+}
+
+export function detectProjectType(workspaceRoot: string): string[] {
+  const extensions = new Set<string>()
+  
+  for (const [marker, exts] of Object.entries(PROJECT_TYPE_MARKERS)) {
+    const markerPath = path.join(workspaceRoot, marker)
+    if (fs.existsSync(markerPath)) {
+      for (const ext of exts) {
+        extensions.add(ext)
+      }
+    }
+  }
+  
+  extensions.add('.md')
+  
+  if (extensions.size === 1) {
+    return ['.ts', '.tsx', '.js', '.jsx', '.py', '.go', '.rs', '.java', '.rb', '.md']
+  }
+  
+  return Array.from(extensions)
+}
+
+export function loadGitignorePatterns(workspaceRoot: string): string[] {
+  const gitignorePath = path.join(workspaceRoot, '.gitignore')
+  
+  if (!fs.existsSync(gitignorePath)) {
+    return []
+  }
+  
+  try {
+    const content = fs.readFileSync(gitignorePath, 'utf-8')
+    const patterns: string[] = []
+    
+    for (const line of content.split('\n')) {
+      const trimmed = line.trim()
+      if (trimmed && !trimmed.startsWith('#')) {
+        patterns.push(trimmed)
+      }
+    }
+    
+    return patterns
+  } catch {
+    return []
+  }
+}
+
+export function mergeExcludePatterns(config: CodebaseConfig, workspaceRoot: string): string[] {
+  const patterns = new Set<string>(BUILTIN_EXCLUDE_PATTERNS)
+  
+  const gitignorePatterns = loadGitignorePatterns(workspaceRoot)
+  for (const pattern of gitignorePatterns) {
+    patterns.add(pattern)
+  }
+  
+  if (config.exclude) {
+    for (const pattern of config.exclude) {
+      patterns.add(pattern)
+    }
+  }
+  
+  return Array.from(patterns)
+}
+
+export function resolveExtensions(config: CodebaseConfig, workspaceRoot: string): string[] {
+  if (config.extensions && config.extensions.length > 0) {
+    return config.extensions
+  }
+  
+  return detectProjectType(workspaceRoot)
+}
+
+export async function scanCodebaseFiles(
+  workspaceRoot: string,
+  config: CodebaseConfig
+): Promise<{ files: string[]; skippedTooLarge: number }> {
+  const extensions = resolveExtensions(config, workspaceRoot)
+  const excludePatterns = mergeExcludePatterns(config, workspaceRoot)
+  
+  const maxFileSize = config.maxFileSize
+    ? parseSize(config.maxFileSize)
+    : DEFAULT_MAX_FILE_SIZE
+  
+  const effectiveMaxSize = maxFileSize > 0 ? maxFileSize : DEFAULT_MAX_FILE_SIZE
+  
+  const patterns = extensions.map(ext => `**/*${ext}`)
+  
+  const allFiles = await fg(patterns, {
+    cwd: workspaceRoot,
+    absolute: true,
+    onlyFiles: true,
+    ignore: excludePatterns,
+  })
+  
+  const files: string[] = []
+  let skippedTooLarge = 0
+  
+  for (const filePath of allFiles) {
+    try {
+      const stats = fs.statSync(filePath)
+      if (stats.size <= effectiveMaxSize) {
+        files.push(filePath)
+      } else {
+        skippedTooLarge++
+      }
+    } catch {
+      continue
+    }
+  }
+  
+  return { files, skippedTooLarge }
+}
+
+export async function indexCodebase(
+  store: Store,
+  workspaceRoot: string,
+  config: CodebaseConfig,
+  projectHash: string,
+  _embedder?: { embed(text: string): Promise<{ embedding: number[] }> } | null
+): Promise<CodebaseIndexResult> {
+  const { files, skippedTooLarge } = await scanCodebaseFiles(workspaceRoot, config)
+  const maxSizeBytes = config.maxSize
+    ? parseSize(config.maxSize)
+    : DEFAULT_CODEBASE_MAX_SIZE
+  const effectiveMaxSize = maxSizeBytes > 0 ? maxSizeBytes : DEFAULT_CODEBASE_MAX_SIZE
+  const batchSize = config.batchSize ?? 50
+  let currentStorageUsed = store.getCollectionStorageSize('codebase')
+  let filesIndexed = 0
+  let filesSkippedUnchanged = 0
+  let filesSkippedBudget = 0
+  let chunksCreated = 0
+  const activePaths: string[] = []
+  let batchNum = 0
+  
+  for (let i = 0; i < files.length; i++) {
+    const filePath = files[i]
+    try {
+      const content = fs.readFileSync(filePath, 'utf-8')
+      const contentSize = Buffer.byteLength(content, 'utf-8')
+      const hash = computeHash(content)
+      const existingDoc = store.findDocument(filePath)
+      if (existingDoc && existingDoc.hash === hash) {
+        filesSkippedUnchanged++
+        activePaths.push(filePath)
+        continue
+      }
+      const existingSize = existingDoc ? Buffer.byteLength(store.getDocumentBody(existingDoc.hash) ?? '', 'utf-8') : 0
+      const netIncrease = contentSize - existingSize
+      if (currentStorageUsed + netIncrease > effectiveMaxSize) {
+        filesSkippedBudget++
+        if (existingDoc) {
+          activePaths.push(filePath)
+        }
+        continue
+      }
+      store.insertContent(hash, content)
+      const chunks = chunkSourceCode(content, hash, filePath, workspaceRoot)
+      chunksCreated += chunks.length
+      const title = path.basename(filePath)
+      const now = new Date().toISOString()
+      store.insertDocument({
+        collection: 'codebase',
+        path: filePath,
+        title,
+        hash,
+        createdAt: existingDoc?.createdAt ?? now,
+        modifiedAt: now,
+        active: true,
+        projectHash,
+      })
+      currentStorageUsed += netIncrease
+      filesIndexed++
+      activePaths.push(filePath)
+    } catch {
+      continue
+    }
+    
+    if ((i + 1) % batchSize === 0) {
+      batchNum++
+      console.error(`[codebase] Batch ${batchNum}: indexed ${i + 1}/${files.length} files`)
+      await new Promise(resolve => setImmediate(resolve))
+    }
+  }
+  
+  store.bulkDeactivateExcept('codebase', activePaths)
+  const finalStorageUsed = store.getCollectionStorageSize('codebase')
+  return {
+    filesScanned: files.length,
+    filesIndexed,
+    filesSkippedUnchanged,
+    filesSkippedTooLarge: skippedTooLarge,
+    filesSkippedBudget,
+    chunksCreated,
+    storageUsedBytes: finalStorageUsed,
+    maxSizeBytes: effectiveMaxSize,
+  }
+}
+
+const MAX_EMBED_CHARS = 1800
+
+function truncateForEmbedding(text: string): string {
+  if (text.length <= MAX_EMBED_CHARS) return text
+  return text.substring(0, MAX_EMBED_CHARS)
+}
+
+export async function embedPendingCodebase(
+  store: Store,
+  embedder: { embed(text: string): Promise<{ embedding: number[] }>; embedBatch?(texts: string[]): Promise<Array<{ embedding: number[] }>> },
+  batchSize: number = 10,
+  projectHash?: string
+): Promise<number> {
+  let embedded = 0
+  while (true) {
+    const batch: Array<{ hash: string; body: string; path: string }> = []
+    for (let i = 0; i < batchSize; i++) {
+      const row = store.getNextHashNeedingEmbedding(projectHash)
+      if (!row) break
+      batch.push(row)
+    }
+    if (batch.length === 0) break
+
+    const texts = batch.map(row => truncateForEmbedding(row.body))
+
+    console.error(`[embed] Batch ${batch.length} docs: ${batch.map((b, i) => `${b.path.split('/').pop()}(${texts[i].length}ch)`).join(', ')}`)
+    try {
+      if (embedder.embedBatch && batch.length > 1) {
+        const results = await embedder.embedBatch(texts)
+        for (let i = 0; i < batch.length; i++) {
+          store.insertEmbedding(batch[i].hash, 0, 0, results[i].embedding, 'nomic-embed-text-v1.5')
+        }
+        embedded += batch.length
+      } else {
+        for (let i = 0; i < batch.length; i++) {
+          try {
+            const result = await embedder.embed(texts[i])
+            store.insertEmbedding(batch[i].hash, 0, 0, result.embedding, 'nomic-embed-text-v1.5')
+            embedded++
+          } catch {
+            console.warn(`[embed] Failed to embed ${batch[i].path}, skipping`)
+            continue
+          }
+        }
+      }
+    } catch (err) {
+      console.warn('[embed] Batch failed, falling back to sequential:', err)
+      for (const item of batch) {
+        try {
+          const result = await embedder.embed(truncateForEmbedding(item.body))
+          store.insertEmbedding(item.hash, 0, 0, result.embedding, 'nomic-embed-text-v1.5')
+          embedded++
+        } catch {
+          console.warn(`[embed] Skipping ${item.path}`)
+          continue
+        }
+      }
+    }
+
+    if (embedded > 0 && embedded % 50 === 0) {
+      console.log(`[embed] Embedded ${embedded} document(s)...`)
+    }
+
+    await new Promise(resolve => setImmediate(resolve))
+  }
+  return embedded
+}
+
+export function getCodebaseStats(
+  store: Store,
+  config: CodebaseConfig | undefined,
+  workspaceRoot: string
+): { enabled: boolean; documents: number; chunks: number; extensions: string[]; excludeCount: number; storageUsed: number; maxSize: number } | undefined {
+  if (!config?.enabled) {
+    return undefined
+  }
+  const health = store.getIndexHealth()
+  const codebaseCollection = health.collections.find(c => c.name === 'codebase')
+  const extensions = resolveExtensions(config, workspaceRoot)
+  const excludePatterns = mergeExcludePatterns(config, workspaceRoot)
+  const storageUsed = store.getCollectionStorageSize('codebase')
+  const maxSize = config.maxSize
+    ? parseSize(config.maxSize)
+    : DEFAULT_CODEBASE_MAX_SIZE
+  const effectiveMaxSize = maxSize > 0 ? maxSize : DEFAULT_CODEBASE_MAX_SIZE
+  return {
+    enabled: true,
+    documents: codebaseCollection?.documentCount ?? 0,
+    chunks: 0,
+    extensions,
+    excludeCount: excludePatterns.length,
+    storageUsed,
+    maxSize: effectiveMaxSize,
+  }
+}

package/src/collections.ts

@@ -0,0 +1,217 @@
+import type { Collection, CollectionConfig, WorkspaceConfig } from './types.js';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { parse, stringify } from 'yaml';
+import fg from 'fast-glob';
+
+export function loadCollectionConfig(configPath: string): CollectionConfig | null {
+  if (!fs.existsSync(configPath)) {
+    return null;
+  }
+  
+  const content = fs.readFileSync(configPath, 'utf-8');
+  const config = parse(content) as CollectionConfig;
+  return config;
+}
+
+export function saveCollectionConfig(configPath: string, config: CollectionConfig): void {
+  const dir = path.dirname(configPath);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+  
+  const yaml = stringify(config);
+  fs.writeFileSync(configPath, yaml, 'utf-8');
+}
+
+export function getCollections(config: CollectionConfig): Collection[] {
+  const collections: Collection[] = [];
+  
+  for (const [name, collectionData] of Object.entries(config.collections)) {
+    collections.push({
+      name,
+      path: collectionData.path,
+      pattern: collectionData.pattern || '**/*.md',
+      context: collectionData.context,
+    });
+  }
+  
+  return collections;
+}
+
+export function addCollection(
+  configPath: string,
+  name: string,
+  collectionPath: string,
+  pattern?: string
+): CollectionConfig {
+  let config = loadCollectionConfig(configPath);
+  
+  if (!config) {
+    config = {
+      collections: {},
+    };
+  }
+  
+  config.collections[name] = {
+    path: collectionPath,
+    pattern: pattern || '**/*.md',
+    update: 'auto',
+  };
+  
+  saveCollectionConfig(configPath, config);
+  return config;
+}
+
+export function removeCollection(configPath: string, name: string): CollectionConfig {
+  const config = loadCollectionConfig(configPath);
+  
+  if (!config) {
+    throw new Error('Config file not found');
+  }
+  
+  delete config.collections[name];
+  
+  saveCollectionConfig(configPath, config);
+  return config;
+}
+
+export function listCollections(config: CollectionConfig): string[] {
+  return Object.keys(config.collections);
+}
+
+export function renameCollection(
+  configPath: string,
+  oldName: string,
+  newName: string
+): CollectionConfig {
+  const config = loadCollectionConfig(configPath);
+  
+  if (!config) {
+    throw new Error('Config file not found');
+  }
+  
+  if (!config.collections[oldName]) {
+    throw new Error(`Collection "${oldName}" not found`);
+  }
+  
+  config.collections[newName] = config.collections[oldName];
+  delete config.collections[oldName];
+  
+  saveCollectionConfig(configPath, config);
+  return config;
+}
+
+export function addContext(
+  configPath: string,
+  collectionName: string,
+  pathPrefix: string,
+  description: string
+): CollectionConfig {
+  const config = loadCollectionConfig(configPath);
+  
+  if (!config) {
+    throw new Error('Config file not found');
+  }
+  
+  if (!config.collections[collectionName]) {
+    throw new Error(`Collection "${collectionName}" not found`);
+  }
+  
+  if (!config.collections[collectionName].context) {
+    config.collections[collectionName].context = {};
+  }
+  
+  config.collections[collectionName].context![pathPrefix] = description;
+  
+  saveCollectionConfig(configPath, config);
+  return config;
+}
+
+export function findContextForPath(config: CollectionConfig, filePath: string): string | null {
+  let longestMatch: { prefix: string; description: string } | null = null;
+  
+  for (const collectionData of Object.values(config.collections)) {
+    if (!collectionData.context) {
+      continue;
+    }
+    
+    for (const [prefix, description] of Object.entries(collectionData.context)) {
+      if (filePath.includes(prefix)) {
+        if (!longestMatch || prefix.length > longestMatch.prefix.length) {
+          longestMatch = { prefix, description };
+        }
+      }
+    }
+  }
+  
+  return longestMatch ? longestMatch.description : null;
+}
+
+export function listAllContexts(
+  config: CollectionConfig
+): Array<{ collection: string; prefix: string; description: string }> {
+  const contexts: Array<{ collection: string; prefix: string; description: string }> = [];
+  
+  for (const [collectionName, collectionData] of Object.entries(config.collections)) {
+    if (!collectionData.context) {
+      continue;
+    }
+    
+    for (const [prefix, description] of Object.entries(collectionData.context)) {
+      contexts.push({
+        collection: collectionName,
+        prefix,
+        description,
+      });
+    }
+  }
+  
+  return contexts;
+}
+
+export async function scanCollectionFiles(collection: Collection): Promise<string[]> {
+  const expandedPath = collection.path.replace(/^~/, os.homedir());
+  
+  if (!fs.existsSync(expandedPath)) {
+    return [];
+  }
+  
+  const files = await fg(collection.pattern, {
+    cwd: expandedPath,
+    absolute: true,
+    onlyFiles: true,
+  });
+  
+  return files;
+}
+
+export function resolveCollectionPath(collection: Collection, basePath: string): string {
+  return collection.path;
+}
+
+export function getWorkspaceConfig(config: CollectionConfig | null, workspaceRoot: string): WorkspaceConfig {
+  // 1. Check workspaces map for exact match
+  if (config?.workspaces?.[workspaceRoot]) {
+    return config.workspaces[workspaceRoot]
+  }
+  // 2. Fall back to top-level codebase (backward compat)
+  if (config?.codebase) {
+    return { codebase: config.codebase }
+  }
+  // 3. Default: codebase enabled, auto-detect everything
+  return { codebase: { enabled: true } }
+}
+
+export function setWorkspaceConfig(configPath: string, workspaceRoot: string, wsConfig: WorkspaceConfig): void {
+  let config = loadCollectionConfig(configPath)
+  if (!config) {
+    config = { collections: {} }
+  }
+  if (!config.workspaces) {
+    config.workspaces = {}
+  }
+  config.workspaces[workspaceRoot] = wsConfig
+  saveCollectionConfig(configPath, config)
+}