nano-brain 2026.1.0

package/src/embeddings.ts

@@ -0,0 +1,293 @@
+import { getLlama } from 'node-llama-cpp';
+import { promises as fs } from 'fs';
+import { join, dirname } from 'path';
+import { homedir, cpus } from 'os';
+import type { EmbeddingResult, EmbeddingConfig } from './types.js';
+
+export interface EmbeddingProvider {
+  embed(text: string): Promise<EmbeddingResult>;
+  embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+  getDimensions(): number;
+  getModel(): string;
+  dispose(): void;
+}
+
+export interface EmbeddingProviderOptions {
+  modelPath?: string;
+  cacheDir?: string;
+  embeddingConfig?: EmbeddingConfig;
+}
+
+const DEFAULT_MODEL_URI = 'hf:nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.Q4_K_M.gguf';
+const MODEL_NAME = 'nomic-embed-text-v1.5';
+const DIMENSIONS = 768;
+
+interface ParsedModelURI {
+  org: string;
+  repo: string;
+  file: string;
+}
+
+function parseModelURI(uri: string): ParsedModelURI | null {
+  const match = uri.match(/^hf:([^/]+)\/([^/]+)\/(.+\.gguf)$/);
+  if (!match) return null;
+  return {
+    org: match[1],
+    repo: match[2],
+    file: match[3],
+  };
+}
+
+async function downloadModel(url: string, destPath: string): Promise<void> {
+  console.log(`Downloading model from ${url}...`);
+  
+  await fs.mkdir(dirname(destPath), { recursive: true });
+  
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download model: ${response.statusText}`);
+  }
+  
+  const totalSize = parseInt(response.headers.get('content-length') || '0', 10);
+  let downloadedSize = 0;
+  
+  const tempPath = `${destPath}.tmp`;
+  const fileHandle = await fs.open(tempPath, 'w');
+  
+  try {
+    const reader = response.body?.getReader();
+    if (!reader) throw new Error('No response body');
+    
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      
+      await fileHandle.write(value);
+      downloadedSize += value.length;
+      
+      if (totalSize > 0) {
+        const percent = ((downloadedSize / totalSize) * 100).toFixed(1);
+        process.stdout.write(`\rDownload progress: ${percent}%`);
+      }
+    }
+    
+    console.log('\nDownload complete');
+  } finally {
+    await fileHandle.close();
+  }
+  
+  await fs.rename(tempPath, destPath);
+}
+
+export async function resolveModelPath(
+  uri: string,
+  cacheDir?: string
+): Promise<string> {
+  const parsed = parseModelURI(uri);
+  if (!parsed) {
+    throw new Error(`Invalid model URI format: ${uri}`);
+  }
+  
+  const baseDir = cacheDir || join(homedir(), '.cache', 'nano-brain', 'models');
+  const modelPath = join(baseDir, parsed.org, parsed.repo, parsed.file);
+  
+  try {
+    await fs.access(modelPath);
+    return modelPath;
+  } catch {
+    const url = `https://huggingface.co/${parsed.org}/${parsed.repo}/resolve/main/${parsed.file}`;
+    await downloadModel(url, modelPath);
+    return modelPath;
+  }
+}
+
+function formatQueryPrompt(query: string): string {
+  return `search_query: ${query}`;
+}
+
+function formatDocumentPrompt(title: string, content: string): string {
+  return `search_document: ${content}`;
+}
+
+// Ollama's truncate:true is broken (github.com/ollama/ollama/issues/14186)
+// Client-side truncation: 1800 chars ≈ ~450 tokens, safe for 2048 context
+const OLLAMA_MAX_CHARS = 1800;
+
+function truncateForOllama(text: string): string {
+  if (text.length <= OLLAMA_MAX_CHARS) return text;
+  return text.substring(0, OLLAMA_MAX_CHARS);
+}
+class OllamaEmbeddingProvider implements EmbeddingProvider {
+  private url: string;
+  private model: string;
+  constructor(url: string, model: string) {
+    this.url = url.replace(/\/$/, '');
+    this.model = model;
+  }
+  async embed(text: string): Promise<EmbeddingResult> {
+    const response = await fetch(`${this.url}/api/embed`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: this.model,
+        input: [truncateForOllama(text)],
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Ollama embed failed: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json() as { embeddings: number[][] };
+    return {
+      embedding: data.embeddings[0],
+      model: this.model,
+      dimensions: data.embeddings[0].length,
+    };
+  }
+  async embedBatch(texts: string[]): Promise<EmbeddingResult[]> {
+    const response = await fetch(`${this.url}/api/embed`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: this.model,
+        input: texts.map(truncateForOllama),
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Ollama embedBatch failed: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json() as { embeddings: number[][] };
+    return data.embeddings.map(emb => ({
+      embedding: emb,
+      model: this.model,
+      dimensions: emb.length,
+    }));
+  }
+  getDimensions(): number {
+    return DIMENSIONS;
+  }
+  getModel(): string {
+    return this.model;
+  }
+  dispose(): void {
+  }
+}
+
+class EmbeddingProviderImpl implements EmbeddingProvider {
+  private contexts: any[] = [];
+  private currentContextIndex = 0;
+  
+  constructor(
+    private model: any,
+    private parallelism: number
+  ) {}
+  
+  async initialize(): Promise<void> {
+    for (let i = 0; i < this.parallelism; i++) {
+      const context = await this.model.createEmbeddingContext();
+      this.contexts.push(context);
+    }
+  }
+  
+  async embed(text: string): Promise<EmbeddingResult> {
+    const context = this.contexts[0];
+    const result = await context.getEmbeddingFor(text);
+    
+    return {
+      embedding: Array.from(result.vector),
+      model: MODEL_NAME,
+      dimensions: DIMENSIONS,
+    };
+  }
+  
+  async embedBatch(texts: string[]): Promise<EmbeddingResult[]> {
+    const results: EmbeddingResult[] = [];
+    const batchSize = Math.min(4, this.parallelism);
+    
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+      const batchPromises = batch.map(async (text, idx) => {
+        const contextIdx = idx % this.contexts.length;
+        const context = this.contexts[contextIdx];
+        const result = await context.getEmbeddingFor(text);
+        
+        return {
+          embedding: Array.from(result.vector) as number[],
+          model: MODEL_NAME,
+          dimensions: DIMENSIONS,
+        };
+      });
+      
+      const batchResults = await Promise.all(batchPromises);
+      results.push(...batchResults);
+    }
+    
+    return results;
+  }
+  
+  getDimensions(): number {
+    return DIMENSIONS;
+  }
+  
+  getModel(): string {
+    return MODEL_NAME;
+  }
+  
+  dispose(): void {
+    this.contexts = [];
+  }
+}
+
+export async function createEmbeddingProvider(
+  options?: EmbeddingProviderOptions
+): Promise<EmbeddingProvider | null> {
+  const config = options?.embeddingConfig;
+
+  // Try Ollama if configured (or by default)
+  if (!config || config.provider !== 'local') {
+    const url = config?.url || 'http://host.docker.internal:11434';
+    const model = config?.model || 'nomic-embed-text';
+
+    try {
+      // Health check — verify Ollama is reachable
+      const healthResp = await fetch(`${url}/api/tags`, { signal: AbortSignal.timeout(3000) });
+      if (healthResp.ok) {
+        const provider = new OllamaEmbeddingProvider(url, model);
+        // Verify the model works with a test embed
+        await provider.embed('test');
+        console.error(`[embedding] Using Ollama provider: ${model} at ${url}`);
+        return provider;
+      }
+    } catch (err) {
+      console.warn(`[embedding] Ollama not reachable at ${url}: ${err instanceof Error ? err.message : String(err)}`);
+      if (config?.provider === 'ollama') {
+        // Explicitly configured Ollama but it's not available
+        console.error('[embedding] Ollama explicitly configured but not reachable, no fallback');
+        return null;
+      }
+      console.warn('[embedding] Falling back to local node-llama-cpp...');
+    }
+  }
+
+  // Fallback to local node-llama-cpp
+  try {
+    const modelUri = options?.modelPath || DEFAULT_MODEL_URI;
+    const modelPath = await resolveModelPath(modelUri, options?.cacheDir);
+    const llama = await getLlama();
+    const model = await llama.loadModel({ modelPath });
+    const cpuCount = cpus().length;
+    const parallelism = Math.max(1, Math.min(4, Math.floor(cpuCount / 4)));
+    const provider = new EmbeddingProviderImpl(model, parallelism);
+    await provider.initialize();
+    console.error(`[embedding] Using local provider: ${MODEL_NAME}`);
+    return provider;
+  } catch (error) {
+    console.warn('Failed to load embedding model:', error instanceof Error ? error.message : String(error));
+    return null;
+  }
+}
+
+export { formatQueryPrompt, formatDocumentPrompt, parseModelURI };

package/src/expansion.ts

@@ -0,0 +1,79 @@
+import { getLlama } from 'node-llama-cpp';
+import { resolveModelPath } from './embeddings.js';
+
+export interface QueryExpander {
+  expand(query: string): Promise<string[]>;
+  dispose(): void;
+}
+
+export interface QueryExpanderOptions {
+  modelPath?: string;
+  cacheDir?: string;
+}
+
+const DEFAULT_MODEL_URI = 'hf:tobi/qmd-query-expansion-1.7B-GGUF/qmd-query-expansion-1.7B-Q8_0.gguf';
+const MODEL_NAME = 'qmd-query-expansion-1.7B';
+
+class QueryExpanderImpl implements QueryExpander {
+  constructor(
+    private model: any,
+    private context: any
+  ) {}
+  
+  async expand(query: string): Promise<string[]> {
+    try {
+      const prompt = `Generate 2 alternative search queries for: ${query}\n\n1.`;
+      
+      const result = await this.context.evaluate([prompt], {
+        maxTokens: 200,
+        temperature: 0.7,
+      });
+      
+      const generated = result?.text || '';
+      
+      const lines = generated.split('\n').filter(line => line.trim());
+      const variants: string[] = [];
+      
+      for (const line of lines) {
+        const match = line.match(/^\d+\.\s*(.+)$/);
+        if (match && match[1]) {
+          variants.push(match[1].trim());
+        }
+      }
+      
+      if (variants.length >= 2) {
+        return variants.slice(0, 2);
+      }
+      
+      return [query];
+    } catch (error) {
+      console.warn('Query expansion failed:', error instanceof Error ? error.message : String(error));
+      return [query];
+    }
+  }
+  
+  dispose(): void {
+    this.context = null;
+  }
+}
+
+export async function createQueryExpander(
+  options?: QueryExpanderOptions
+): Promise<QueryExpander | null> {
+  try {
+    const modelUri = options?.modelPath || DEFAULT_MODEL_URI;
+    const modelPath = await resolveModelPath(modelUri, options?.cacheDir);
+    
+    const llama = await getLlama();
+    const model = await llama.loadModel({ modelPath });
+    
+    const context = await model.createContext({
+      contextSize: 2048,
+    });
+    
+    return new QueryExpanderImpl(model, context);
+  } catch (error) {
+    console.warn('Failed to load query expander model:', error instanceof Error ? error.message : String(error));
+    return null;
+  }
+}

package/src/harvester.ts

@@ -0,0 +1,306 @@
+import { readFileSync, readdirSync, existsSync, mkdirSync, writeFileSync, statSync } from 'fs';
+import { join, dirname } from 'path';
+import { createHash } from 'crypto';
+import type { HarvestedSession } from './types.js';
+
+export interface HarvesterOptions {
+  sessionDir: string;
+  outputDir: string;
+  stateFile?: string;
+}
+
+interface SessionMetadata {
+  id: string;
+  slug: string;
+  title: string;
+  projectID: string;
+  directory: string;
+  created: number;
+}
+
+interface ParsedMessage {
+  id: string;
+  role: 'user' | 'assistant';
+  agent?: string;
+  created: number;
+}
+
+export function parseSession(sessionPath: string): SessionMetadata | null {
+  try {
+    if (!existsSync(sessionPath)) {
+      return null;
+    }
+    
+    const content = readFileSync(sessionPath, 'utf-8');
+    const data = JSON.parse(content);
+    
+    return {
+      id: data.id,
+      slug: data.slug || data.id || 'untitled',
+      title: data.title || '',
+      projectID: data.projectID,
+      directory: data.directory,
+      created: data.time?.created || 0
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function parseMessages(sessionId: string, storageDir: string): ParsedMessage[] {
+  const messageDir = join(storageDir, 'message', sessionId);
+  
+  if (!existsSync(messageDir)) {
+    return [];
+  }
+  
+  const messages: ParsedMessage[] = [];
+  
+  try {
+    const files = readdirSync(messageDir).filter(f => f.startsWith('msg_') && f.endsWith('.json'));
+    
+    for (const file of files) {
+      const filePath = join(messageDir, file);
+      const content = readFileSync(filePath, 'utf-8');
+      const data = JSON.parse(content);
+      
+      messages.push({
+        id: data.id,
+        role: data.role,
+        agent: data.agent,
+        created: data.time?.created || 0
+      });
+    }
+  } catch {
+    return [];
+  }
+  
+  messages.sort((a, b) => a.created - b.created);
+  
+  return messages;
+}
+
+export function parseParts(messageId: string, storageDir: string): string {
+  const partDir = join(storageDir, 'part', messageId);
+  
+  if (!existsSync(partDir)) {
+    return '';
+  }
+  
+  const textParts: string[] = [];
+  
+  try {
+    const files = readdirSync(partDir).filter(f => f.startsWith('prt_') && f.endsWith('.json'));
+    
+    for (const file of files) {
+      const filePath = join(partDir, file);
+      const content = readFileSync(filePath, 'utf-8');
+      const data = JSON.parse(content);
+      
+      if (data.type === 'text' && !data.synthetic && data.text) {
+        textParts.push(data.text);
+      }
+    }
+  } catch {
+    return '';
+  }
+  
+  return textParts.join('\n');
+}
+
+export function sessionToMarkdown(session: HarvestedSession): string {
+  const lines: string[] = [];
+  
+  lines.push('---');
+  lines.push(`session: ${session.sessionId}`);
+  lines.push(`agent: ${session.agent}`);
+  lines.push(`date: "${session.date}"`);
+  lines.push(`title: "${session.title}"`);
+  lines.push(`project: ${session.project}`);
+  lines.push(`projectHash: ${session.projectHash}`);
+  lines.push('---');
+  lines.push('');
+  
+  for (const message of session.messages) {
+    if (message.role === 'user') {
+      lines.push('## User');
+    } else {
+      const agentName = message.agent || 'assistant';
+      lines.push(`## Assistant (${agentName})`);
+    }
+    lines.push('');
+    lines.push(message.text);
+    lines.push('');
+  }
+  
+  return lines.join('\n');
+}
+
+export function getOutputPath(outputDir: string, projectPath: string, date: string, slug: string): string {
+  const hash = createHash('sha256').update(projectPath).digest('hex');
+  const projectHash = hash.substring(0, 12);
+  
+  const sanitizedSlug = (slug || 'untitled')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .replace(/-+/g, '-');
+  
+  return join(outputDir, projectHash, `${date}-${sanitizedSlug}.md`);
+}
+
+export function loadHarvestState(stateFile: string): Record<string, number> {
+  try {
+    if (!existsSync(stateFile)) {
+      return {};
+    }
+    
+    const content = readFileSync(stateFile, 'utf-8');
+    return JSON.parse(content);
+  } catch {
+    return {};
+  }
+}
+
+export function saveHarvestState(stateFile: string, state: Record<string, number>): void {
+  const dir = dirname(stateFile);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  
+  writeFileSync(stateFile, JSON.stringify(state, null, 2), 'utf-8');
+}
+
+export async function harvestSessions(options: HarvesterOptions): Promise<HarvestedSession[]> {
+  const { sessionDir, outputDir, stateFile: customStateFile } = options;
+  const stateFile = customStateFile || join(outputDir, '.harvest-state.json');
+  const state = loadHarvestState(stateFile);
+  const harvested: HarvestedSession[] = [];
+  
+  const sessionRoot = join(sessionDir, 'session');
+  
+  if (!existsSync(sessionRoot)) {
+    return [];
+  }
+  
+  const projectDirs = readdirSync(sessionRoot);
+  let stateChanged = false;
+  
+  for (const projectHash of projectDirs) {
+    const projectSessionDir = join(sessionRoot, projectHash);
+    
+    if (!existsSync(projectSessionDir)) {
+      continue;
+    }
+    
+    const sessionFiles = readdirSync(projectSessionDir).filter(f => f.startsWith('ses_') && f.endsWith('.json'));
+    
+    for (const sessionFile of sessionFiles) {
+      const sessionPath = join(projectSessionDir, sessionFile);
+      
+      const stat = statSync(sessionPath);
+      const lastMtime = stat.mtimeMs;
+      
+      // Check if already harvested AND output file still exists
+      if (state[sessionFile] && state[sessionFile] >= lastMtime) {
+        // Verify the output file actually exists — if not, re-harvest
+        const session = parseSession(sessionPath);
+        if (session) {
+          const date = new Date(session.created);
+          const dateStr = date.toISOString().split('T')[0];
+          const outputPath = getOutputPath(outputDir, session.directory, dateStr, session.slug);
+          if (existsSync(outputPath)) {
+            continue;
+          }
+          // Output file missing — fall through to re-harvest
+          console.log(`[harvester] Re-harvesting ${sessionFile}: output file missing`);
+        } else {
+          continue;
+        }
+      }
+      
+      const session = parseSession(sessionPath);
+      
+      if (!session) {
+        continue;
+      }
+      
+      const messages = parseMessages(session.id, sessionDir);
+      
+      // Skip sessions with no messages (nothing useful to index)
+      if (messages.length === 0) {
+        state[sessionFile] = lastMtime;
+        stateChanged = true;
+        continue;
+      }
+      
+      const parsedMessages = messages.map(msg => ({
+        role: msg.role,
+        agent: msg.agent,
+        text: parseParts(msg.id, sessionDir)
+      }));
+      
+      // Skip sessions where all messages have empty text
+      const hasContent = parsedMessages.some(m => m.text.trim().length > 0);
+      if (!hasContent) {
+        state[sessionFile] = lastMtime;
+        stateChanged = true;
+        continue;
+      }
+      
+      const date = new Date(session.created);
+      const dateStr = date.toISOString().split('T')[0];
+      
+      const hash = createHash('sha256').update(session.directory).digest('hex');
+      const projectHashStr = hash.substring(0, 12);
+      
+      const harvestedSession: HarvestedSession = {
+        sessionId: session.id,
+        slug: session.slug,
+        title: session.title,
+        agent: messages.find(m => m.role === 'assistant')?.agent || 'assistant',
+        date: dateStr,
+        project: session.directory,
+        projectHash: projectHashStr,
+        messages: parsedMessages
+      };
+      
+      const outputPath = getOutputPath(outputDir, session.directory, dateStr, session.slug);
+      const outputDirPath = dirname(outputPath);
+      
+      if (!existsSync(outputDirPath)) {
+        mkdirSync(outputDirPath, { recursive: true });
+      }
+      
+      const markdown = sessionToMarkdown(harvestedSession);
+      
+      try {
+        writeFileSync(outputPath, markdown, 'utf-8');
+        
+        // Verify the file was actually written before updating state
+        if (!existsSync(outputPath)) {
+          console.warn(`[harvester] Write succeeded but file not found: ${outputPath}`);
+          continue;
+        }
+        
+        harvested.push(harvestedSession);
+        state[sessionFile] = lastMtime;
+        stateChanged = true;
+      } catch (err) {
+        console.warn(`[harvester] Failed to write ${outputPath}:`, err);
+        // Do NOT update state — will retry on next cycle
+        continue;
+      }
+    }
+  }
+  
+  if (stateChanged) {
+    saveHarvestState(stateFile, state);
+  }
+  
+  if (harvested.length > 0) {
+    console.log(`[harvester] Harvested ${harvested.length} session(s)`);
+  }
+  
+  return harvested;
+}