@astrocyteai/local 0.1.0

package/src/search.ts

@@ -0,0 +1,249 @@
+/**
+ * Search engine — SQLite FTS5 full-text search.
+ *
+ * See docs/search-contract.md for behavior specification.
+ * All operations are sync. Uses better-sqlite3 for SQLite access.
+ */
+
+import Database from "better-sqlite3";
+import path from "node:path";
+import fs from "node:fs";
+import type { MemoryEntry, SearchHit } from "./types.js";
+import type { ContextTree } from "./context-tree.js";
+
+export class SearchEngine {
+  private db: Database.Database;
+
+  constructor(private dbPath: string) {
+    const dir = path.dirname(dbPath);
+    fs.mkdirSync(dir, { recursive: true });
+    this.db = new Database(dbPath);
+    this.db.pragma("journal_mode = WAL");
+    this.createTables();
+  }
+
+  private createTables(): void {
+    this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
+        id,
+        bank_id,
+        text,
+        tags,
+        domain,
+        memory_layer,
+        fact_type,
+        file_path,
+        tokenize='porter unicode61'
+      );
+    `);
+  }
+
+  /**
+   * Rebuild the FTS index from the Context Tree. Returns count indexed.
+   */
+  buildIndex(tree: ContextTree, bankId?: string): number {
+    if (bankId) {
+      this.db.prepare("DELETE FROM memory_fts WHERE bank_id = ?").run(bankId);
+    } else {
+      this.db.exec("DELETE FROM memory_fts");
+    }
+
+    const entries = tree.scanAll(bankId);
+    const insert = this.db.prepare(
+      `INSERT INTO memory_fts (id, bank_id, text, tags, domain, memory_layer, fact_type, file_path)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+    );
+
+    const insertMany = this.db.transaction((entries: MemoryEntry[]) => {
+      for (const entry of entries) {
+        insert.run(
+          entry.id,
+          entry.bank_id,
+          entry.text,
+          entry.tags.join(" "),
+          entry.domain,
+          entry.memory_layer,
+          entry.fact_type,
+          entry.file_path
+        );
+      }
+    });
+
+    insertMany(entries);
+    return entries.length;
+  }
+
+  /**
+   * Full-text search. Returns scored hits sorted by relevance.
+   */
+  search(
+    query: string,
+    bankId: string,
+    options?: {
+      limit?: number;
+      tags?: string[];
+      layers?: string[];
+    }
+  ): SearchHit[] {
+    const limit = options?.limit ?? 10;
+    const tags = options?.tags;
+    const layers = options?.layers;
+
+    if (query.trim() === "*") {
+      return this.searchAll(bankId, { limit, tags, layers });
+    }
+
+    const ftsQuery = SearchEngine.escapeFtsQuery(query);
+    if (!ftsQuery) return [];
+
+    let rows: Record<string, unknown>[];
+    try {
+      rows = this.db
+        .prepare(
+          `SELECT id, bank_id, text, tags, domain, memory_layer, fact_type, file_path, rank
+           FROM memory_fts
+           WHERE memory_fts MATCH ? AND bank_id = ?
+           ORDER BY rank
+           LIMIT ?`
+        )
+        .all(ftsQuery, bankId, limit * 3) as Record<string, unknown>[];
+    } catch {
+      return [];
+    }
+
+    let hits = this.rowsToHits(rows);
+
+    // Post-filter by tags
+    if (tags && tags.length > 0) {
+      const tagSet = new Set(tags);
+      hits = hits.filter(
+        (h) => h.tags && tagSet.size > 0 && [...tagSet].every((t) => h.tags.includes(t))
+      );
+    }
+
+    // Post-filter by layers
+    if (layers && layers.length > 0) {
+      hits = hits.filter((h) => h.memory_layer && layers.includes(h.memory_layer));
+    }
+
+    return hits.slice(0, limit);
+  }
+
+  /**
+   * Add a single entry to the index (incremental update).
+   */
+  addDocument(entry: MemoryEntry): void {
+    this.db
+      .prepare(
+        `INSERT INTO memory_fts (id, bank_id, text, tags, domain, memory_layer, fact_type, file_path)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+      )
+      .run(
+        entry.id,
+        entry.bank_id,
+        entry.text,
+        entry.tags.join(" "),
+        entry.domain,
+        entry.memory_layer,
+        entry.fact_type,
+        entry.file_path
+      );
+  }
+
+  /**
+   * Remove a single entry from the index.
+   */
+  removeDocument(entryId: string): void {
+    this.db.prepare("DELETE FROM memory_fts WHERE id = ?").run(entryId);
+  }
+
+  /**
+   * Close the database connection.
+   */
+  close(): void {
+    this.db.close();
+  }
+
+  // ── Internal ──
+
+  private searchAll(
+    bankId: string,
+    options: { limit: number; tags?: string[]; layers?: string[] }
+  ): SearchHit[] {
+    const rows = this.db
+      .prepare(
+        `SELECT id, bank_id, text, tags, domain, memory_layer, fact_type, file_path, 0 as rank
+         FROM memory_fts WHERE bank_id = ? LIMIT ?`
+      )
+      .all(bankId, options.limit) as Record<string, unknown>[];
+
+    let hits = this.rowsToHits(rows, 1.0);
+
+    if (options.tags && options.tags.length > 0) {
+      const tagSet = new Set(options.tags);
+      hits = hits.filter(
+        (h) => h.tags && [...tagSet].every((t) => h.tags.includes(t))
+      );
+    }
+
+    if (options.layers && options.layers.length > 0) {
+      hits = hits.filter(
+        (h) => h.memory_layer && options.layers!.includes(h.memory_layer)
+      );
+    }
+
+    return hits;
+  }
+
+  private rowsToHits(
+    rows: Record<string, unknown>[],
+    defaultScore?: number
+  ): SearchHit[] {
+    if (rows.length === 0) return [];
+
+    // Normalize BM25 scores (more negative = more relevant)
+    const rawScores = rows.map((r) => Math.abs(Number(r.rank)));
+    const maxScore = Math.max(...rawScores) || 1.0;
+
+    return rows.map((row) => {
+      const score =
+        defaultScore !== undefined
+          ? defaultScore
+          : maxScore > 0
+            ? Math.abs(Number(row.rank)) / maxScore
+            : 0.5;
+
+      const tagStr = (row.tags as string) || "";
+      const tags = tagStr.split(/\s+/).filter(Boolean);
+
+      return {
+        id: row.id as string,
+        text: row.text as string,
+        score,
+        bank_id: row.bank_id as string,
+        domain: row.domain as string,
+        file_path: row.file_path as string,
+        memory_layer: (row.memory_layer as string) || undefined,
+        fact_type: (row.fact_type as string) || undefined,
+        tags,
+      };
+    });
+  }
+
+  /**
+   * Escape special FTS5 characters for safe querying.
+   * Does NOT quote individual tokens — quoting disables stemming.
+   */
+  static escapeFtsQuery(query: string): string {
+    let cleaned = query
+      .replace(/"/g, " ")
+      .replace(/'/g, " ")
+      .replace(/\(/g, " ")
+      .replace(/\)/g, " ")
+      .replace(/:/g, " ")
+      .replace(/\^/g, " ");
+
+    const tokens = cleaned.split(/\s+/).filter(Boolean);
+    return tokens.join(" ");
+  }
+}

package/src/tiered-retrieval.ts

@@ -0,0 +1,229 @@
+/**
+ * Tiered retrieval for local Context Tree — cache → FTS5 → LLM-guided.
+ *
+ * 3-tier progressive escalation adapted for file-based storage:
+ *   Tier 0: In-memory result cache (exact/fuzzy query match)
+ *   Tier 1: FTS5 keyword search (standard)
+ *   Tier 2: LLM-guided query reformulation + FTS5 retry
+ *
+ * Stops when sufficient results are found. No embeddings needed.
+ */
+
+import type { SearchHit } from "./types.js";
+import type { SearchEngine } from "./search.js";
+import type { LLMProvider } from "./curated-retain.js";
+
+// ── Recall Cache ──
+
+interface CacheEntry {
+  query: string;
+  bankId: string;
+  hits: SearchHit[];
+  timestamp: number;
+}
+
+export class LocalRecallCache {
+  private cache = new Map<string, CacheEntry>();
+
+  constructor(
+    private maxEntries: number = 128,
+    private ttlMs: number = 120_000
+  ) {}
+
+  get(query: string, bankId: string): SearchHit[] | null {
+    const key = `${bankId}:${query.toLowerCase().trim()}`;
+    const entry = this.cache.get(key);
+    if (!entry) return null;
+
+    // Check TTL
+    if (performance.now() - entry.timestamp > this.ttlMs) {
+      this.cache.delete(key);
+      return null;
+    }
+
+    return entry.hits;
+  }
+
+  put(query: string, bankId: string, hits: SearchHit[]): void {
+    const key = `${bankId}:${query.toLowerCase().trim()}`;
+
+    // LRU eviction
+    while (this.cache.size >= this.maxEntries) {
+      const oldestKey = this.cache.keys().next().value!;
+      this.cache.delete(oldestKey);
+    }
+
+    this.cache.set(key, {
+      query,
+      bankId,
+      hits,
+      timestamp: performance.now(),
+    });
+  }
+
+  invalidateBank(bankId: string): void {
+    const prefix = `${bankId}:`;
+    for (const key of [...this.cache.keys()]) {
+      if (key.startsWith(prefix)) {
+        this.cache.delete(key);
+      }
+    }
+  }
+
+  invalidateAll(): void {
+    this.cache.clear();
+  }
+
+  size(): number {
+    return this.cache.size;
+  }
+}
+
+// ── Tiered Retriever ──
+
+export class LocalTieredRetriever {
+  readonly maxTier: number;
+
+  constructor(
+    private search: SearchEngine,
+    private cache: LocalRecallCache | null = null,
+    private llmProvider: LLMProvider | null = null,
+    private minResults: number = 2,
+    private minScore: number = 0.3,
+    maxTier: number = 1
+  ) {
+    this.maxTier = Math.min(maxTier, 2);
+  }
+
+  /**
+   * Run tiered retrieval. Returns [hits, tierUsed].
+   */
+  retrieve(
+    query: string,
+    bankId: string,
+    options?: { limit?: number; tags?: string[] }
+  ): [SearchHit[], number] {
+    const limit = options?.limit ?? 10;
+    const tags = options?.tags;
+
+    // ── Tier 0: Cache ──
+    if (this.cache && this.maxTier >= 0) {
+      const cached = this.cache.get(query, bankId);
+      if (cached !== null) {
+        return [cached.slice(0, limit), 0];
+      }
+    }
+
+    // ── Tier 1: FTS5 search ──
+    let hits: SearchHit[] = [];
+    if (this.maxTier >= 1) {
+      hits = this.search.search(query, bankId, { limit, tags });
+      if (this.sufficient(hits) || this.maxTier <= 1) {
+        if (this.cache && hits.length > 0) {
+          this.cache.put(query, bankId, hits);
+        }
+        return [hits, 1];
+      }
+    }
+
+    // ── Tier 2: LLM-guided reformulation (sync path — skip if in async context) ──
+    // TypeScript doesn't have asyncio.run() equivalent, so tier 2
+    // is only available via aretrieve(). Cache whatever we have.
+    if (this.cache && hits.length > 0) {
+      this.cache.put(query, bankId, hits);
+    }
+    return [hits.slice(0, limit), Math.max(this.maxTier >= 1 ? 1 : 0, 0)];
+  }
+
+  /**
+   * Async version of retrieve — supports LLM reformulation natively.
+   */
+  async aretrieve(
+    query: string,
+    bankId: string,
+    options?: { limit?: number; tags?: string[] }
+  ): Promise<[SearchHit[], number]> {
+    const limit = options?.limit ?? 10;
+    const tags = options?.tags;
+
+    // ── Tier 0: Cache ──
+    if (this.cache && this.maxTier >= 0) {
+      const cached = this.cache.get(query, bankId);
+      if (cached !== null) {
+        return [cached.slice(0, limit), 0];
+      }
+    }
+
+    // ── Tier 1: FTS5 search ──
+    let hits: SearchHit[] = [];
+    if (this.maxTier >= 1) {
+      hits = this.search.search(query, bankId, { limit, tags });
+      if (this.sufficient(hits) || this.maxTier <= 1) {
+        if (this.cache && hits.length > 0) {
+          this.cache.put(query, bankId, hits);
+        }
+        return [hits, 1];
+      }
+    }
+
+    // ── Tier 2: LLM-guided reformulation ──
+    if (this.maxTier >= 2 && this.llmProvider) {
+      const reformulated = await this.reformulate(query);
+      if (reformulated !== query) {
+        const hits2 = this.search.search(reformulated, bankId, { limit, tags });
+        const merged = LocalTieredRetriever.mergeHits(hits, hits2);
+        if (this.cache) {
+          this.cache.put(query, bankId, merged);
+        }
+        return [merged.slice(0, limit), 2];
+      }
+    }
+
+    // Cache whatever we have from tier 1
+    if (this.cache && hits.length > 0) {
+      this.cache.put(query, bankId, hits);
+    }
+    return [hits.slice(0, limit), Math.max(this.maxTier, 0)];
+  }
+
+  private sufficient(hits: SearchHit[]): boolean {
+    if (hits.length < this.minResults) return false;
+    const avgScore =
+      hits.reduce((sum, h) => sum + h.score, 0) / Math.max(hits.length, 1);
+    return avgScore >= this.minScore;
+  }
+
+  private async reformulate(query: string): Promise<string> {
+    if (!this.llmProvider) return query;
+
+    const prompt =
+      "Reformulate this search query to improve keyword-based search results. " +
+      "Add synonyms and related terms. Return only the reformulated query.\n\n" +
+      `Query: ${query}`;
+
+    try {
+      const completion = await this.llmProvider.complete({
+        messages: [{ role: "user", content: prompt }],
+        maxTokens: 100,
+        temperature: 0.3,
+      });
+      return completion.text.trim() || query;
+    } catch {
+      return query;
+    }
+  }
+
+  /**
+   * Merge two hit lists, deduplicate by ID, keep highest score.
+   */
+  static mergeHits(hitsA: SearchHit[], hitsB: SearchHit[]): SearchHit[] {
+    const best = new Map<string, SearchHit>();
+    for (const h of [...hitsA, ...hitsB]) {
+      const prev = best.get(h.id);
+      if (!prev || h.score > prev.score) {
+        best.set(h.id, h);
+      }
+    }
+    return [...best.values()].sort((a, b) => b.score - a.score);
+  }
+}

package/src/types.ts

@@ -0,0 +1,68 @@
+/**
+ * Shared types for astrocyte-local.
+ * Matches the Context Tree format spec in docs/context-tree-format.md.
+ */
+
+export interface MemoryEntry {
+  id: string;
+  bank_id: string;
+  text: string;
+  domain: string;
+  file_path: string;
+  memory_layer: "fact" | "observation" | "model";
+  fact_type: "world" | "experience" | "observation";
+  tags: string[];
+  created_at: string; // ISO 8601
+  updated_at: string;
+  occurred_at?: string;
+  recall_count: number;
+  last_recalled_at?: string;
+  source?: string;
+  metadata: Record<string, string | number | boolean | null>;
+}
+
+export interface SearchHit {
+  id: string;
+  text: string;
+  score: number; // 0.0 - 1.0 normalized
+  bank_id: string;
+  domain: string;
+  file_path: string;
+  memory_layer?: string;
+  fact_type?: string;
+  tags: string[];
+  occurred_at?: string;
+  metadata?: Record<string, string | number | boolean | null>;
+}
+
+export interface RetainResult {
+  stored: boolean;
+  memory_id: string;
+  domain: string;
+  file: string;
+  error?: string;
+}
+
+export interface RecallResult {
+  hits: SearchHit[];
+  total: number;
+}
+
+export interface BrowseResult {
+  path: string;
+  domains: string[];
+  entries: Array<{
+    file: string;
+    title: string;
+    memory_id: string;
+    recall_count: number;
+  }>;
+  total_memories: number;
+}
+
+export interface LocalConfig {
+  default_bank_id: string;
+  expose_reflect: boolean;
+  expose_forget: boolean;
+  root: string;
+}