@dot-ai/ext-sqlite-memory 0.9.0

package/src/index.spec.ts

@@ -0,0 +1,138 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { SqliteMemoryProvider } from './sqlite-memory.js';
+
+describe('SqliteMemoryProvider.describe()', () => {
+  let provider: SqliteMemoryProvider;
+
+  beforeEach(() => {
+    provider = new SqliteMemoryProvider({ path: ':memory:' });
+  });
+
+  afterEach(() => {
+    provider.close();
+  });
+
+  it('returns a string containing "SQLite"', () => {
+    const result = provider.describe();
+    expect(typeof result).toBe('string');
+    expect(result).toContain('SQLite');
+  });
+
+  it('returns a string containing "FTS5"', () => {
+    const result = provider.describe();
+    expect(result).toContain('FTS5');
+  });
+
+  it('shows "0 entries" for an empty database', () => {
+    const result = provider.describe();
+    expect(result).toContain('0 entries');
+  });
+
+  it('shows correct entry count after storing 3 entries', async () => {
+    await provider.store({ content: 'First memory entry about auth', type: 'log' });
+    await provider.store({ content: 'Second memory entry about routing', type: 'log' });
+    await provider.store({ content: 'Third memory entry about testing', type: 'log' });
+
+    const result = provider.describe();
+    expect(result).toContain('3 entries');
+  });
+});
+
+describe('SqliteMemoryProvider.search() — source field', () => {
+  let provider: SqliteMemoryProvider;
+
+  beforeEach(() => {
+    provider = new SqliteMemoryProvider({ path: ':memory:' });
+  });
+
+  afterEach(() => {
+    provider.close();
+  });
+
+  it('returns entries with source "sqlite-memory"', async () => {
+    await provider.store({ content: 'The authentication middleware was refactored', type: 'log' });
+    const results = await provider.search('authentication');
+
+    expect(results.length).toBeGreaterThan(0);
+    for (const entry of results) {
+      expect(entry.source).toBe('sqlite-memory');
+    }
+  });
+});
+
+describe('SqliteMemoryProvider store/search roundtrip', () => {
+  let provider: SqliteMemoryProvider;
+
+  beforeEach(() => {
+    provider = new SqliteMemoryProvider({ path: ':memory:' });
+  });
+
+  afterEach(() => {
+    provider.close();
+  });
+
+  it('stores an entry and retrieves it via search', async () => {
+    await provider.store({
+      content: 'Fixed the N+1 query in the task loader',
+      type: 'log',
+      date: '2026-03-04',
+    });
+
+    const results = await provider.search('task loader');
+
+    expect(results).toHaveLength(1);
+    expect(results[0].content).toContain('task loader');
+    expect(results[0].source).toBe('sqlite-memory');
+    expect(results[0].date).toBe('2026-03-04');
+  });
+
+  it('stores multiple entries and retrieves them all', async () => {
+    await provider.store({ content: 'Auth middleware fix applied', type: 'log' });
+    await provider.store({ content: 'Auth token expiry extended to 7 days', type: 'decision' });
+    await provider.store({ content: 'Database migration ran without issues', type: 'log' });
+
+    const results = await provider.search('auth');
+
+    expect(results.length).toBeGreaterThanOrEqual(2);
+    const contents = results.map(r => r.content);
+    expect(contents.some(c => c.includes('middleware fix'))).toBe(true);
+    expect(contents.some(c => c.includes('token expiry'))).toBe(true);
+  });
+
+  it('FTS5 search matches partial words', async () => {
+    await provider.store({ content: 'Authentication system redesigned', type: 'log' });
+
+    // FTS5 prefix search with trailing *
+    // The provider uses OR semantics — the word "authen" won't match without *
+    // But a full word "authentication" in the query should match
+    const results = await provider.search('authentication');
+
+    expect(results).toHaveLength(1);
+    expect(results[0].content).toContain('Authentication');
+  });
+
+  it('does not return unrelated entries', async () => {
+    await provider.store({ content: 'React component lifecycle overview', type: 'fact' });
+    await provider.store({ content: 'Vue router configuration patterns', type: 'fact' });
+    await provider.store({ content: 'Database indexing strategy', type: 'fact' });
+
+    const results = await provider.search('authentication bearer token');
+
+    expect(results).toHaveLength(0);
+  });
+
+  it('stores with labels and retrieves by label', async () => {
+    await provider.store({
+      content: 'Added input validation to all endpoints',
+      type: 'decision',
+      date: '2026-03-01',
+      labels: ['security', 'api'],
+    });
+
+    const results = await provider.search('unrelated query', ['security']);
+
+    expect(results).toHaveLength(1);
+    expect(results[0].labels).toContain('security');
+    expect(results[0].date).toBe('2026-03-01');
+  });
+});

package/src/index.ts

@@ -0,0 +1,2 @@
+// Public API — nothing exported yet.
+// Utilities can be exposed here when other extensions need to compose with this one.

package/src/migrate-files.ts

@@ -0,0 +1,193 @@
+#!/usr/bin/env node
+/**
+ * Migrate file-based memory to SQLite.
+ * Usage: npx tsx migrate-files.ts --root /path/to/workspace --db /path/to/memory.db
+ */
+import { readdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { SqliteMemoryProvider } from './sqlite-memory.js';
+
+// Parse a single memory markdown file into entries
+function parseMemoryFile(content: string, filename: string, node: string): Array<{
+  content: string;
+  type: string;
+  date: string | undefined;
+  labels: string[];
+  node: string;
+}> {
+  // Extract date from filename if it matches YYYY-MM-DD pattern
+  const dateMatch = filename.match(/^(\d{4}-\d{2}-\d{2})/);
+  const date = dateMatch ? dateMatch[1] : undefined;
+
+  // Split by ## headings into sections
+  const sections = content.split(/^## /m).filter(Boolean);
+
+  if (sections.length <= 1) {
+    // Single entry — whole file is one memory
+    return [{
+      content: content.trim().slice(0, 2000),
+      type: inferType(filename),
+      date,
+      labels: inferLabels(filename, content),
+      node,
+    }];
+  }
+
+  // Multiple sections — each ## heading becomes an entry
+  return sections.map(section => {
+    const lines = section.split('\n');
+    const heading = lines[0]?.trim() ?? '';
+    const body = lines.slice(1).join('\n').trim();
+    const entryContent = heading ? `## ${heading}\n${body}` : body;
+
+    return {
+      content: entryContent.slice(0, 2000),
+      type: inferType(filename),
+      date,
+      labels: inferLabels(filename, entryContent),
+      node,
+    };
+  }).filter(e => e.content.length > 10);
+}
+
+function inferType(filename: string): string {
+  if (filename.includes('lesson')) return 'lesson';
+  if (filename.match(/^\d{4}-\d{2}-\d{2}/)) return 'log';
+  if (filename.includes('research') || filename.includes('analysis')) return 'research';
+  return 'note';
+}
+
+function inferLabels(filename: string, _content: string): string[] {
+  const labels: string[] = [];
+  const parts = filename.replace('.md', '').split('-');
+  const meaningful = new Set(['cockpit', 'pro', 'roule', 'caillou', 'van', 'todo', 'home', 'assistant', 'dot', 'ai', 'nx', 'blog', 'property', 'poi', 'digest', 'email', 'api']);
+  for (const part of parts) {
+    if (meaningful.has(part.toLowerCase())) {
+      labels.push(part.toLowerCase());
+    }
+  }
+  return [...new Set(labels)];
+}
+
+async function scanMemoryDir(dir: string, node: string): Promise<Array<{ content: string; type: string; date?: string; labels: string[]; node: string }>> {
+  const entries: Array<{ content: string; type: string; date?: string; labels: string[]; node: string }> = [];
+
+  let files: string[];
+  try {
+    files = await readdir(dir);
+  } catch {
+    return entries;
+  }
+
+  for (const file of files) {
+    if (!file.endsWith('.md')) continue;
+    if (file === 'projects-index.md') continue;
+
+    try {
+      const content = await readFile(join(dir, file), 'utf-8');
+      const parsed = parseMemoryFile(content, file, node);
+      entries.push(...parsed);
+    } catch {
+      // Skip unreadable files
+    }
+  }
+
+  // Also scan _archive/ subdirectory
+  try {
+    const archiveFiles = await readdir(join(dir, '_archive'));
+    for (const file of archiveFiles) {
+      if (!file.endsWith('.md')) continue;
+      try {
+        const content = await readFile(join(dir, '_archive', file), 'utf-8');
+        const parsed = parseMemoryFile(content, file, node);
+        entries.push(...parsed);
+      } catch {
+        // Skip
+      }
+    }
+  } catch {
+    // No archive dir
+  }
+
+  // Also scan lessons/ subdirectory
+  try {
+    const lessonFiles = await readdir(join(dir, 'lessons'));
+    for (const file of lessonFiles) {
+      if (!file.endsWith('.md')) continue;
+      try {
+        const content = await readFile(join(dir, 'lessons', file), 'utf-8');
+        const parsed = parseMemoryFile(content, file, node);
+        for (const entry of parsed) {
+          entry.type = 'lesson';
+        }
+        entries.push(...parsed);
+      } catch {
+        // Skip
+      }
+    }
+  } catch {
+    // No lessons dir
+  }
+
+  return entries;
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const rootIdx = args.indexOf('--root');
+  const dbIdx = args.indexOf('--db');
+
+  const root = rootIdx >= 0 ? args[rootIdx + 1] : process.cwd();
+  const dbPath = dbIdx >= 0 ? args[dbIdx + 1] : join(root, '.ai', 'memory.db');
+
+  console.log(`Migrating memory files to SQLite`);
+  console.log(`  Root: ${root}`);
+  console.log(`  DB: ${dbPath}`);
+
+  // Create provider (creates DB and tables)
+  const provider = new SqliteMemoryProvider({ path: dbPath });
+
+  // Scan root memory
+  const rootEntries = await scanMemoryDir(join(root, '.ai', 'memory'), 'root');
+  console.log(`  Root memory: ${rootEntries.length} entries`);
+
+  // Scan project memories
+  let projectEntries = 0;
+  try {
+    const projects = await readdir(join(root, 'projects'));
+    for (const project of projects) {
+      const memDir = join(root, 'projects', project, '.ai', 'memory');
+      const entries = await scanMemoryDir(memDir, project);
+      if (entries.length > 0) {
+        console.log(`  ${project}: ${entries.length} entries`);
+        for (const entry of entries) {
+          await provider.store(entry);
+        }
+        projectEntries += entries.length;
+      }
+    }
+  } catch {
+    // No projects dir
+  }
+
+  // Store root entries
+  for (const entry of rootEntries) {
+    await provider.store(entry);
+  }
+
+  const total = rootEntries.length + projectEntries;
+  console.log(`\nMigrated ${total} entries total`);
+
+  // Test search
+  console.log(`\nTest search for "cockpit":`);
+  const results = await provider.search('cockpit');
+  console.log(`  Found ${results.length} results`);
+  if (results.length > 0) {
+    console.log(`  First: ${results[0].content.slice(0, 100)}...`);
+  }
+
+  provider.close();
+  console.log(`\nDone. DB saved to ${dbPath}`);
+}
+
+main().catch(console.error);

package/src/sqlite-memory.ts

@@ -0,0 +1,242 @@
+import Database from 'better-sqlite3';
+import { join } from 'node:path';
+import type { MemoryEntry } from '@dot-ai/core';
+
+/**
+ * Compute Jaccard similarity between two strings (word-level).
+ * similarity = |intersection| / |union|
+ */
+function jaccardSimilarity(a: string, b: string): number {
+  const wordsA = new Set(
+    a.toLowerCase().replace(/[^\w\s]/g, ' ').split(/\s+/).filter(w => w.length > 1)
+  );
+  const wordsB = new Set(
+    b.toLowerCase().replace(/[^\w\s]/g, ' ').split(/\s+/).filter(w => w.length > 1)
+  );
+  if (wordsA.size === 0 && wordsB.size === 0) return 1;
+  if (wordsA.size === 0 || wordsB.size === 0) return 0;
+
+  let intersection = 0;
+  for (const w of wordsA) {
+    if (wordsB.has(w)) intersection++;
+  }
+  const union = wordsA.size + wordsB.size - intersection;
+  return intersection / union;
+}
+
+export class SqliteMemoryProvider {
+  private db: Database.Database;
+
+  constructor(options: Record<string, unknown> = {}) {
+    const root = (options.root as string) ?? process.cwd();
+    const rawPath = (options.path as string) ?? ':memory:';
+    const dbPath = rawPath === ':memory:' ? rawPath : (rawPath.startsWith('/') ? rawPath : join(root, rawPath));
+    this.db = new Database(dbPath);
+    this.db.pragma('journal_mode = WAL');
+    this.init();
+  }
+
+  private init(): void {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS memories (
+        id           INTEGER PRIMARY KEY AUTOINCREMENT,
+        content      TEXT    NOT NULL,
+        type         TEXT    NOT NULL DEFAULT 'log',
+        date         TEXT,
+        labels       TEXT    DEFAULT '[]',
+        node         TEXT,
+        source       TEXT    DEFAULT 'sqlite-memory',
+        created      INTEGER DEFAULT (unixepoch())
+      );
+
+      CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+        content,
+        labels,
+        node,
+        content='memories',
+        content_rowid='id'
+      );
+
+      CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+        INSERT INTO memories_fts(rowid, content, labels, node)
+        VALUES (new.id, new.content, new.labels, new.node);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+        INSERT INTO memories_fts(memories_fts, rowid, content, labels, node)
+        VALUES ('delete', old.id, old.content, old.labels, old.node);
+      END;
+    `);
+
+    // Graceful migration: add lifecycle columns if they don't exist
+    const existingCols = (this.db.prepare(`PRAGMA table_info(memories)`).all() as Array<{ name: string }>)
+      .map(c => c.name);
+
+    if (!existingCols.includes('score')) {
+      this.db.exec(`ALTER TABLE memories ADD COLUMN score REAL DEFAULT 1.0`);
+    }
+    if (!existingCols.includes('last_recalled')) {
+      this.db.exec(`ALTER TABLE memories ADD COLUMN last_recalled TEXT`);
+    }
+    if (!existingCols.includes('recall_count')) {
+      this.db.exec(`ALTER TABLE memories ADD COLUMN recall_count INTEGER DEFAULT 0`);
+    }
+  }
+
+  async store(entry: Omit<MemoryEntry, 'source'>): Promise<void> {
+    const content = entry.content;
+
+    // Extract key terms for FTS dedup check
+    const queryWords = content
+      .replace(/[^\w\s]/g, ' ')
+      .split(/\s+/)
+      .filter(w => w.length > 1);
+
+    if (queryWords.length > 0) {
+      const ftsQuery = queryWords.join(' OR ');
+      let candidates: Array<{ id: number; content: string }> = [];
+      try {
+        candidates = this.db.prepare(`
+          SELECT m.id, m.content
+          FROM memories_fts
+          JOIN memories m ON m.id = memories_fts.rowid
+          WHERE memories_fts MATCH ?
+          LIMIT 10
+        `).all(ftsQuery) as typeof candidates;
+      } catch {
+        // FTS error — fall through to insert
+      }
+
+      for (const candidate of candidates) {
+        const similarity = jaccardSimilarity(content, candidate.content);
+        if (similarity > 0.85) {
+          // Duplicate found — update existing entry instead of inserting
+          this.db.prepare(
+            `UPDATE memories SET content = ?, date = ?, score = MIN(score + 0.1, 5.0) WHERE id = ?`
+          ).run(
+            content,
+            entry.date ?? new Date().toISOString().slice(0, 10),
+            candidate.id,
+          );
+          // Update FTS index for the modified row
+          this.db.prepare(
+            `INSERT INTO memories_fts(memories_fts, rowid, content, labels, node) VALUES ('delete', ?, ?, ?, ?)`
+          ).run(candidate.id, candidate.content, '[]', null);
+          const updated = this.db.prepare(`SELECT content, labels, node FROM memories WHERE id = ?`).get(candidate.id) as { content: string; labels: string; node: string | null };
+          this.db.prepare(
+            `INSERT INTO memories_fts(rowid, content, labels, node) VALUES (?, ?, ?, ?)`
+          ).run(candidate.id, updated.content, updated.labels, updated.node);
+          return;
+        }
+      }
+    }
+
+    // No duplicate found — insert new entry
+    const stmt = this.db.prepare(
+      'INSERT INTO memories (content, type, date, labels, node, score, recall_count) VALUES (?, ?, ?, ?, ?, 1.0, 0)'
+    );
+    stmt.run(
+      entry.content,
+      entry.type ?? 'log',
+      entry.date ?? new Date().toISOString().slice(0, 10),
+      JSON.stringify(entry.labels ?? []),
+      entry.node ?? null,
+    );
+  }
+
+  async search(query: string, labels?: string[]): Promise<MemoryEntry[]> {
+    const queryWords = query
+      .replace(/[^\w\s]/g, ' ')
+      .split(/\s+/)
+      .filter(w => w.length > 1);
+
+    const labelWords = (labels ?? [])
+      .map(l => l.replace(/[^\w\s]/g, '').trim())
+      .filter(w => w.length > 1);
+
+    const allTerms = [...new Set([...queryWords, ...labelWords])];
+    const cleanQuery = allTerms.join(' OR ');
+
+    if (!cleanQuery) return [];
+
+    let rows: Array<{
+      id: number;
+      content: string;
+      type: string;
+      date: string | null;
+      labels: string;
+      node: string | null;
+      score: number;
+      rank: number;
+    }>;
+
+    try {
+      rows = this.db.prepare(`
+        SELECT m.id, m.content, m.type, m.date, m.labels, m.node, m.score,
+               bm25(memories_fts) AS rank
+        FROM memories_fts
+        JOIN memories m ON m.id = memories_fts.rowid
+        WHERE memories_fts MATCH ?
+        ORDER BY rank
+        LIMIT 20
+      `).all(cleanQuery) as typeof rows;
+    } catch {
+      return [];
+    }
+
+    // Score bump: increment score, update last_recalled, increment recall_count
+    if (rows.length > 0) {
+      const today = new Date().toISOString().slice(0, 10);
+      const updateStmt = this.db.prepare(`
+        UPDATE memories
+        SET score = MIN(score + 0.1, 5.0),
+            last_recalled = ?,
+            recall_count = recall_count + 1
+        WHERE id = ?
+      `);
+      for (const row of rows) {
+        updateStmt.run(today, row.id);
+      }
+    }
+
+    return rows.map(row => ({
+      content: row.content,
+      type: row.type,
+      source: 'sqlite-memory' as const,
+      date: row.date ?? undefined,
+      labels: JSON.parse(row.labels) as string[],
+      node: row.node ?? undefined,
+    }));
+  }
+
+  async consolidate(): Promise<{ archived: number; deleted: number }> {
+    const now = Date.now();
+    const day = 86400000;
+
+    const deletedLogs = this.db.prepare(
+      `DELETE FROM memories WHERE type = 'log' AND date < ? AND score < 0.3`
+    ).run(new Date(now - 30 * day).toISOString().slice(0, 10));
+
+    const deletedNotes = this.db.prepare(
+      `DELETE FROM memories WHERE type = 'note' AND date < ? AND score < 0.3`
+    ).run(new Date(now - 60 * day).toISOString().slice(0, 10));
+
+    const deletedOld = this.db.prepare(
+      `DELETE FROM memories WHERE type NOT IN ('lesson', 'decision', 'fact') AND date < ? AND score < 0.1`
+    ).run(new Date(now - 90 * day).toISOString().slice(0, 10));
+
+    return {
+      archived: 0,
+      deleted: (deletedLogs.changes as number) + (deletedNotes.changes as number) + (deletedOld.changes as number),
+    };
+  }
+
+  describe(): string {
+    const count = (this.db.prepare('SELECT COUNT(*) as count FROM memories').get() as { count: number }).count;
+    return `Memory: SQLite with FTS5 full-text search (${count} entries). Use memory_recall to search, memory_store to save. This is the ONLY memory system — do not read or write memory/*.md files.`;
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "outDir": "dist",
+    "rootDir": "src",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "strict": true,
+    "esModuleInterop": true,
+    "composite": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["src/**/*.test.ts"]
+}