gsd-pi 2.75.0-dev.2203010a0 → 2.75.0-dev.96d4bb599

package/src/resources/extensions/gsd/gsd-db.ts

@@ -180,7 +180,7 @@ function openRawDb(path: string): unknown {
   return new Database(path);
 }
 
-const SCHEMA_VERSION = 17;
+const SCHEMA_VERSION = 20;
 
 function indexExists(db: DbAdapter, name: string): boolean {
   return !!db.prepare(
@@ -281,7 +281,9 @@ function initSchema(db: DbAdapter, fileBacked: boolean): void {
         created_at TEXT NOT NULL,
         updated_at TEXT NOT NULL,
         superseded_by TEXT DEFAULT NULL,
-        hit_count INTEGER NOT NULL DEFAULT 0
+        hit_count INTEGER NOT NULL DEFAULT 0,
+        scope TEXT NOT NULL DEFAULT 'project',
+        tags TEXT NOT NULL DEFAULT '[]'
       )
     `);
 
@@ -293,6 +295,45 @@ function initSchema(db: DbAdapter, fileBacked: boolean): void {
       )
     `);
 
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS memory_sources (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        uri TEXT,
+        title TEXT,
+        content TEXT NOT NULL,
+        content_hash TEXT NOT NULL UNIQUE,
+        imported_at TEXT NOT NULL,
+        scope TEXT NOT NULL DEFAULT 'project',
+        tags TEXT NOT NULL DEFAULT '[]'
+      )
+    `);
+
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS memory_embeddings (
+        memory_id TEXT PRIMARY KEY,
+        model TEXT NOT NULL,
+        dim INTEGER NOT NULL,
+        vector BLOB NOT NULL,
+        updated_at TEXT NOT NULL
+      )
+    `);
+
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS memory_relations (
+        from_id TEXT NOT NULL,
+        to_id TEXT NOT NULL,
+        rel TEXT NOT NULL,
+        confidence REAL NOT NULL DEFAULT 0.8,
+        created_at TEXT NOT NULL,
+        PRIMARY KEY (from_id, to_id, rel)
+      )
+    `);
+
+    // FTS5 virtual table mirroring memories.content for fast keyword search.
+    // Optional — if the SQLite build lacks FTS5, we fall back to LIKE scans.
+    tryCreateMemoriesFts(db);
+
     db.exec(`
       CREATE TABLE IF NOT EXISTS milestones (
         id TEXT PRIMARY KEY,
@@ -516,6 +557,11 @@ function initSchema(db: DbAdapter, fileBacked: boolean): void {
     `);
 
     db.exec("CREATE INDEX IF NOT EXISTS idx_memories_active ON memories(superseded_by)");
+    db.exec("CREATE INDEX IF NOT EXISTS idx_memories_scope ON memories(scope)");
+    db.exec("CREATE INDEX IF NOT EXISTS idx_memory_sources_kind ON memory_sources(kind)");
+    db.exec("CREATE INDEX IF NOT EXISTS idx_memory_sources_scope ON memory_sources(scope)");
+    db.exec("CREATE INDEX IF NOT EXISTS idx_memory_relations_from ON memory_relations(from_id)");
+    db.exec("CREATE INDEX IF NOT EXISTS idx_memory_relations_to ON memory_relations(to_id)");
     db.exec("CREATE INDEX IF NOT EXISTS idx_replan_history_milestone ON replan_history(milestone_id, created_at)");
 
     // v13 indexes — hot-path dispatch queries
@@ -565,6 +611,56 @@ function columnExists(db: DbAdapter, table: string, column: string): boolean {
   return rows.some((row) => row["name"] === column);
 }
 
+/**
+ * Create the FTS5 virtual table for memories plus the triggers that keep it
+ * in sync with the base table. FTS5 may be unavailable on stripped-down
+ * SQLite builds — callers should treat failure as non-fatal and fall back
+ * to LIKE-based scans in `memory-store.queryMemoriesRanked`.
+ */
+export function tryCreateMemoriesFts(db: DbAdapter): boolean {
+  try {
+    db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts
+      USING fts5(content, content='memories', content_rowid='seq', tokenize='porter unicode61')
+    `);
+    // Triggers mirror inserts / updates / deletes on the base memories table.
+    db.exec(`
+      CREATE TRIGGER IF NOT EXISTS memories_ai
+      AFTER INSERT ON memories BEGIN
+        INSERT INTO memories_fts(rowid, content) VALUES (new.seq, new.content);
+      END
+    `);
+    db.exec(`
+      CREATE TRIGGER IF NOT EXISTS memories_ad
+      AFTER DELETE ON memories BEGIN
+        INSERT INTO memories_fts(memories_fts, rowid, content) VALUES ('delete', old.seq, old.content);
+      END
+    `);
+    db.exec(`
+      CREATE TRIGGER IF NOT EXISTS memories_au
+      AFTER UPDATE OF content ON memories BEGIN
+        INSERT INTO memories_fts(memories_fts, rowid, content) VALUES ('delete', old.seq, old.content);
+        INSERT INTO memories_fts(rowid, content) VALUES (new.seq, new.content);
+      END
+    `);
+    return true;
+  } catch (err) {
+    logWarning("db", `FTS5 unavailable — memory queries will use LIKE fallback: ${(err as Error).message}`);
+    return false;
+  }
+}
+
+export function isMemoriesFtsAvailable(db: DbAdapter): boolean {
+  try {
+    const row = db
+      .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memories_fts'")
+      .get();
+    return !!row;
+  } catch {
+    return false;
+  }
+}
+
 function ensureColumn(db: DbAdapter, table: string, column: string, ddl: string): void {
   if (!columnExists(db, table, column)) db.exec(ddl);
 }
@@ -988,6 +1084,79 @@ function migrateSchema(db: DbAdapter): void {
       });
     }
 
+    if (currentVersion < 18) {
+      // Memory system Phase 2: scope + tags on memories, plus memory_sources
+      // table for raw ingested content (notes, files, URLs, artifacts).
+      ensureColumn(db, "memories", "scope", `ALTER TABLE memories ADD COLUMN scope TEXT NOT NULL DEFAULT 'project'`);
+      ensureColumn(db, "memories", "tags", `ALTER TABLE memories ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'`);
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS memory_sources (
+          id TEXT PRIMARY KEY,
+          kind TEXT NOT NULL,
+          uri TEXT,
+          title TEXT,
+          content TEXT NOT NULL,
+          content_hash TEXT NOT NULL UNIQUE,
+          imported_at TEXT NOT NULL,
+          scope TEXT NOT NULL DEFAULT 'project',
+          tags TEXT NOT NULL DEFAULT '[]'
+        )
+      `);
+      db.exec("CREATE INDEX IF NOT EXISTS idx_memories_scope ON memories(scope)");
+      db.exec("CREATE INDEX IF NOT EXISTS idx_memory_sources_kind ON memory_sources(kind)");
+      db.exec("CREATE INDEX IF NOT EXISTS idx_memory_sources_scope ON memory_sources(scope)");
+      db.prepare("INSERT INTO schema_version (version, applied_at) VALUES (:version, :applied_at)").run({
+        ":version": 18,
+        ":applied_at": new Date().toISOString(),
+      });
+    }
+
+    if (currentVersion < 19) {
+      // Memory system Phase 3: embeddings + FTS5 for hybrid retrieval.
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS memory_embeddings (
+          memory_id TEXT PRIMARY KEY,
+          model TEXT NOT NULL,
+          dim INTEGER NOT NULL,
+          vector BLOB NOT NULL,
+          updated_at TEXT NOT NULL
+        )
+      `);
+      tryCreateMemoriesFts(db);
+      // Backfill FTS5 with any existing memories (triggers only cover future writes).
+      if (isMemoriesFtsAvailable(db)) {
+        try {
+          db.exec(`INSERT INTO memories_fts(rowid, content) SELECT seq, content FROM memories`);
+        } catch (err) {
+          logWarning("db", `FTS5 backfill failed: ${(err as Error).message}`);
+        }
+      }
+      db.prepare("INSERT INTO schema_version (version, applied_at) VALUES (:version, :applied_at)").run({
+        ":version": 19,
+        ":applied_at": new Date().toISOString(),
+      });
+    }
+
+    if (currentVersion < 20) {
+      // Memory system Phase 4: knowledge-graph relations between memories.
+      db.exec(`
+        CREATE TABLE IF NOT EXISTS memory_relations (
+          from_id TEXT NOT NULL,
+          to_id TEXT NOT NULL,
+          rel TEXT NOT NULL,
+          confidence REAL NOT NULL DEFAULT 0.8,
+          created_at TEXT NOT NULL,
+          PRIMARY KEY (from_id, to_id, rel)
+        )
+      `);
+      db.exec("CREATE INDEX IF NOT EXISTS idx_memory_relations_from ON memory_relations(from_id)");
+      db.exec("CREATE INDEX IF NOT EXISTS idx_memory_relations_to ON memory_relations(to_id)");
+      db.prepare("INSERT INTO schema_version (version, applied_at) VALUES (:version, :applied_at)").run({
+        ":version": 20,
+        ":applied_at": new Date().toISOString(),
+      });
+    }
+
     db.exec("COMMIT");
   } catch (err) {
     db.exec("ROLLBACK");
@@ -3423,11 +3592,13 @@ export function insertMemoryRow(args: {
   sourceUnitId: string | null;
   createdAt: string;
   updatedAt: string;
+  scope?: string;
+  tags?: string[];
 }): void {
   if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
   currentDb.prepare(
-    `INSERT INTO memories (id, category, content, confidence, source_unit_type, source_unit_id, created_at, updated_at)
-     VALUES (:id, :category, :content, :confidence, :source_unit_type, :source_unit_id, :created_at, :updated_at)`,
+    `INSERT INTO memories (id, category, content, confidence, source_unit_type, source_unit_id, created_at, updated_at, scope, tags)
+     VALUES (:id, :category, :content, :confidence, :source_unit_type, :source_unit_id, :created_at, :updated_at, :scope, :tags)`,
   ).run({
     ":id": args.id,
     ":category": args.category,
@@ -3437,9 +3608,107 @@ export function insertMemoryRow(args: {
     ":source_unit_id": args.sourceUnitId,
     ":created_at": args.createdAt,
     ":updated_at": args.updatedAt,
+    ":scope": args.scope ?? "project",
+    ":tags": JSON.stringify(args.tags ?? []),
   });
 }
 
+export function insertMemorySourceRow(args: {
+  id: string;
+  kind: string;
+  uri: string | null;
+  title: string | null;
+  content: string;
+  contentHash: string;
+  importedAt: string;
+  scope?: string;
+  tags?: string[];
+}): void {
+  if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
+  currentDb.prepare(
+    `INSERT OR IGNORE INTO memory_sources (id, kind, uri, title, content, content_hash, imported_at, scope, tags)
+     VALUES (:id, :kind, :uri, :title, :content, :content_hash, :imported_at, :scope, :tags)`,
+  ).run({
+    ":id": args.id,
+    ":kind": args.kind,
+    ":uri": args.uri,
+    ":title": args.title,
+    ":content": args.content,
+    ":content_hash": args.contentHash,
+    ":imported_at": args.importedAt,
+    ":scope": args.scope ?? "project",
+    ":tags": JSON.stringify(args.tags ?? []),
+  });
+}
+
+export function deleteMemorySourceRow(id: string): boolean {
+  if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
+  const res = currentDb
+    .prepare("DELETE FROM memory_sources WHERE id = :id")
+    .run({ ":id": id }) as { changes?: number };
+  return (res?.changes ?? 0) > 0;
+}
+
+export function upsertMemoryEmbedding(args: {
+  memoryId: string;
+  model: string;
+  dim: number;
+  vector: Uint8Array;
+  updatedAt: string;
+}): void {
+  if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
+  currentDb.prepare(
+    `INSERT INTO memory_embeddings (memory_id, model, dim, vector, updated_at)
+     VALUES (:memory_id, :model, :dim, :vector, :updated_at)
+     ON CONFLICT(memory_id) DO UPDATE SET
+       model = excluded.model,
+       dim = excluded.dim,
+       vector = excluded.vector,
+       updated_at = excluded.updated_at`,
+  ).run({
+    ":memory_id": args.memoryId,
+    ":model": args.model,
+    ":dim": args.dim,
+    ":vector": args.vector,
+    ":updated_at": args.updatedAt,
+  });
+}
+
+export function deleteMemoryEmbedding(memoryId: string): boolean {
+  if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
+  const res = currentDb
+    .prepare("DELETE FROM memory_embeddings WHERE memory_id = :id")
+    .run({ ":id": memoryId }) as { changes?: number };
+  return (res?.changes ?? 0) > 0;
+}
+
+export function insertMemoryRelationRow(args: {
+  fromId: string;
+  toId: string;
+  rel: string;
+  confidence: number;
+  createdAt: string;
+}): void {
+  if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
+  currentDb.prepare(
+    `INSERT OR REPLACE INTO memory_relations (from_id, to_id, rel, confidence, created_at)
+     VALUES (:from_id, :to_id, :rel, :confidence, :created_at)`,
+  ).run({
+    ":from_id": args.fromId,
+    ":to_id": args.toId,
+    ":rel": args.rel,
+    ":confidence": args.confidence,
+    ":created_at": args.createdAt,
+  });
+}
+
+export function deleteMemoryRelationsFor(memoryId: string): void {
+  if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
+  currentDb
+    .prepare("DELETE FROM memory_relations WHERE from_id = :id OR to_id = :id")
+    .run({ ":id": memoryId });
+}
+
 export function rewriteMemoryId(placeholderId: string, realId: string): void {
   if (!currentDb) throw new GSDError(GSD_STALE_STATE, "gsd-db: No database open");
   currentDb.prepare("UPDATE memories SET id = :real_id WHERE id = :placeholder").run({

package/src/resources/extensions/gsd/memory-embeddings.ts

@@ -0,0 +1,235 @@
+// GSD Memory Embeddings — provider-agnostic embedding layer
+//
+// Same model-discovery pattern as buildMemoryLLMCall: prefers a dedicated
+// embedding-capable model when available, and returns null when none is
+// found (which is the common case — not every provider exposes embeddings).
+//
+// When embeddings are unavailable, all calls become no-ops and
+// queryMemoriesRanked falls back to keyword-only scoring.
+
+import type { ExtensionContext } from "@gsd/pi-coding-agent";
+
+import { _getAdapter, isDbAvailable, upsertMemoryEmbedding, deleteMemoryEmbedding } from "./gsd-db.js";
+import { logWarning } from "./workflow-logger.js";
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+export type EmbedFn = (texts: string[]) => Promise<Float32Array[]>;
+
+export interface EmbeddingModelInfo {
+  id: string;
+}
+
+export interface MemoryEmbeddingRow {
+  memoryId: string;
+  model: string;
+  dim: number;
+  vector: Float32Array;
+}
+
+// ─── Model selection ────────────────────────────────────────────────────────
+
+const EMBEDDING_ID_HINTS = [
+  "embed",
+  "embedding",
+  "voyage",
+  "text-embedding",
+  "nomic",
+  "jina-embed",
+  "bge",
+  "mxbai-embed",
+];
+
+/**
+ * Try to build an embedding function from the model registry. Returns null
+ * when no embedding-capable model is obvious from the registry metadata.
+ *
+ * NOTE: the Pi SDK doesn't yet expose a dedicated embeddings API for every
+ * provider. This implementation currently targets Anthropic / OpenAI-shaped
+ * SDKs: when the caller has direct API access via `ctx.modelRegistry`, they
+ * can wire this up by providing an `embedFn` override. We ship the hint-based
+ * detection here so future providers can plug in without touching callers.
+ */
+export function buildEmbeddingFn(ctx: ExtensionContext): EmbedFn | null {
+  try {
+    const available = ctx.modelRegistry?.getAvailable?.();
+    if (!available || available.length === 0) return null;
+    const candidate = available.find((model) => {
+      const id = typeof model?.id === "string" ? model.id.toLowerCase() : "";
+      return EMBEDDING_ID_HINTS.some((hint) => id.includes(hint));
+    });
+    if (!candidate) return null;
+    // We don't currently have a provider-neutral embedding call in Pi; the
+    // detection surface is in place so wiring can happen once Pi offers it.
+    return null;
+  } catch (err) {
+    logWarning("memory-embeddings", `model discovery failed: ${(err as Error).message}`);
+    return null;
+  }
+}
+
+// ─── Vector (de)serialization ───────────────────────────────────────────────
+
+export function packFloat32(vec: Float32Array): Uint8Array {
+  return new Uint8Array(vec.buffer, vec.byteOffset, vec.byteLength);
+}
+
+export function unpackFloat32(blob: unknown): Float32Array | null {
+  if (!blob) return null;
+  try {
+    if (blob instanceof Float32Array) return blob;
+    let view: Uint8Array;
+    if (blob instanceof Uint8Array) {
+      view = blob;
+    } else if (blob instanceof ArrayBuffer) {
+      view = new Uint8Array(blob);
+    } else if ((blob as Buffer).buffer && (blob as Buffer).byteLength != null) {
+      const buf = blob as Buffer;
+      view = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+    } else if (Array.isArray(blob)) {
+      return new Float32Array(blob as number[]);
+    } else {
+      return null;
+    }
+    if (view.byteLength % 4 !== 0) return null;
+    // Copy into an aligned buffer — BLOBs may arrive at odd byte offsets.
+    const aligned = new ArrayBuffer(view.byteLength);
+    new Uint8Array(aligned).set(view);
+    return new Float32Array(aligned);
+  } catch {
+    return null;
+  }
+}
+
+// ─── Math ───────────────────────────────────────────────────────────────────
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  if (a.length === 0 || a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    const x = a[i];
+    const y = b[i];
+    dot += x * y;
+    normA += x * x;
+    normB += y * y;
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+// ─── Read helpers ───────────────────────────────────────────────────────────
+
+export function getEmbeddingForMemory(memoryId: string): MemoryEmbeddingRow | null {
+  if (!isDbAvailable()) return null;
+  const adapter = _getAdapter();
+  if (!adapter) return null;
+  try {
+    const row = adapter
+      .prepare(
+        "SELECT memory_id, model, dim, vector FROM memory_embeddings WHERE memory_id = :id",
+      )
+      .get({ ":id": memoryId });
+    if (!row) return null;
+    const vector = unpackFloat32(row["vector"]);
+    if (!vector) return null;
+    return {
+      memoryId: row["memory_id"] as string,
+      model: row["model"] as string,
+      dim: row["dim"] as number,
+      vector,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function loadAllEmbeddings(): MemoryEmbeddingRow[] {
+  if (!isDbAvailable()) return [];
+  const adapter = _getAdapter();
+  if (!adapter) return [];
+  try {
+    const rows = adapter
+      .prepare(
+        `SELECT e.memory_id, e.model, e.dim, e.vector
+         FROM memory_embeddings e
+         JOIN memories m ON m.id = e.memory_id
+         WHERE m.superseded_by IS NULL`,
+      )
+      .all();
+    const out: MemoryEmbeddingRow[] = [];
+    for (const row of rows) {
+      const vector = unpackFloat32(row["vector"]);
+      if (!vector) continue;
+      out.push({
+        memoryId: row["memory_id"] as string,
+        model: row["model"] as string,
+        dim: row["dim"] as number,
+        vector,
+      });
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}
+
+// ─── Write helpers ──────────────────────────────────────────────────────────
+
+export function saveEmbedding(
+  memoryId: string,
+  vector: Float32Array,
+  model: string,
+): boolean {
+  if (!isDbAvailable()) return false;
+  try {
+    upsertMemoryEmbedding({
+      memoryId,
+      model,
+      dim: vector.length,
+      vector: packFloat32(vector),
+      updatedAt: new Date().toISOString(),
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function removeEmbedding(memoryId: string): boolean {
+  if (!isDbAvailable()) return false;
+  try {
+    return deleteMemoryEmbedding(memoryId);
+  } catch {
+    return false;
+  }
+}
+
+// ─── Orchestration ──────────────────────────────────────────────────────────
+
+/**
+ * Embed each memory's content via `embedFn` (if provided) and persist the
+ * resulting vectors. Returns the number of successfully embedded memories.
+ * Safe to call with embedFn=null — it becomes a no-op.
+ */
+export async function embedMemories(
+  memories: Array<{ id: string; content: string }>,
+  embedFn: EmbedFn | null,
+  model: string,
+): Promise<number> {
+  if (!embedFn || memories.length === 0) return 0;
+  try {
+    const vectors = await embedFn(memories.map((m) => m.content));
+    let count = 0;
+    for (let i = 0; i < memories.length && i < vectors.length; i++) {
+      const vector = vectors[i];
+      if (!vector || vector.length === 0) continue;
+      if (saveEmbedding(memories[i].id, vector, model)) count++;
+    }
+    return count;
+  } catch (err) {
+    logWarning("memory-embeddings", `embed failed: ${(err as Error).message}`);
+    return 0;
+  }
+}