gsd-pi 2.75.0 → 2.76.0-dev.4100bd590

package/dist/resources/extensions/gsd/memory-embeddings.js

@@ -0,0 +1,219 @@
+// GSD Memory Embeddings — provider-agnostic embedding layer
+//
+// Same model-discovery pattern as buildMemoryLLMCall: prefers a dedicated
+// embedding-capable model when available, and returns null when none is
+// found (which is the common case — not every provider exposes embeddings).
+//
+// When embeddings are unavailable, all calls become no-ops and
+// queryMemoriesRanked falls back to keyword-only scoring.
+import { _getAdapter, isDbAvailable, upsertMemoryEmbedding, deleteMemoryEmbedding } from "./gsd-db.js";
+import { logWarning } from "./workflow-logger.js";
+// ─── Model selection ────────────────────────────────────────────────────────
+const EMBEDDING_ID_HINTS = [
+    "embed",
+    "embedding",
+    "voyage",
+    "text-embedding",
+    "nomic",
+    "jina-embed",
+    "bge",
+    "mxbai-embed",
+];
+/**
+ * Try to build an embedding function from the model registry. Returns null
+ * when no embedding-capable model is obvious from the registry metadata.
+ *
+ * NOTE: the Pi SDK doesn't yet expose a dedicated embeddings API for every
+ * provider. This implementation currently targets Anthropic / OpenAI-shaped
+ * SDKs: when the caller has direct API access via `ctx.modelRegistry`, they
+ * can wire this up by providing an `embedFn` override. We ship the hint-based
+ * detection here so future providers can plug in without touching callers.
+ */
+export function buildEmbeddingFn(ctx) {
+    try {
+        const available = ctx.modelRegistry?.getAvailable?.();
+        if (!available || available.length === 0)
+            return null;
+        const candidate = available.find((model) => {
+            const id = typeof model?.id === "string" ? model.id.toLowerCase() : "";
+            return EMBEDDING_ID_HINTS.some((hint) => id.includes(hint));
+        });
+        if (!candidate)
+            return null;
+        // We don't currently have a provider-neutral embedding call in Pi; the
+        // detection surface is in place so wiring can happen once Pi offers it.
+        return null;
+    }
+    catch (err) {
+        logWarning("memory-embeddings", `model discovery failed: ${err.message}`);
+        return null;
+    }
+}
+// ─── Vector (de)serialization ───────────────────────────────────────────────
+export function packFloat32(vec) {
+    return new Uint8Array(vec.buffer, vec.byteOffset, vec.byteLength);
+}
+export function unpackFloat32(blob) {
+    if (!blob)
+        return null;
+    try {
+        if (blob instanceof Float32Array)
+            return blob;
+        let view;
+        if (blob instanceof Uint8Array) {
+            view = blob;
+        }
+        else if (blob instanceof ArrayBuffer) {
+            view = new Uint8Array(blob);
+        }
+        else if (blob.buffer && blob.byteLength != null) {
+            const buf = blob;
+            view = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+        }
+        else if (Array.isArray(blob)) {
+            return new Float32Array(blob);
+        }
+        else {
+            return null;
+        }
+        if (view.byteLength % 4 !== 0)
+            return null;
+        // Copy into an aligned buffer — BLOBs may arrive at odd byte offsets.
+        const aligned = new ArrayBuffer(view.byteLength);
+        new Uint8Array(aligned).set(view);
+        return new Float32Array(aligned);
+    }
+    catch {
+        return null;
+    }
+}
+// ─── Math ───────────────────────────────────────────────────────────────────
+export function cosineSimilarity(a, b) {
+    if (a.length === 0 || a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        const x = a[i];
+        const y = b[i];
+        dot += x * y;
+        normA += x * x;
+        normB += y * y;
+    }
+    if (normA === 0 || normB === 0)
+        return 0;
+    return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+// ─── Read helpers ───────────────────────────────────────────────────────────
+export function getEmbeddingForMemory(memoryId) {
+    if (!isDbAvailable())
+        return null;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return null;
+    try {
+        const row = adapter
+            .prepare("SELECT memory_id, model, dim, vector FROM memory_embeddings WHERE memory_id = :id")
+            .get({ ":id": memoryId });
+        if (!row)
+            return null;
+        const vector = unpackFloat32(row["vector"]);
+        if (!vector)
+            return null;
+        return {
+            memoryId: row["memory_id"],
+            model: row["model"],
+            dim: row["dim"],
+            vector,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+export function loadAllEmbeddings() {
+    if (!isDbAvailable())
+        return [];
+    const adapter = _getAdapter();
+    if (!adapter)
+        return [];
+    try {
+        const rows = adapter
+            .prepare(`SELECT e.memory_id, e.model, e.dim, e.vector
+         FROM memory_embeddings e
+         JOIN memories m ON m.id = e.memory_id
+         WHERE m.superseded_by IS NULL`)
+            .all();
+        const out = [];
+        for (const row of rows) {
+            const vector = unpackFloat32(row["vector"]);
+            if (!vector)
+                continue;
+            out.push({
+                memoryId: row["memory_id"],
+                model: row["model"],
+                dim: row["dim"],
+                vector,
+            });
+        }
+        return out;
+    }
+    catch {
+        return [];
+    }
+}
+// ─── Write helpers ──────────────────────────────────────────────────────────
+export function saveEmbedding(memoryId, vector, model) {
+    if (!isDbAvailable())
+        return false;
+    try {
+        upsertMemoryEmbedding({
+            memoryId,
+            model,
+            dim: vector.length,
+            vector: packFloat32(vector),
+            updatedAt: new Date().toISOString(),
+        });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function removeEmbedding(memoryId) {
+    if (!isDbAvailable())
+        return false;
+    try {
+        return deleteMemoryEmbedding(memoryId);
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Orchestration ──────────────────────────────────────────────────────────
+/**
+ * Embed each memory's content via `embedFn` (if provided) and persist the
+ * resulting vectors. Returns the number of successfully embedded memories.
+ * Safe to call with embedFn=null — it becomes a no-op.
+ */
+export async function embedMemories(memories, embedFn, model) {
+    if (!embedFn || memories.length === 0)
+        return 0;
+    try {
+        const vectors = await embedFn(memories.map((m) => m.content));
+        let count = 0;
+        for (let i = 0; i < memories.length && i < vectors.length; i++) {
+            const vector = vectors[i];
+            if (!vector || vector.length === 0)
+                continue;
+            if (saveEmbedding(memories[i].id, vector, model))
+                count++;
+        }
+        return count;
+    }
+    catch (err) {
+        logWarning("memory-embeddings", `embed failed: ${err.message}`);
+        return 0;
+    }
+}

package/dist/resources/extensions/gsd/memory-extractor.js

@@ -95,6 +95,14 @@ Actions (return JSON array):
 - UPDATE: {"action": "UPDATE", "id": "<MEM###>", "content": "<revised text>"}
 - REINFORCE: {"action": "REINFORCE", "id": "<MEM###>"}
 - SUPERSEDE: {"action": "SUPERSEDE", "id": "<MEM###>", "superseded_by": "<MEM###>"}
+- LINK: {"action": "LINK", "from": "<MEM###>", "to": "<MEM###>", "rel": "<rel>", "confidence": <0.6-0.95>}
+
+Link relation types:
+- related_to   — two memories cover the same area
+- depends_on   — "to" is a prerequisite for "from"
+- contradicts  — "from" conflicts with "to"
+- elaborates   — "from" expands on "to"
+- supersedes   — "from" replaces "to" (rarely needed; prefer SUPERSEDE)
 
 Rules:
 - Don't create memories for one-off bug fixes or temporary state
@@ -102,6 +110,7 @@ Rules:
 - Keep content to 1-3 sentences
 - Confidence: 0.6 tentative, 0.8 solid, 0.95 well-confirmed
 - Prefer fewer high-quality memories over many low-quality ones
+- Only LINK memories that genuinely relate — don't fabricate edges
 - Return empty array [] if nothing worth remembering
 - NEVER include secrets, API keys, or passwords
 
@@ -219,6 +228,19 @@ export function parseMemoryResponse(raw) {
                         });
                     }
                     break;
+                case 'LINK':
+                    if (typeof item.from === 'string' &&
+                        typeof item.to === 'string' &&
+                        typeof item.rel === 'string') {
+                        actions.push({
+                            action: 'LINK',
+                            from: item.from,
+                            to: item.to,
+                            rel: item.rel,
+                            confidence: typeof item.confidence === 'number' ? item.confidence : undefined,
+                        });
+                    }
+                    break;
             }
         }
         return actions;
@@ -227,27 +249,72 @@ export function parseMemoryResponse(raw) {
         return [];
     }
 }
-// ─── Main Extraction Function ───────────────────────────────────────────────
+/**
+ * Core extractor — shared by unit post-processing and explicit ingest.
+ * Returns the applied actions for observability. Never throws.
+ */
+export async function extractMemoriesFromTranscript(transcript, llmCallFn, opts) {
+    if (!opts.force) {
+        if (_extracting)
+            return [];
+        const now = Date.now();
+        if (now - _lastExtractionTime < MIN_EXTRACTION_INTERVAL_MS)
+            return [];
+        _lastExtractionTime = now;
+    }
+    const acquireMutex = !_extracting;
+    if (acquireMutex)
+        _extracting = true;
+    try {
+        const trimmed = transcript.trim();
+        if (!trimmed)
+            return [];
+        const safeTranscript = redactSecrets(trimmed);
+        const activeMemories = getActiveMemories().map((m) => ({
+            id: m.id,
+            category: m.category,
+            content: m.content,
+        }));
+        const userPrompt = buildExtractionUserPrompt(opts.sourceType, opts.sourceId, activeMemories, safeTranscript);
+        const response = await llmCallFn(EXTRACTION_SYSTEM, userPrompt);
+        const actions = parseMemoryResponse(response);
+        if (actions.length === 0)
+            return [];
+        const decorated = actions.map((action) => {
+            if (action.action !== 'CREATE')
+                return action;
+            return {
+                ...action,
+                scope: action.scope ?? opts.scope,
+                tags: action.tags ?? opts.tags,
+            };
+        });
+        applyMemoryActions(decorated, opts.sourceType, opts.sourceId);
+        return decorated;
+    }
+    catch {
+        return [];
+    }
+    finally {
+        if (acquireMutex)
+            _extracting = false;
+    }
+}
 /**
  * Extract memories from a completed unit's activity log.
  * Fire-and-forget — never throws, mutex-guarded, respects rate limiting.
  */
 export async function extractMemoriesFromUnit(activityFile, unitType, unitId, llmCallFn) {
-    // Mutex guard
     if (_extracting)
         return;
-    // Rate limit
     const now = Date.now();
     if (now - _lastExtractionTime < MIN_EXTRACTION_INTERVAL_MS)
         return;
-    // Skip certain unit types
     if (SKIP_TYPES.has(unitType))
         return;
     const unitKey = `${unitType}/${unitId}`;
-    // Already processed
     if (isUnitProcessed(unitKey))
         return;
-    // Check file size
     try {
         const stat = statSync(activityFile);
         if (stat.size < MIN_ACTIVITY_SIZE)
@@ -259,32 +326,16 @@ export async function extractMemoriesFromUnit(activityFile, unitType, unitId, ll
     _extracting = true;
     _lastExtractionTime = now;
     try {
-        // Read and parse activity file
         const raw = readFileSync(activityFile, 'utf-8');
         const transcript = extractTranscriptFromActivity(raw);
         if (!transcript.trim())
             return;
-        // Redact secrets
-        const safeTranscript = redactSecrets(transcript);
-        // Get current memories for context
-        const activeMemories = getActiveMemories().map(m => ({
-            id: m.id,
-            category: m.category,
-            content: m.content,
-        }));
-        // Build prompts
-        const userPrompt = buildExtractionUserPrompt(unitType, unitId, activeMemories, safeTranscript);
-        // Call LLM
-        const response = await llmCallFn(EXTRACTION_SYSTEM, userPrompt);
-        // Parse response
-        const actions = parseMemoryResponse(response);
-        // Apply actions
-        if (actions.length > 0) {
-            applyMemoryActions(actions, unitType, unitId);
-        }
-        // Decay stale memories periodically
+        await extractMemoriesFromTranscript(transcript, llmCallFn, {
+            sourceType: unitType,
+            sourceId: unitId,
+            force: true,
+        });
         decayStaleMemories(20);
-        // Mark unit as processed
         markUnitProcessed(unitKey, activityFile);
     }
     catch {

package/dist/resources/extensions/gsd/memory-ingest.js

@@ -0,0 +1,218 @@
+// GSD Memory Ingest — turn raw content into memories
+//
+// Provides four entry points: ingestNote (inline text), ingestFile (local
+// path), ingestUrl (HTTP resource), and ingestArtifact (a named .gsd/ artifact
+// for a given milestone). Each one inserts a row into `memory_sources` and,
+// if an LLM call is available, fires the extractor against the content with
+// source-specific scope/tags.
+//
+// All four functions are safe to call without an LLM — they still persist the
+// source. This means ingestion is decoupled from extraction; a later
+// `/gsd memory rebuild` can re-extract from persisted sources.
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { basename, isAbsolute, resolve } from "node:path";
+import { createMemorySource } from "./memory-source-store.js";
+import { buildMemoryLLMCall, extractMemoriesFromTranscript } from "./memory-extractor.js";
+import { resolveMilestoneFile } from "./paths.js";
+import { logWarning } from "./workflow-logger.js";
+const DEFAULT_MAX_BYTES = 256 * 1024;
+function truncate(content, maxBytes) {
+    const buf = Buffer.from(content, "utf-8");
+    if (buf.byteLength <= maxBytes)
+        return content;
+    return `${buf.subarray(0, maxBytes).toString("utf-8")}\n\n…[truncated to ${maxBytes} bytes]`;
+}
+async function maybeExtract(ctx, source, content, opts) {
+    if (opts.extract === false || !ctx)
+        return [];
+    const llmCallFn = buildMemoryLLMCall(ctx);
+    if (!llmCallFn)
+        return [];
+    try {
+        return await extractMemoriesFromTranscript(content, llmCallFn, {
+            sourceType: source.kind,
+            sourceId: source.id,
+            scope: opts.scope,
+            tags: opts.tags,
+            force: true,
+        });
+    }
+    catch (err) {
+        logWarning("memory-ingest", `extraction failed: ${err.message}`);
+        return [];
+    }
+}
+function sourceCreateFailure(kind) {
+    return {
+        sourceId: "",
+        duplicate: false,
+        extracted: [],
+        kind,
+        title: null,
+        uri: null,
+    };
+}
+// ─── ingestNote ─────────────────────────────────────────────────────────────
+export async function ingestNote(note, ctx, opts = {}) {
+    const trimmed = note.trim();
+    if (!trimmed)
+        return sourceCreateFailure("note");
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    const content = truncate(trimmed, maxBytes);
+    const created = createMemorySource({
+        kind: "note",
+        uri: null,
+        title: content.slice(0, 80).replace(/\s+/g, " ").trim(),
+        content,
+        scope: opts.scope,
+        tags: opts.tags,
+    });
+    if (!created)
+        return sourceCreateFailure("note");
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "note", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "note",
+        title: content.slice(0, 80),
+        uri: null,
+    };
+}
+// ─── ingestFile ─────────────────────────────────────────────────────────────
+export async function ingestFile(path, ctx, opts = {}) {
+    const abs = isAbsolute(path) ? path : resolve(process.cwd(), path);
+    if (!existsSync(abs)) {
+        throw new Error(`File not found: ${abs}`);
+    }
+    const stat = statSync(abs);
+    if (!stat.isFile()) {
+        throw new Error(`Not a file: ${abs}`);
+    }
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    const raw = readFileSync(abs, "utf-8");
+    const content = truncate(raw, maxBytes);
+    const title = basename(abs);
+    const created = createMemorySource({
+        kind: "file",
+        uri: abs,
+        title,
+        content,
+        scope: opts.scope,
+        tags: opts.tags,
+    });
+    if (!created)
+        return { ...sourceCreateFailure("file"), uri: abs, title };
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "file", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "file",
+        title,
+        uri: abs,
+    };
+}
+// ─── ingestUrl ──────────────────────────────────────────────────────────────
+export async function ingestUrl(url, ctx, opts = {}) {
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    let body;
+    let title = null;
+    try {
+        const res = await fetch(url, { redirect: "follow" });
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status} ${res.statusText}`);
+        body = await res.text();
+        const titleMatch = body.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+        if (titleMatch)
+            title = titleMatch[1].trim().slice(0, 200);
+    }
+    catch (err) {
+        throw new Error(`Fetch failed for ${url}: ${err.message}`);
+    }
+    const content = truncate(stripHtml(body), maxBytes);
+    if (!content.trim()) {
+        throw new Error(`URL produced empty content: ${url}`);
+    }
+    const created = createMemorySource({
+        kind: "url",
+        uri: url,
+        title: title ?? url,
+        content,
+        scope: opts.scope,
+        tags: opts.tags,
+    });
+    if (!created)
+        return { ...sourceCreateFailure("url"), uri: url, title };
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "url", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "url",
+        title: title ?? url,
+        uri: url,
+    };
+}
+function stripHtml(html) {
+    return html
+        .replace(/<script[\s\S]*?<\/script>/gi, " ")
+        .replace(/<style[\s\S]*?<\/style>/gi, " ")
+        .replace(/<!--[\s\S]*?-->/g, " ")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+// ─── ingestArtifact ─────────────────────────────────────────────────────────
+/**
+ * Ingest a named artifact from a milestone directory (e.g. LEARNINGS,
+ * SUMMARY, CONTEXT). Resolves through `resolveMilestoneFile` so worktree
+ * layouts are handled correctly.
+ */
+export async function ingestArtifact(basePath, milestoneId, artifactType, ctx, opts = {}) {
+    const file = resolveMilestoneFile(basePath, milestoneId, artifactType);
+    if (!file || !existsSync(file)) {
+        throw new Error(`Artifact not found: ${milestoneId}-${artifactType}.md`);
+    }
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    const content = truncate(readFileSync(file, "utf-8"), maxBytes);
+    const title = `${milestoneId}-${artifactType}`;
+    const created = createMemorySource({
+        kind: "artifact",
+        uri: file,
+        title,
+        content,
+        scope: opts.scope,
+        tags: [...(opts.tags ?? []), milestoneId, artifactType.toLowerCase()],
+    });
+    if (!created)
+        return { ...sourceCreateFailure("artifact"), uri: file, title };
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "artifact", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "artifact",
+        title,
+        uri: file,
+    };
+}
+// ─── Helpers ────────────────────────────────────────────────────────────────
+export function summarizeIngest(result) {
+    if (!result.sourceId)
+        return "Ingest failed: could not persist source.";
+    const status = result.duplicate ? "duplicate (content_hash match)" : "new source";
+    const extracted = result.extracted.length === 0
+        ? "no memories extracted"
+        : `${result.extracted.length} memor${result.extracted.length === 1 ? "y" : "ies"} applied`;
+    const label = result.title ? ` "${result.title}"` : "";
+    return `Ingested ${result.kind}${label} as ${result.sourceId} (${status}, ${extracted}).`;
+}