gsd-pi 2.75.0-dev.2203010a0 → 2.75.0-dev.96d4bb599

package/dist/resources/extensions/gsd/memory-ingest.js

@@ -0,0 +1,218 @@
+// GSD Memory Ingest — turn raw content into memories
+//
+// Provides four entry points: ingestNote (inline text), ingestFile (local
+// path), ingestUrl (HTTP resource), and ingestArtifact (a named .gsd/ artifact
+// for a given milestone). Each one inserts a row into `memory_sources` and,
+// if an LLM call is available, fires the extractor against the content with
+// source-specific scope/tags.
+//
+// All four functions are safe to call without an LLM — they still persist the
+// source. This means ingestion is decoupled from extraction; a later
+// `/gsd memory rebuild` can re-extract from persisted sources.
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { basename, isAbsolute, resolve } from "node:path";
+import { createMemorySource } from "./memory-source-store.js";
+import { buildMemoryLLMCall, extractMemoriesFromTranscript } from "./memory-extractor.js";
+import { resolveMilestoneFile } from "./paths.js";
+import { logWarning } from "./workflow-logger.js";
+const DEFAULT_MAX_BYTES = 256 * 1024;
+function truncate(content, maxBytes) {
+    const buf = Buffer.from(content, "utf-8");
+    if (buf.byteLength <= maxBytes)
+        return content;
+    return `${buf.subarray(0, maxBytes).toString("utf-8")}\n\n…[truncated to ${maxBytes} bytes]`;
+}
+async function maybeExtract(ctx, source, content, opts) {
+    if (opts.extract === false || !ctx)
+        return [];
+    const llmCallFn = buildMemoryLLMCall(ctx);
+    if (!llmCallFn)
+        return [];
+    try {
+        return await extractMemoriesFromTranscript(content, llmCallFn, {
+            sourceType: source.kind,
+            sourceId: source.id,
+            scope: opts.scope,
+            tags: opts.tags,
+            force: true,
+        });
+    }
+    catch (err) {
+        logWarning("memory-ingest", `extraction failed: ${err.message}`);
+        return [];
+    }
+}
+function sourceCreateFailure(kind) {
+    return {
+        sourceId: "",
+        duplicate: false,
+        extracted: [],
+        kind,
+        title: null,
+        uri: null,
+    };
+}
+// ─── ingestNote ─────────────────────────────────────────────────────────────
+export async function ingestNote(note, ctx, opts = {}) {
+    const trimmed = note.trim();
+    if (!trimmed)
+        return sourceCreateFailure("note");
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    const content = truncate(trimmed, maxBytes);
+    const created = createMemorySource({
+        kind: "note",
+        uri: null,
+        title: content.slice(0, 80).replace(/\s+/g, " ").trim(),
+        content,
+        scope: opts.scope,
+        tags: opts.tags,
+    });
+    if (!created)
+        return sourceCreateFailure("note");
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "note", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "note",
+        title: content.slice(0, 80),
+        uri: null,
+    };
+}
+// ─── ingestFile ─────────────────────────────────────────────────────────────
+export async function ingestFile(path, ctx, opts = {}) {
+    const abs = isAbsolute(path) ? path : resolve(process.cwd(), path);
+    if (!existsSync(abs)) {
+        throw new Error(`File not found: ${abs}`);
+    }
+    const stat = statSync(abs);
+    if (!stat.isFile()) {
+        throw new Error(`Not a file: ${abs}`);
+    }
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    const raw = readFileSync(abs, "utf-8");
+    const content = truncate(raw, maxBytes);
+    const title = basename(abs);
+    const created = createMemorySource({
+        kind: "file",
+        uri: abs,
+        title,
+        content,
+        scope: opts.scope,
+        tags: opts.tags,
+    });
+    if (!created)
+        return { ...sourceCreateFailure("file"), uri: abs, title };
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "file", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "file",
+        title,
+        uri: abs,
+    };
+}
+// ─── ingestUrl ──────────────────────────────────────────────────────────────
+export async function ingestUrl(url, ctx, opts = {}) {
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    let body;
+    let title = null;
+    try {
+        const res = await fetch(url, { redirect: "follow" });
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status} ${res.statusText}`);
+        body = await res.text();
+        const titleMatch = body.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+        if (titleMatch)
+            title = titleMatch[1].trim().slice(0, 200);
+    }
+    catch (err) {
+        throw new Error(`Fetch failed for ${url}: ${err.message}`);
+    }
+    const content = truncate(stripHtml(body), maxBytes);
+    if (!content.trim()) {
+        throw new Error(`URL produced empty content: ${url}`);
+    }
+    const created = createMemorySource({
+        kind: "url",
+        uri: url,
+        title: title ?? url,
+        content,
+        scope: opts.scope,
+        tags: opts.tags,
+    });
+    if (!created)
+        return { ...sourceCreateFailure("url"), uri: url, title };
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "url", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "url",
+        title: title ?? url,
+        uri: url,
+    };
+}
+function stripHtml(html) {
+    return html
+        .replace(/<script[\s\S]*?<\/script>/gi, " ")
+        .replace(/<style[\s\S]*?<\/style>/gi, " ")
+        .replace(/<!--[\s\S]*?-->/g, " ")
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+// ─── ingestArtifact ─────────────────────────────────────────────────────────
+/**
+ * Ingest a named artifact from a milestone directory (e.g. LEARNINGS,
+ * SUMMARY, CONTEXT). Resolves through `resolveMilestoneFile` so worktree
+ * layouts are handled correctly.
+ */
+export async function ingestArtifact(basePath, milestoneId, artifactType, ctx, opts = {}) {
+    const file = resolveMilestoneFile(basePath, milestoneId, artifactType);
+    if (!file || !existsSync(file)) {
+        throw new Error(`Artifact not found: ${milestoneId}-${artifactType}.md`);
+    }
+    const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    const content = truncate(readFileSync(file, "utf-8"), maxBytes);
+    const title = `${milestoneId}-${artifactType}`;
+    const created = createMemorySource({
+        kind: "artifact",
+        uri: file,
+        title,
+        content,
+        scope: opts.scope,
+        tags: [...(opts.tags ?? []), milestoneId, artifactType.toLowerCase()],
+    });
+    if (!created)
+        return { ...sourceCreateFailure("artifact"), uri: file, title };
+    const extracted = created.duplicate
+        ? []
+        : await maybeExtract(ctx, { kind: "artifact", id: created.id }, content, opts);
+    return {
+        sourceId: created.id,
+        duplicate: created.duplicate,
+        extracted,
+        kind: "artifact",
+        title,
+        uri: file,
+    };
+}
+// ─── Helpers ────────────────────────────────────────────────────────────────
+export function summarizeIngest(result) {
+    if (!result.sourceId)
+        return "Ingest failed: could not persist source.";
+    const status = result.duplicate ? "duplicate (content_hash match)" : "new source";
+    const extracted = result.extracted.length === 0
+        ? "no memories extracted"
+        : `${result.extracted.length} memor${result.extracted.length === 1 ? "y" : "ies"} applied`;
+    const label = result.title ? ` "${result.title}"` : "";
+    return `Ingested ${result.kind}${label} as ${result.sourceId} (${status}, ${extracted}).`;
+}

package/dist/resources/extensions/gsd/memory-relations.js

@@ -0,0 +1,189 @@
+// GSD Memory Relations — knowledge-graph edges between memories
+//
+// Phase 4 companion to memory-store.ts. Edges live in the `memory_relations`
+// table and are created by (a) explicit LINK actions emitted by the memory
+// extractor, or (b) future `/gsd memory link` CLI commands. All writes go
+// through the single-writer gate in `gsd-db.ts`.
+import { _getAdapter, deleteMemoryRelationsFor, insertMemoryRelationRow, isDbAvailable, } from "./gsd-db.js";
+export const VALID_RELATIONS = [
+    "related_to",
+    "depends_on",
+    "contradicts",
+    "elaborates",
+    "supersedes",
+];
+// ─── Helpers ────────────────────────────────────────────────────────────────
+export function isValidRelation(value) {
+    return typeof value === "string" && VALID_RELATIONS.includes(value);
+}
+function clampConfidence(value) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return 0.8;
+    if (value < 0.1)
+        return 0.1;
+    if (value > 0.99)
+        return 0.99;
+    return value;
+}
+// ─── Mutations ──────────────────────────────────────────────────────────────
+export function createMemoryRelation(from, to, rel, confidence) {
+    if (!isDbAvailable())
+        return false;
+    if (!from || !to || from === to || !isValidRelation(rel))
+        return false;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return false;
+    try {
+        // Verify both memories exist — relation rows without endpoints are useless.
+        const fromRow = adapter.prepare("SELECT 1 FROM memories WHERE id = :id").get({ ":id": from });
+        const toRow = adapter.prepare("SELECT 1 FROM memories WHERE id = :id").get({ ":id": to });
+        if (!fromRow || !toRow)
+            return false;
+        insertMemoryRelationRow({
+            fromId: from,
+            toId: to,
+            rel,
+            confidence: clampConfidence(confidence),
+            createdAt: new Date().toISOString(),
+        });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function removeMemoryRelationsFor(memoryId) {
+    if (!isDbAvailable() || !memoryId)
+        return;
+    try {
+        deleteMemoryRelationsFor(memoryId);
+    }
+    catch {
+        // non-fatal
+    }
+}
+// ─── Queries ────────────────────────────────────────────────────────────────
+export function listRelationsFor(memoryId) {
+    if (!isDbAvailable())
+        return [];
+    const adapter = _getAdapter();
+    if (!adapter)
+        return [];
+    try {
+        const rows = adapter
+            .prepare("SELECT from_id, to_id, rel, confidence, created_at FROM memory_relations WHERE from_id = :id OR to_id = :id")
+            .all({ ":id": memoryId });
+        return rows.map(rowToRelation);
+    }
+    catch {
+        return [];
+    }
+}
+export function traverseGraph(startId, depth) {
+    const emptyResult = { nodes: [], edges: [] };
+    if (!isDbAvailable() || !startId)
+        return emptyResult;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return emptyResult;
+    const hop = Math.max(0, Math.min(5, Math.floor(depth || 0)));
+    try {
+        const visited = new Set();
+        const queue = [{ id: startId, hop: 0 }];
+        const nodes = new Map();
+        const edges = [];
+        while (queue.length > 0) {
+            const { id, hop: level } = queue.shift();
+            if (visited.has(id))
+                continue;
+            visited.add(id);
+            const nodeRow = adapter
+                .prepare("SELECT id, category, content, confidence, superseded_by FROM memories WHERE id = :id")
+                .get({ ":id": id });
+            if (!nodeRow)
+                continue;
+            nodes.set(id, {
+                id: nodeRow["id"],
+                category: nodeRow["category"],
+                content: nodeRow["content"],
+                confidence: nodeRow["confidence"],
+            });
+            // Include supersedes edges from the base table so old graphs remain
+            // connected even before the extractor starts emitting LINK actions.
+            const successor = nodeRow["superseded_by"];
+            if (successor && successor !== "CAP_EXCEEDED") {
+                edges.push({
+                    from: id,
+                    to: successor,
+                    rel: "supersedes",
+                    confidence: 1,
+                    createdAt: "",
+                });
+                if (!visited.has(successor) && level < hop) {
+                    queue.push({ id: successor, hop: level + 1 });
+                }
+            }
+            const predecessors = adapter
+                .prepare("SELECT id FROM memories WHERE superseded_by = :id")
+                .all({ ":id": id });
+            for (const pred of predecessors) {
+                const predId = pred["id"];
+                edges.push({ from: predId, to: id, rel: "supersedes", confidence: 1, createdAt: "" });
+                if (!visited.has(predId) && level < hop) {
+                    queue.push({ id: predId, hop: level + 1 });
+                }
+            }
+            if (level >= hop)
+                continue;
+            const outgoing = adapter
+                .prepare("SELECT from_id, to_id, rel, confidence, created_at FROM memory_relations WHERE from_id = :id")
+                .all({ ":id": id });
+            for (const row of outgoing) {
+                const edge = rowToRelation(row);
+                edges.push(edge);
+                if (!visited.has(edge.to))
+                    queue.push({ id: edge.to, hop: level + 1 });
+            }
+            const incoming = adapter
+                .prepare("SELECT from_id, to_id, rel, confidence, created_at FROM memory_relations WHERE to_id = :id")
+                .all({ ":id": id });
+            for (const row of incoming) {
+                const edge = rowToRelation(row);
+                edges.push(edge);
+                if (!visited.has(edge.from))
+                    queue.push({ id: edge.from, hop: level + 1 });
+            }
+        }
+        return {
+            nodes: [...nodes.values()],
+            edges: dedupeEdges(edges),
+        };
+    }
+    catch {
+        return emptyResult;
+    }
+}
+function rowToRelation(row) {
+    const relRaw = row["rel"];
+    const rel = isValidRelation(relRaw) ? relRaw : "related_to";
+    return {
+        from: row["from_id"],
+        to: row["to_id"],
+        rel,
+        confidence: row["confidence"] ?? 0.8,
+        createdAt: row["created_at"] ?? "",
+    };
+}
+function dedupeEdges(edges) {
+    const seen = new Set();
+    const out = [];
+    for (const e of edges) {
+        const key = `${e.from}|${e.to}|${e.rel}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(e);
+    }
+    return out;
+}

package/dist/resources/extensions/gsd/memory-source-store.js

@@ -0,0 +1,113 @@
+// GSD Memory Sources — CRUD for raw ingested content (notes, files, URLs, artifacts)
+//
+// Distinct from `memories`: a `memory_source` row is the preserved raw input
+// that an extractor may (or may not) distill into one or more memories.
+// Storing the source makes ingestion idempotent (content_hash) and gives the
+// user a way to trace a memory back to its origin.
+import { createHash, randomUUID } from "node:crypto";
+import { _getAdapter, isDbAvailable, insertMemorySourceRow, deleteMemorySourceRow } from "./gsd-db.js";
+function rowToSource(row) {
+    const tagsRaw = typeof row["tags"] === "string" ? row["tags"] : "[]";
+    let tags = [];
+    try {
+        const parsed = JSON.parse(tagsRaw);
+        if (Array.isArray(parsed))
+            tags = parsed.filter((t) => typeof t === "string");
+    }
+    catch {
+        // leave tags empty
+    }
+    return {
+        id: row["id"],
+        kind: row["kind"],
+        uri: row["uri"] ?? null,
+        title: row["title"] ?? null,
+        content: row["content"],
+        content_hash: row["content_hash"],
+        imported_at: row["imported_at"],
+        scope: row["scope"] ?? "project",
+        tags,
+    };
+}
+export function hashContent(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+export function newSourceId() {
+    return `SRC-${randomUUID().slice(0, 8)}`;
+}
+/**
+ * Insert a memory_source. Idempotent — if the content_hash already exists,
+ * returns the existing source's id and duplicate=true instead of inserting.
+ */
+export function createMemorySource(opts) {
+    if (!isDbAvailable())
+        return null;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return null;
+    try {
+        const contentHash = hashContent(opts.content);
+        const existing = adapter
+            .prepare("SELECT id FROM memory_sources WHERE content_hash = :h")
+            .get({ ":h": contentHash });
+        if (existing && typeof existing["id"] === "string") {
+            return { id: existing["id"], duplicate: true };
+        }
+        const id = newSourceId();
+        insertMemorySourceRow({
+            id,
+            kind: opts.kind,
+            uri: opts.uri ?? null,
+            title: opts.title ?? null,
+            content: opts.content,
+            contentHash,
+            importedAt: new Date().toISOString(),
+            scope: opts.scope ?? "project",
+            tags: opts.tags ?? [],
+        });
+        return { id, duplicate: false };
+    }
+    catch {
+        return null;
+    }
+}
+export function listMemorySources(limit = 50) {
+    if (!isDbAvailable())
+        return [];
+    const adapter = _getAdapter();
+    if (!adapter)
+        return [];
+    try {
+        const rows = adapter
+            .prepare("SELECT * FROM memory_sources ORDER BY imported_at DESC LIMIT :limit")
+            .all({ ":limit": limit });
+        return rows.map(rowToSource);
+    }
+    catch {
+        return [];
+    }
+}
+export function getMemorySource(id) {
+    if (!isDbAvailable())
+        return null;
+    const adapter = _getAdapter();
+    if (!adapter)
+        return null;
+    try {
+        const row = adapter.prepare("SELECT * FROM memory_sources WHERE id = :id").get({ ":id": id });
+        return row ? rowToSource(row) : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function deleteMemorySource(id) {
+    if (!isDbAvailable())
+        return false;
+    try {
+        return deleteMemorySourceRow(id);
+    }
+    catch {
+        return false;
+    }
+}