gsd-pi 2.75.0 → 2.76.0-dev.b072ebb73

package/src/resources/extensions/gsd/memory-ingest.ts

@@ -0,0 +1,286 @@
+// GSD Memory Ingest — turn raw content into memories
+//
+// Provides four entry points: ingestNote (inline text), ingestFile (local
+// path), ingestUrl (HTTP resource), and ingestArtifact (a named .gsd/ artifact
+// for a given milestone). Each one inserts a row into `memory_sources` and,
+// if an LLM call is available, fires the extractor against the content with
+// source-specific scope/tags.
+//
+// All four functions are safe to call without an LLM — they still persist the
+// source. This means ingestion is decoupled from extraction; a later
+// `/gsd memory rebuild` can re-extract from persisted sources.
+
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { basename, isAbsolute, resolve } from "node:path";
+import type { ExtensionContext } from "@gsd/pi-coding-agent";
+
+import { createMemorySource, type MemorySource, type MemorySourceKind } from "./memory-source-store.js";
+import { buildMemoryLLMCall, extractMemoriesFromTranscript } from "./memory-extractor.js";
+import type { MemoryAction } from "./memory-store.js";
+import { resolveMilestoneFile } from "./paths.js";
+import { logWarning } from "./workflow-logger.js";
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+export interface IngestOptions {
+  scope?: string;
+  tags?: string[];
+  /** Skip LLM extraction — just persist the source row. */
+  extract?: boolean;
+  /**
+   * Soft upper bound on source content size (bytes). Files/URLs above this
+   * are truncated before hashing and storing. Default 256 KiB.
+   */
+  maxBytes?: number;
+}
+
+export interface IngestResult {
+  sourceId: string;
+  duplicate: boolean;
+  extracted: MemoryAction[];
+  kind: MemorySourceKind;
+  title: string | null;
+  uri: string | null;
+}
+
+const DEFAULT_MAX_BYTES = 256 * 1024;
+
+function truncate(content: string, maxBytes: number): string {
+  const buf = Buffer.from(content, "utf-8");
+  if (buf.byteLength <= maxBytes) return content;
+  return `${buf.subarray(0, maxBytes).toString("utf-8")}\n\n…[truncated to ${maxBytes} bytes]`;
+}
+
+async function maybeExtract(
+  ctx: ExtensionContext | null,
+  source: { kind: MemorySourceKind; id: string },
+  content: string,
+  opts: IngestOptions,
+): Promise<MemoryAction[]> {
+  if (opts.extract === false || !ctx) return [];
+  const llmCallFn = buildMemoryLLMCall(ctx);
+  if (!llmCallFn) return [];
+  try {
+    return await extractMemoriesFromTranscript(content, llmCallFn, {
+      sourceType: source.kind,
+      sourceId: source.id,
+      scope: opts.scope,
+      tags: opts.tags,
+      force: true,
+    });
+  } catch (err) {
+    logWarning("memory-ingest", `extraction failed: ${(err as Error).message}`);
+    return [];
+  }
+}
+
+function sourceCreateFailure(kind: MemorySourceKind): IngestResult {
+  return {
+    sourceId: "",
+    duplicate: false,
+    extracted: [],
+    kind,
+    title: null,
+    uri: null,
+  };
+}
+
+// ─── ingestNote ─────────────────────────────────────────────────────────────
+
+export async function ingestNote(
+  note: string,
+  ctx: ExtensionContext | null,
+  opts: IngestOptions = {},
+): Promise<IngestResult> {
+  const trimmed = note.trim();
+  if (!trimmed) return sourceCreateFailure("note");
+
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+  const content = truncate(trimmed, maxBytes);
+
+  const created = createMemorySource({
+    kind: "note",
+    uri: null,
+    title: content.slice(0, 80).replace(/\s+/g, " ").trim(),
+    content,
+    scope: opts.scope,
+    tags: opts.tags,
+  });
+  if (!created) return sourceCreateFailure("note");
+
+  const extracted = created.duplicate
+    ? []
+    : await maybeExtract(ctx, { kind: "note", id: created.id }, content, opts);
+
+  return {
+    sourceId: created.id,
+    duplicate: created.duplicate,
+    extracted,
+    kind: "note",
+    title: content.slice(0, 80),
+    uri: null,
+  };
+}
+
+// ─── ingestFile ─────────────────────────────────────────────────────────────
+
+export async function ingestFile(
+  path: string,
+  ctx: ExtensionContext | null,
+  opts: IngestOptions = {},
+): Promise<IngestResult> {
+  const abs = isAbsolute(path) ? path : resolve(process.cwd(), path);
+  if (!existsSync(abs)) {
+    throw new Error(`File not found: ${abs}`);
+  }
+  const stat = statSync(abs);
+  if (!stat.isFile()) {
+    throw new Error(`Not a file: ${abs}`);
+  }
+
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+  const raw = readFileSync(abs, "utf-8");
+  const content = truncate(raw, maxBytes);
+  const title = basename(abs);
+
+  const created = createMemorySource({
+    kind: "file",
+    uri: abs,
+    title,
+    content,
+    scope: opts.scope,
+    tags: opts.tags,
+  });
+  if (!created) return { ...sourceCreateFailure("file"), uri: abs, title };
+
+  const extracted = created.duplicate
+    ? []
+    : await maybeExtract(ctx, { kind: "file", id: created.id }, content, opts);
+
+  return {
+    sourceId: created.id,
+    duplicate: created.duplicate,
+    extracted,
+    kind: "file",
+    title,
+    uri: abs,
+  };
+}
+
+// ─── ingestUrl ──────────────────────────────────────────────────────────────
+
+export async function ingestUrl(
+  url: string,
+  ctx: ExtensionContext | null,
+  opts: IngestOptions = {},
+): Promise<IngestResult> {
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+  let body: string;
+  let title: string | null = null;
+  try {
+    const res = await fetch(url, { redirect: "follow" });
+    if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
+    body = await res.text();
+    const titleMatch = body.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    if (titleMatch) title = titleMatch[1].trim().slice(0, 200);
+  } catch (err) {
+    throw new Error(`Fetch failed for ${url}: ${(err as Error).message}`);
+  }
+
+  const content = truncate(stripHtml(body), maxBytes);
+  if (!content.trim()) {
+    throw new Error(`URL produced empty content: ${url}`);
+  }
+
+  const created = createMemorySource({
+    kind: "url",
+    uri: url,
+    title: title ?? url,
+    content,
+    scope: opts.scope,
+    tags: opts.tags,
+  });
+  if (!created) return { ...sourceCreateFailure("url"), uri: url, title };
+
+  const extracted = created.duplicate
+    ? []
+    : await maybeExtract(ctx, { kind: "url", id: created.id }, content, opts);
+
+  return {
+    sourceId: created.id,
+    duplicate: created.duplicate,
+    extracted,
+    kind: "url",
+    title: title ?? url,
+    uri: url,
+  };
+}
+
+function stripHtml(html: string): string {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<!--[\s\S]*?-->/g, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+// ─── ingestArtifact ─────────────────────────────────────────────────────────
+
+/**
+ * Ingest a named artifact from a milestone directory (e.g. LEARNINGS,
+ * SUMMARY, CONTEXT). Resolves through `resolveMilestoneFile` so worktree
+ * layouts are handled correctly.
+ */
+export async function ingestArtifact(
+  basePath: string,
+  milestoneId: string,
+  artifactType: string,
+  ctx: ExtensionContext | null,
+  opts: IngestOptions = {},
+): Promise<IngestResult> {
+  const file = resolveMilestoneFile(basePath, milestoneId, artifactType);
+  if (!file || !existsSync(file)) {
+    throw new Error(`Artifact not found: ${milestoneId}-${artifactType}.md`);
+  }
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+  const content = truncate(readFileSync(file, "utf-8"), maxBytes);
+  const title = `${milestoneId}-${artifactType}`;
+  const created = createMemorySource({
+    kind: "artifact",
+    uri: file,
+    title,
+    content,
+    scope: opts.scope,
+    tags: [...(opts.tags ?? []), milestoneId, artifactType.toLowerCase()],
+  });
+  if (!created) return { ...sourceCreateFailure("artifact"), uri: file, title };
+
+  const extracted = created.duplicate
+    ? []
+    : await maybeExtract(ctx, { kind: "artifact", id: created.id }, content, opts);
+
+  return {
+    sourceId: created.id,
+    duplicate: created.duplicate,
+    extracted,
+    kind: "artifact",
+    title,
+    uri: file,
+  };
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+export function summarizeIngest(result: IngestResult): string {
+  if (!result.sourceId) return "Ingest failed: could not persist source.";
+  const status = result.duplicate ? "duplicate (content_hash match)" : "new source";
+  const extracted = result.extracted.length === 0
+    ? "no memories extracted"
+    : `${result.extracted.length} memor${result.extracted.length === 1 ? "y" : "ies"} applied`;
+  const label = result.title ? ` "${result.title}"` : "";
+  return `Ingested ${result.kind}${label} as ${result.sourceId} (${status}, ${extracted}).`;
+}
+
+export type { MemorySource };

package/src/resources/extensions/gsd/memory-relations.ts

@@ -0,0 +1,240 @@
+// GSD Memory Relations — knowledge-graph edges between memories
+//
+// Phase 4 companion to memory-store.ts. Edges live in the `memory_relations`
+// table and are created by (a) explicit LINK actions emitted by the memory
+// extractor, or (b) future `/gsd memory link` CLI commands. All writes go
+// through the single-writer gate in `gsd-db.ts`.
+
+import {
+  _getAdapter,
+  deleteMemoryRelationsFor,
+  insertMemoryRelationRow,
+  isDbAvailable,
+} from "./gsd-db.js";
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+export type RelationType =
+  | "related_to"
+  | "depends_on"
+  | "contradicts"
+  | "elaborates"
+  | "supersedes";
+
+export const VALID_RELATIONS: readonly RelationType[] = [
+  "related_to",
+  "depends_on",
+  "contradicts",
+  "elaborates",
+  "supersedes",
+];
+
+export interface MemoryRelation {
+  from: string;
+  to: string;
+  rel: RelationType;
+  confidence: number;
+  createdAt: string;
+}
+
+export interface MemoryGraphNode {
+  id: string;
+  category: string;
+  content: string;
+  confidence: number;
+}
+
+export interface MemoryGraph {
+  nodes: MemoryGraphNode[];
+  edges: MemoryRelation[];
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+export function isValidRelation(value: unknown): value is RelationType {
+  return typeof value === "string" && (VALID_RELATIONS as readonly string[]).includes(value);
+}
+
+function clampConfidence(value: unknown): number {
+  if (typeof value !== "number" || !Number.isFinite(value)) return 0.8;
+  if (value < 0.1) return 0.1;
+  if (value > 0.99) return 0.99;
+  return value;
+}
+
+// ─── Mutations ──────────────────────────────────────────────────────────────
+
+export function createMemoryRelation(
+  from: string,
+  to: string,
+  rel: RelationType,
+  confidence?: number,
+): boolean {
+  if (!isDbAvailable()) return false;
+  if (!from || !to || from === to || !isValidRelation(rel)) return false;
+
+  const adapter = _getAdapter();
+  if (!adapter) return false;
+
+  try {
+    // Verify both memories exist — relation rows without endpoints are useless.
+    const fromRow = adapter.prepare("SELECT 1 FROM memories WHERE id = :id").get({ ":id": from });
+    const toRow = adapter.prepare("SELECT 1 FROM memories WHERE id = :id").get({ ":id": to });
+    if (!fromRow || !toRow) return false;
+
+    insertMemoryRelationRow({
+      fromId: from,
+      toId: to,
+      rel,
+      confidence: clampConfidence(confidence),
+      createdAt: new Date().toISOString(),
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function removeMemoryRelationsFor(memoryId: string): void {
+  if (!isDbAvailable() || !memoryId) return;
+  try {
+    deleteMemoryRelationsFor(memoryId);
+  } catch {
+    // non-fatal
+  }
+}
+
+// ─── Queries ────────────────────────────────────────────────────────────────
+
+export function listRelationsFor(memoryId: string): MemoryRelation[] {
+  if (!isDbAvailable()) return [];
+  const adapter = _getAdapter();
+  if (!adapter) return [];
+  try {
+    const rows = adapter
+      .prepare(
+        "SELECT from_id, to_id, rel, confidence, created_at FROM memory_relations WHERE from_id = :id OR to_id = :id",
+      )
+      .all({ ":id": memoryId });
+    return rows.map(rowToRelation);
+  } catch {
+    return [];
+  }
+}
+
+export function traverseGraph(startId: string, depth: number): MemoryGraph {
+  const emptyResult: MemoryGraph = { nodes: [], edges: [] };
+  if (!isDbAvailable() || !startId) return emptyResult;
+  const adapter = _getAdapter();
+  if (!adapter) return emptyResult;
+
+  const hop = Math.max(0, Math.min(5, Math.floor(depth || 0)));
+
+  try {
+    const visited = new Set<string>();
+    const queue: Array<{ id: string; hop: number }> = [{ id: startId, hop: 0 }];
+    const nodes = new Map<string, MemoryGraphNode>();
+    const edges: MemoryRelation[] = [];
+
+    while (queue.length > 0) {
+      const { id, hop: level } = queue.shift()!;
+      if (visited.has(id)) continue;
+      visited.add(id);
+
+      const nodeRow = adapter
+        .prepare(
+          "SELECT id, category, content, confidence, superseded_by FROM memories WHERE id = :id",
+        )
+        .get({ ":id": id });
+      if (!nodeRow) continue;
+
+      nodes.set(id, {
+        id: nodeRow["id"] as string,
+        category: nodeRow["category"] as string,
+        content: nodeRow["content"] as string,
+        confidence: nodeRow["confidence"] as number,
+      });
+
+      // Include supersedes edges from the base table so old graphs remain
+      // connected even before the extractor starts emitting LINK actions.
+      const successor = nodeRow["superseded_by"] as string | null;
+      if (successor && successor !== "CAP_EXCEEDED") {
+        edges.push({
+          from: id,
+          to: successor,
+          rel: "supersedes",
+          confidence: 1,
+          createdAt: "",
+        });
+        if (!visited.has(successor) && level < hop) {
+          queue.push({ id: successor, hop: level + 1 });
+        }
+      }
+      const predecessors = adapter
+        .prepare("SELECT id FROM memories WHERE superseded_by = :id")
+        .all({ ":id": id });
+      for (const pred of predecessors) {
+        const predId = pred["id"] as string;
+        edges.push({ from: predId, to: id, rel: "supersedes", confidence: 1, createdAt: "" });
+        if (!visited.has(predId) && level < hop) {
+          queue.push({ id: predId, hop: level + 1 });
+        }
+      }
+
+      if (level >= hop) continue;
+
+      const outgoing = adapter
+        .prepare(
+          "SELECT from_id, to_id, rel, confidence, created_at FROM memory_relations WHERE from_id = :id",
+        )
+        .all({ ":id": id });
+      for (const row of outgoing) {
+        const edge = rowToRelation(row);
+        edges.push(edge);
+        if (!visited.has(edge.to)) queue.push({ id: edge.to, hop: level + 1 });
+      }
+
+      const incoming = adapter
+        .prepare(
+          "SELECT from_id, to_id, rel, confidence, created_at FROM memory_relations WHERE to_id = :id",
+        )
+        .all({ ":id": id });
+      for (const row of incoming) {
+        const edge = rowToRelation(row);
+        edges.push(edge);
+        if (!visited.has(edge.from)) queue.push({ id: edge.from, hop: level + 1 });
+      }
+    }
+
+    return {
+      nodes: [...nodes.values()],
+      edges: dedupeEdges(edges),
+    };
+  } catch {
+    return emptyResult;
+  }
+}
+
+function rowToRelation(row: Record<string, unknown>): MemoryRelation {
+  const relRaw = row["rel"] as string;
+  const rel = isValidRelation(relRaw) ? relRaw : ("related_to" as RelationType);
+  return {
+    from: row["from_id"] as string,
+    to: row["to_id"] as string,
+    rel,
+    confidence: (row["confidence"] as number) ?? 0.8,
+    createdAt: (row["created_at"] as string) ?? "",
+  };
+}
+
+function dedupeEdges(edges: MemoryRelation[]): MemoryRelation[] {
+  const seen = new Set<string>();
+  const out: MemoryRelation[] = [];
+  for (const e of edges) {
+    const key = `${e.from}|${e.to}|${e.rel}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(e);
+  }
+  return out;
+}

package/src/resources/extensions/gsd/memory-source-store.ts

@@ -0,0 +1,138 @@
+// GSD Memory Sources — CRUD for raw ingested content (notes, files, URLs, artifacts)
+//
+// Distinct from `memories`: a `memory_source` row is the preserved raw input
+// that an extractor may (or may not) distill into one or more memories.
+// Storing the source makes ingestion idempotent (content_hash) and gives the
+// user a way to trace a memory back to its origin.
+
+import { createHash, randomUUID } from "node:crypto";
+import { _getAdapter, isDbAvailable, insertMemorySourceRow, deleteMemorySourceRow } from "./gsd-db.js";
+
+export type MemorySourceKind = "note" | "url" | "file" | "artifact" | "capture" | "learning";
+
+export interface MemorySource {
+  id: string;
+  kind: MemorySourceKind;
+  uri: string | null;
+  title: string | null;
+  content: string;
+  content_hash: string;
+  imported_at: string;
+  scope: string;
+  tags: string[];
+}
+
+function rowToSource(row: Record<string, unknown>): MemorySource {
+  const tagsRaw = typeof row["tags"] === "string" ? (row["tags"] as string) : "[]";
+  let tags: string[] = [];
+  try {
+    const parsed = JSON.parse(tagsRaw);
+    if (Array.isArray(parsed)) tags = parsed.filter((t): t is string => typeof t === "string");
+  } catch {
+    // leave tags empty
+  }
+  return {
+    id: row["id"] as string,
+    kind: row["kind"] as MemorySourceKind,
+    uri: (row["uri"] as string) ?? null,
+    title: (row["title"] as string) ?? null,
+    content: row["content"] as string,
+    content_hash: row["content_hash"] as string,
+    imported_at: row["imported_at"] as string,
+    scope: (row["scope"] as string) ?? "project",
+    tags,
+  };
+}
+
+export function hashContent(content: string): string {
+  return createHash("sha256").update(content).digest("hex");
+}
+
+export function newSourceId(): string {
+  return `SRC-${randomUUID().slice(0, 8)}`;
+}
+
+export interface CreateSourceOptions {
+  kind: MemorySourceKind;
+  uri?: string | null;
+  title?: string | null;
+  content: string;
+  scope?: string;
+  tags?: string[];
+}
+
+export interface CreateSourceResult {
+  id: string;
+  duplicate: boolean;
+}
+
+/**
+ * Insert a memory_source. Idempotent — if the content_hash already exists,
+ * returns the existing source's id and duplicate=true instead of inserting.
+ */
+export function createMemorySource(opts: CreateSourceOptions): CreateSourceResult | null {
+  if (!isDbAvailable()) return null;
+  const adapter = _getAdapter();
+  if (!adapter) return null;
+
+  try {
+    const contentHash = hashContent(opts.content);
+    const existing = adapter
+      .prepare("SELECT id FROM memory_sources WHERE content_hash = :h")
+      .get({ ":h": contentHash });
+    if (existing && typeof existing["id"] === "string") {
+      return { id: existing["id"] as string, duplicate: true };
+    }
+
+    const id = newSourceId();
+    insertMemorySourceRow({
+      id,
+      kind: opts.kind,
+      uri: opts.uri ?? null,
+      title: opts.title ?? null,
+      content: opts.content,
+      contentHash,
+      importedAt: new Date().toISOString(),
+      scope: opts.scope ?? "project",
+      tags: opts.tags ?? [],
+    });
+    return { id, duplicate: false };
+  } catch {
+    return null;
+  }
+}
+
+export function listMemorySources(limit = 50): MemorySource[] {
+  if (!isDbAvailable()) return [];
+  const adapter = _getAdapter();
+  if (!adapter) return [];
+  try {
+    const rows = adapter
+      .prepare("SELECT * FROM memory_sources ORDER BY imported_at DESC LIMIT :limit")
+      .all({ ":limit": limit });
+    return rows.map(rowToSource);
+  } catch {
+    return [];
+  }
+}
+
+export function getMemorySource(id: string): MemorySource | null {
+  if (!isDbAvailable()) return null;
+  const adapter = _getAdapter();
+  if (!adapter) return null;
+  try {
+    const row = adapter.prepare("SELECT * FROM memory_sources WHERE id = :id").get({ ":id": id });
+    return row ? rowToSource(row) : null;
+  } catch {
+    return null;
+  }
+}
+
+export function deleteMemorySource(id: string): boolean {
+  if (!isDbAvailable()) return false;
+  try {
+    return deleteMemorySourceRow(id);
+  } catch {
+    return false;
+  }
+}