engrm 0.1.0

package/src/context/inject.ts

@@ -0,0 +1,382 @@
+/**
+ * Context injection for session start.
+ *
+ * When a Claude Code session begins, we search memory for relevant
+ * observations from the current project and inject them as context.
+ * This gives the agent prior knowledge without being asked.
+ *
+ * Optimizations:
+ *   - Token budget (not count limit) prevents context blowup at scale
+ *   - Facts-first: shows facts[] bullets instead of narrative prose (~50% denser)
+ *   - Tiered: top 3 get detail, rest are title-only
+ *   - Blended scoring: quality * 0.6 + recency * 0.4 (recent medium-quality beats old high-quality)
+ */
+
+import { detectProject } from "../storage/projects.js";
+import type { MemDatabase, ObservationRow, SessionSummaryRow } from "../storage/sqlite.js";
+
+export interface ContextOptions {
+  /** Max tokens for context injection (default: 800) */
+  tokenBudget?: number;
+  /** Max observations to return (legacy, overrides tokenBudget if set) */
+  maxCount?: number;
+  /** Number of observations to show with full detail (default: 3) */
+  detailedCount?: number;
+}
+
+export interface InjectedContext {
+  project_name: string;
+  canonical_id: string;
+  observations: ContextObservation[];
+  /** Number of observations included in context */
+  session_count: number;
+  /** Total active observations in project (for footer) */
+  total_active: number;
+  /** Recent session summaries for lessons learned */
+  summaries?: SessionSummaryRow[];
+}
+
+export interface ContextObservation {
+  id: number;
+  type: string;
+  title: string;
+  narrative: string | null;
+  facts: string | null;
+  quality: number;
+  created_at: string;
+}
+
+/** Decay window for recency scoring (30 days in seconds). */
+const RECENCY_WINDOW_SECONDS = 30 * 86400;
+
+/**
+ * Compute a blended relevance score combining quality and recency.
+ * Quality contributes 60%, recency 40%. Both are 0-1 normalised.
+ * Recency decays linearly over 30 days to 0.
+ */
+export function computeBlendedScore(
+  quality: number,
+  createdAtEpoch: number,
+  nowEpoch: number
+): number {
+  const age = nowEpoch - createdAtEpoch;
+  const recencyNorm = Math.max(0, Math.min(1, 1 - age / RECENCY_WINDOW_SECONDS));
+  return quality * 0.6 + recencyNorm * 0.4;
+}
+
+/**
+ * Estimate token count from text.
+ * Uses ~4 chars per token heuristic (standard for English).
+ */
+export function estimateTokens(text: string): number {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Build context for a new session.
+ *
+ * Strategy:
+ *   1. Get pinned observations (always relevant, always included)
+ *   2. Fetch candidates sorted by quality, apply token budget
+ *   3. Tier output: top N detailed, rest title-only
+ */
+export function buildSessionContext(
+  db: MemDatabase,
+  cwd: string,
+  options: ContextOptions | number = {}
+): InjectedContext | null {
+  // Backwards compat: accept number as legacy maxCount
+  const opts: ContextOptions =
+    typeof options === "number" ? { maxCount: options } : options;
+  const tokenBudget = opts.tokenBudget ?? 800;
+  const maxCount = opts.maxCount;
+
+  const detected = detectProject(cwd);
+  const project = db.getProjectByCanonicalId(detected.canonical_id);
+
+  if (!project) {
+    return {
+      project_name: detected.name,
+      canonical_id: detected.canonical_id,
+      observations: [],
+      session_count: 0,
+      total_active: 0,
+    };
+  }
+
+  // Count total active observations for footer (exclude superseded)
+  const totalActive = (
+    db.db
+      .query<{ c: number }, [number]>(
+        `SELECT COUNT(*) as c FROM observations
+         WHERE project_id = ? AND lifecycle IN ('active', 'aging', 'pinned')
+         AND superseded_by IS NULL`
+      )
+      .get(project.id) ?? { c: 0 }
+  ).c;
+
+  // Get pinned observations (always included, capped to prevent budget exhaustion)
+  const MAX_PINNED = 5;
+  const pinned = db.db
+    .query<ObservationRow, [number, number]>(
+      `SELECT * FROM observations
+       WHERE project_id = ? AND lifecycle = 'pinned'
+       AND superseded_by IS NULL
+       ORDER BY quality DESC, created_at_epoch DESC
+       LIMIT ?`
+    )
+    .all(project.id, MAX_PINNED);
+
+  // Fetch candidates (more than we need, we'll trim by token budget)
+  // Exclude superseded observations — they've been replaced by newer ones
+  const candidateLimit = maxCount ?? 50;
+  const candidates = db.db
+    .query<ObservationRow, [number, number]>(
+      `SELECT * FROM observations
+       WHERE project_id = ? AND lifecycle IN ('active', 'aging')
+       AND quality >= 0.3
+       AND superseded_by IS NULL
+       ORDER BY quality DESC, created_at_epoch DESC
+       LIMIT ?`
+    )
+    .all(project.id, candidateLimit);
+
+  // Deduplicate (pinned might overlap with candidates)
+  const seenIds = new Set(pinned.map((o) => o.id));
+  const deduped = candidates.filter((o) => !seenIds.has(o.id));
+
+  // Re-sort candidates by blended score (quality * 0.6 + recency * 0.4)
+  const nowEpoch = Math.floor(Date.now() / 1000);
+  const sorted = [...deduped].sort((a, b) => {
+    const scoreA = computeBlendedScore(a.quality, a.created_at_epoch, nowEpoch);
+    const scoreB = computeBlendedScore(b.quality, b.created_at_epoch, nowEpoch);
+    return scoreB - scoreA; // descending
+  });
+
+  // If using legacy maxCount mode, just slice
+  if (maxCount !== undefined) {
+    const remaining = Math.max(0, maxCount - pinned.length);
+    const all = [...pinned, ...sorted.slice(0, remaining)];
+    return {
+      project_name: project.name,
+      canonical_id: project.canonical_id,
+      observations: all.map(toContextObservation),
+      session_count: all.length,
+      total_active: totalActive,
+    };
+  }
+
+  // Token budget mode: fill greedily
+  // Reserve ~30 tokens for header + footer
+  let remainingBudget = tokenBudget - 30;
+  const selected: ObservationRow[] = [];
+
+  // Pinned always included (deducted from budget)
+  for (const obs of pinned) {
+    const cost = estimateObservationTokens(obs, selected.length);
+    remainingBudget -= cost;
+    selected.push(obs);
+  }
+
+  // Fill with candidates (sorted by blended score) until budget exhausted
+  for (const obs of sorted) {
+    const cost = estimateObservationTokens(obs, selected.length);
+    if (remainingBudget - cost < 0 && selected.length > 0) break;
+    remainingBudget -= cost;
+    selected.push(obs);
+  }
+
+  // Fetch recent session summaries for lessons learned
+  const summaries = db.getRecentSummaries(project.id, 2);
+
+  return {
+    project_name: project.name,
+    canonical_id: project.canonical_id,
+    observations: selected.map(toContextObservation),
+    session_count: selected.length,
+    total_active: totalActive,
+    summaries: summaries.length > 0 ? summaries : undefined,
+  };
+}
+
+/**
+ * Estimate token cost of an observation in context.
+ * Detailed entries (index < 3) cost more than title-only entries.
+ */
+function estimateObservationTokens(
+  obs: ObservationRow,
+  index: number
+): number {
+  const DETAILED_THRESHOLD = 3;
+  // Title line: "- **[type]** title (date, q=0.X)"
+  const titleCost = estimateTokens(
+    `- **[${obs.type}]** ${obs.title} (2026-01-01, q=0.5)`
+  );
+
+  if (index >= DETAILED_THRESHOLD) {
+    return titleCost;
+  }
+
+  // Detailed: title + facts or narrative snippet
+  const detailText = formatObservationDetail(obs);
+  return titleCost + estimateTokens(detailText);
+}
+
+/**
+ * Format injected context as a readable string for Claude.
+ *
+ * Tiered approach:
+ *   - First 3 observations: title + facts (or narrative snippet)
+ *   - Remaining: title-only
+ *   - Footer: "N more observations available via search"
+ */
+export function formatContextForInjection(
+  context: InjectedContext
+): string {
+  if (context.observations.length === 0) {
+    return `Project: ${context.project_name} (no prior observations)`;
+  }
+
+  const DETAILED_COUNT = 3;
+
+  const lines: string[] = [
+    `## Project Memory: ${context.project_name}`,
+    `${context.session_count} relevant observation(s) from prior sessions:`,
+    "",
+  ];
+
+  for (let i = 0; i < context.observations.length; i++) {
+    const obs = context.observations[i]!;
+    const date = obs.created_at.split("T")[0];
+    lines.push(
+      `- **[${obs.type}]** ${obs.title} (${date}, q=${obs.quality.toFixed(1)})`
+    );
+
+    // Detailed tier: show facts or narrative snippet
+    if (i < DETAILED_COUNT) {
+      const detail = formatObservationDetailFromContext(obs);
+      if (detail) {
+        lines.push(detail);
+      }
+    }
+  }
+
+  // Session summaries (lessons from recent sessions)
+  if (context.summaries && context.summaries.length > 0) {
+    lines.push("");
+    lines.push("Lessons from recent sessions:");
+    for (const summary of context.summaries) {
+      if (summary.request) {
+        lines.push(`- Request: ${summary.request}`);
+      }
+      if (summary.learned) {
+        lines.push(`  Learned: ${truncateText(summary.learned, 100)}`);
+      }
+      if (summary.next_steps) {
+        lines.push(`  Next: ${truncateText(summary.next_steps, 80)}`);
+      }
+    }
+  }
+
+  // Footer: how many more are available
+  const remaining = context.total_active - context.session_count;
+  if (remaining > 0) {
+    lines.push("");
+    lines.push(
+      `${remaining} more observation(s) available via search tool.`
+    );
+  }
+
+  return lines.join("\n");
+}
+
+function truncateText(text: string, maxLen: number): string {
+  if (text.length <= maxLen) return text;
+  return text.slice(0, maxLen - 3) + "...";
+}
+
+/**
+ * Format detail for a top-tier observation.
+ * Prefers facts (bullet points, denser) over narrative (prose, verbose).
+ */
+function formatObservationDetailFromContext(
+  obs: ContextObservation
+): string | null {
+  // Try facts first (denser per token)
+  if (obs.facts) {
+    const bullets = parseFacts(obs.facts);
+    if (bullets.length > 0) {
+      return bullets
+        .slice(0, 4) // Cap at 4 facts
+        .map((f) => `  - ${f}`)
+        .join("\n");
+    }
+  }
+
+  // Fall back to narrative snippet
+  if (obs.narrative) {
+    const snippet =
+      obs.narrative.length > 120
+        ? obs.narrative.slice(0, 117) + "..."
+        : obs.narrative;
+    return `  ${snippet}`;
+  }
+
+  return null;
+}
+
+/**
+ * Format detail for token estimation (from ObservationRow).
+ */
+function formatObservationDetail(obs: ObservationRow): string {
+  if (obs.facts) {
+    const bullets = parseFacts(obs.facts);
+    if (bullets.length > 0) {
+      return bullets
+        .slice(0, 4)
+        .map((f) => `  - ${f}`)
+        .join("\n");
+    }
+  }
+  if (obs.narrative) {
+    const snippet =
+      obs.narrative.length > 120
+        ? obs.narrative.slice(0, 117) + "..."
+        : obs.narrative;
+    return `  ${snippet}`;
+  }
+  return "";
+}
+
+/**
+ * Parse facts from stored JSON string.
+ * Handles malformed JSON gracefully.
+ */
+export function parseFacts(facts: string): string[] {
+  if (!facts) return [];
+  try {
+    const parsed = JSON.parse(facts);
+    if (Array.isArray(parsed)) {
+      return parsed.filter((f) => typeof f === "string" && f.length > 0);
+    }
+  } catch {
+    // Not valid JSON — treat as a single fact
+    if (facts.trim().length > 0) {
+      return [facts.trim()];
+    }
+  }
+  return [];
+}
+
+function toContextObservation(obs: ObservationRow): ContextObservation {
+  return {
+    id: obs.id,
+    type: obs.type,
+    title: obs.title,
+    narrative: obs.narrative,
+    facts: obs.facts,
+    quality: obs.quality,
+    created_at: obs.created_at,
+  };
+}

package/src/embeddings/backfill.ts

@@ -0,0 +1,50 @@
+/**
+ * Backfill embeddings for observations that pre-date sqlite-vec.
+ *
+ * Runs on startup, processing a batch of unembedded observations.
+ * Non-blocking — if embedding is unavailable, silently returns.
+ */
+
+import type { MemDatabase } from "../storage/sqlite.js";
+import {
+  composeEmbeddingText,
+  embedText,
+  isEmbeddingAvailable,
+} from "./embedder.js";
+
+export interface BackfillResult {
+  processed: number;
+  failed: number;
+  remaining: number;
+}
+
+/**
+ * Embed observations that don't yet have vectors.
+ * Processes up to `batchSize` per call. Returns counts.
+ */
+export async function backfillEmbeddings(
+  db: MemDatabase,
+  batchSize: number = 50
+): Promise<BackfillResult> {
+  if (!db.vecAvailable) return { processed: 0, failed: 0, remaining: 0 };
+  if (!(await isEmbeddingAvailable()))
+    return { processed: 0, failed: 0, remaining: 0 };
+
+  const observations = db.getUnembeddedObservations(batchSize);
+  let processed = 0;
+  let failed = 0;
+
+  for (const obs of observations) {
+    const text = composeEmbeddingText(obs);
+    const embedding = await embedText(text);
+    if (embedding) {
+      db.vecInsert(obs.id, embedding);
+      processed++;
+    } else {
+      failed++;
+    }
+  }
+
+  const remaining = db.getUnembeddedCount();
+  return { processed, failed, remaining };
+}

package/src/embeddings/embedder.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, test } from "bun:test";
+import { composeEmbeddingText, EMBEDDING_DIMS } from "./embedder.js";
+
+describe("composeEmbeddingText", () => {
+  test("title only", () => {
+    const text = composeEmbeddingText({
+      title: "Fix auth bug",
+      narrative: null,
+      facts: null,
+      concepts: null,
+    });
+    expect(text).toBe("Fix auth bug");
+  });
+
+  test("title + narrative", () => {
+    const text = composeEmbeddingText({
+      title: "Fix auth bug",
+      narrative: "Token refresh was broken",
+      facts: null,
+      concepts: null,
+    });
+    expect(text).toContain("Fix auth bug");
+    expect(text).toContain("Token refresh was broken");
+  });
+
+  test("title + facts as JSON array", () => {
+    const text = composeEmbeddingText({
+      title: "Choose PostgreSQL",
+      narrative: null,
+      facts: JSON.stringify(["Supports JSONB", "Better indexing"]),
+      concepts: null,
+    });
+    expect(text).toContain("Choose PostgreSQL");
+    expect(text).toContain("- Supports JSONB");
+    expect(text).toContain("- Better indexing");
+  });
+
+  test("title + concepts", () => {
+    const text = composeEmbeddingText({
+      title: "Database decision",
+      narrative: null,
+      facts: null,
+      concepts: JSON.stringify(["postgres", "database"]),
+    });
+    expect(text).toContain("Database decision");
+    expect(text).toContain("postgres, database");
+  });
+
+  test("all fields combined", () => {
+    const text = composeEmbeddingText({
+      title: "Fix auth",
+      narrative: "Token expired",
+      facts: JSON.stringify(["Fact 1"]),
+      concepts: JSON.stringify(["auth"]),
+    });
+    expect(text).toContain("Fix auth");
+    expect(text).toContain("Token expired");
+    expect(text).toContain("- Fact 1");
+    expect(text).toContain("auth");
+  });
+
+  test("handles malformed facts JSON gracefully", () => {
+    const text = composeEmbeddingText({
+      title: "Test",
+      narrative: null,
+      facts: "not valid json",
+      concepts: null,
+    });
+    expect(text).toContain("Test");
+    expect(text).toContain("not valid json");
+  });
+
+  test("EMBEDDING_DIMS is 384", () => {
+    expect(EMBEDDING_DIMS).toBe(384);
+  });
+});

package/src/embeddings/embedder.ts

@@ -0,0 +1,139 @@
+/**
+ * Local embedding model for offline semantic search.
+ *
+ * Uses @xenova/transformers to run all-MiniLM-L6-v2 (384 dims)
+ * entirely in-process via ONNX/WASM. No server needed.
+ *
+ * Lazy-loaded on first use — model downloaded on first run (~23MB),
+ * cached in ~/.cache/huggingface/ thereafter.
+ *
+ * Graceful degradation: if model fails to load, all functions
+ * return null and search falls back to FTS5 only.
+ */
+
+import type { ObservationRow } from "../storage/sqlite.js";
+
+// --- State ---
+
+let _available: boolean | null = null; // null = not yet checked
+let _pipeline: any = null;
+
+export const EMBEDDING_DIMS = 384;
+const MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
+
+// --- Public API ---
+
+/**
+ * Check if local embedding is available.
+ * First call triggers model loading.
+ */
+export async function isEmbeddingAvailable(): Promise<boolean> {
+  if (_available !== null) return _available;
+  try {
+    await getPipeline();
+    return _available!;
+  } catch {
+    _available = false;
+    return false;
+  }
+}
+
+/**
+ * Embed a single text string. Returns Float32Array[384] or null if unavailable.
+ */
+export async function embedText(text: string): Promise<Float32Array | null> {
+  const pipe = await getPipeline();
+  if (!pipe) return null;
+
+  try {
+    const output = await pipe(text, { pooling: "mean", normalize: true });
+    return new Float32Array(output.data);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Batch embed multiple texts. More efficient than individual calls.
+ */
+export async function embedTexts(
+  texts: string[]
+): Promise<(Float32Array | null)[]> {
+  if (texts.length === 0) return [];
+
+  const pipe = await getPipeline();
+  if (!pipe) return texts.map(() => null);
+
+  const results: (Float32Array | null)[] = [];
+  // Process one at a time — @xenova/transformers handles batching internally
+  // but individual calls are more resilient to failures
+  for (const text of texts) {
+    try {
+      const output = await pipe(text, { pooling: "mean", normalize: true });
+      results.push(new Float32Array(output.data));
+    } catch {
+      results.push(null);
+    }
+  }
+  return results;
+}
+
+/**
+ * Compose the text to embed from an observation's fields.
+ * Mirrors the content composition in push.ts buildVectorDocument.
+ */
+export function composeEmbeddingText(obs: {
+  title: string;
+  narrative: string | null;
+  facts: string | null;
+  concepts: string | null;
+}): string {
+  const parts = [obs.title];
+
+  if (obs.narrative) parts.push(obs.narrative);
+
+  if (obs.facts) {
+    try {
+      const facts = JSON.parse(obs.facts) as string[];
+      if (Array.isArray(facts) && facts.length > 0) {
+        parts.push(facts.map((f) => `- ${f}`).join("\n"));
+      }
+    } catch {
+      parts.push(obs.facts);
+    }
+  }
+
+  if (obs.concepts) {
+    try {
+      const concepts = JSON.parse(obs.concepts) as string[];
+      if (Array.isArray(concepts) && concepts.length > 0) {
+        parts.push(concepts.join(", "));
+      }
+    } catch {
+      // ignore
+    }
+  }
+
+  return parts.join("\n\n");
+}
+
+// --- Internal ---
+
+async function getPipeline(): Promise<any> {
+  if (_pipeline) return _pipeline;
+  if (_available === false) return null;
+
+  try {
+    const { pipeline } = await import("@xenova/transformers");
+    _pipeline = await pipeline("feature-extraction", MODEL_NAME);
+    _available = true;
+    return _pipeline;
+  } catch (err) {
+    _available = false;
+    // Log once, then silent
+    console.error(
+      `[engrm] Local embedding model unavailable: ${err instanceof Error ? err.message : String(err)}`
+    );
+    return null;
+  }
+}