engrm 0.1.0

package/src/tools/save.ts

@@ -0,0 +1,231 @@
+/**
+ * save_observation MCP tool.
+ *
+ * Pipeline: detect project → scrub secrets → score quality →
+ *           check dedup → insert → add to outbox
+ */
+
+import { relative, isAbsolute } from "node:path";
+import type { Config } from "../config.js";
+import { scrubSecrets, containsSecrets } from "../capture/scrubber.js";
+import { scoreQuality, meetsQualityThreshold } from "../capture/quality.js";
+import { findDuplicate, type DedupCandidate } from "../capture/dedup.js";
+import { detectProject } from "../storage/projects.js";
+import type { MemDatabase, ObservationRow } from "../storage/sqlite.js";
+import { composeEmbeddingText, embedText } from "../embeddings/embedder.js";
+
+const VALID_TYPES = [
+  "bugfix",
+  "discovery",
+  "decision",
+  "pattern",
+  "change",
+  "feature",
+  "refactor",
+  "digest",
+] as const;
+
+type ObservationType = (typeof VALID_TYPES)[number];
+
+export interface SaveObservationInput {
+  type: string;
+  title: string;
+  narrative?: string;
+  facts?: string[];
+  concepts?: string[];
+  files_read?: string[];
+  files_modified?: string[];
+  sensitivity?: "shared" | "personal" | "secret";
+  session_id?: string;
+  cwd?: string;
+  agent?: string;
+}
+
+export interface SaveObservationResult {
+  success: boolean;
+  observation_id?: number;
+  quality_score?: number;
+  merged_into?: number;
+  reason?: string;
+}
+
+/**
+ * Save an observation through the full capture pipeline.
+ */
+export async function saveObservation(
+  db: MemDatabase,
+  config: Config,
+  input: SaveObservationInput
+): Promise<SaveObservationResult> {
+  // Validate type
+  if (!VALID_TYPES.includes(input.type as ObservationType)) {
+    return {
+      success: false,
+      reason: `Invalid type '${input.type}'. Must be one of: ${VALID_TYPES.join(", ")}`,
+    };
+  }
+
+  // Validate title
+  if (!input.title || input.title.trim().length === 0) {
+    return { success: false, reason: "Title is required" };
+  }
+
+  // Detect project from cwd
+  const cwd = input.cwd ?? process.cwd();
+  const detected = detectProject(cwd);
+  const project = db.upsertProject({
+    canonical_id: detected.canonical_id,
+    name: detected.name,
+    local_path: detected.local_path,
+    remote_url: detected.remote_url,
+  });
+
+  // Scrub secrets from all text fields
+  const customPatterns = config.scrubbing.enabled
+    ? config.scrubbing.custom_patterns
+    : [];
+
+  const title = config.scrubbing.enabled
+    ? scrubSecrets(input.title, customPatterns)
+    : input.title;
+
+  const narrative = input.narrative
+    ? config.scrubbing.enabled
+      ? scrubSecrets(input.narrative, customPatterns)
+      : input.narrative
+    : null;
+
+  const factsJson = input.facts
+    ? config.scrubbing.enabled
+      ? scrubSecrets(JSON.stringify(input.facts), customPatterns)
+      : JSON.stringify(input.facts)
+    : null;
+
+  const conceptsJson = input.concepts
+    ? JSON.stringify(input.concepts)
+    : null;
+
+  // Convert absolute paths to project-relative for cross-machine portability
+  const filesRead = input.files_read
+    ? input.files_read.map((f) => toRelativePath(f, cwd))
+    : null;
+
+  const filesModified = input.files_modified
+    ? input.files_modified.map((f) => toRelativePath(f, cwd))
+    : null;
+
+  const filesReadJson = filesRead ? JSON.stringify(filesRead) : null;
+  const filesModifiedJson = filesModified ? JSON.stringify(filesModified) : null;
+
+  // Determine sensitivity
+  let sensitivity = input.sensitivity ?? config.scrubbing.default_sensitivity;
+  if (
+    config.scrubbing.enabled &&
+    containsSecrets(
+      [input.title, input.narrative, JSON.stringify(input.facts)]
+        .filter(Boolean)
+        .join(" "),
+      customPatterns
+    )
+  ) {
+    // Upgrade to 'personal' if secrets detected (even after scrubbing, flag it)
+    if (sensitivity === "shared") {
+      sensitivity = "personal";
+    }
+  }
+
+  // Check deduplication against last 24h
+  const oneDayAgo = Math.floor(Date.now() / 1000) - 86400;
+  const recentObs = db.getRecentObservations(project.id, oneDayAgo);
+  const candidates: DedupCandidate[] = recentObs.map((o) => ({
+    id: o.id,
+    title: o.title,
+  }));
+  const duplicate = findDuplicate(title, candidates);
+
+  // Score quality
+  const qualityInput = {
+    type: input.type,
+    title,
+    narrative,
+    facts: factsJson,
+    concepts: conceptsJson,
+    filesRead: filesRead,
+    filesModified: filesModified,
+    isDuplicate: duplicate !== null,
+  };
+
+  const qualityScore = scoreQuality(qualityInput);
+
+  if (!meetsQualityThreshold(qualityInput)) {
+    return {
+      success: false,
+      quality_score: qualityScore,
+      reason: `Quality score ${qualityScore.toFixed(2)} below threshold`,
+    };
+  }
+
+  // If duplicate found, report merge (future: update existing observation)
+  if (duplicate) {
+    return {
+      success: true,
+      merged_into: duplicate.id,
+      quality_score: qualityScore,
+      reason: `Merged into existing observation #${duplicate.id}`,
+    };
+  }
+
+  // Insert observation
+  const obs = db.insertObservation({
+    session_id: input.session_id ?? null,
+    project_id: project.id,
+    type: input.type,
+    title,
+    narrative,
+    facts: factsJson,
+    concepts: conceptsJson,
+    files_read: filesReadJson,
+    files_modified: filesModifiedJson,
+    quality: qualityScore,
+    lifecycle: "active",
+    sensitivity,
+    user_id: config.user_id,
+    device_id: config.device_id,
+    agent: input.agent ?? "claude-code",
+  });
+
+  // Add to sync outbox
+  db.addToOutbox("observation", obs.id);
+
+  // Embed for local vector search (best-effort, non-blocking on failure)
+  if (db.vecAvailable) {
+    try {
+      const text = composeEmbeddingText(obs);
+      const embedding = await embedText(text);
+      if (embedding) {
+        db.vecInsert(obs.id, embedding);
+      }
+    } catch {
+      // Embedding failure is non-fatal — FTS5 still works
+    }
+  }
+
+  return {
+    success: true,
+    observation_id: obs.id,
+    quality_score: qualityScore,
+  };
+}
+
+/**
+ * Convert an absolute file path to a project-relative path.
+ * Already-relative paths are returned as-is.
+ * Paths outside the project root are returned as-is (no way to make relative).
+ */
+function toRelativePath(filePath: string, projectRoot: string): string {
+  if (!isAbsolute(filePath)) return filePath;
+  const rel = relative(projectRoot, filePath);
+  // If relative() returns a path starting with "..", the file is outside project root
+  if (rel.startsWith("..")) return filePath;
+  return rel;
+}

package/src/tools/search.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, test } from "bun:test";
+import { mergeResults } from "./search.js";
+import type { FtsMatchRow, VecMatchRow } from "../storage/sqlite.js";
+
+describe("mergeResults (RRF)", () => {
+  test("merges FTS-only results", () => {
+    const fts: FtsMatchRow[] = [
+      { id: 1, rank: -5.0 },
+      { id: 2, rank: -3.0 },
+      { id: 3, rank: -1.0 },
+    ];
+    const vec: VecMatchRow[] = [];
+
+    const merged = mergeResults(fts, vec, 10);
+    expect(merged.length).toBe(3);
+    expect(merged[0]!.id).toBe(1); // rank 0 in FTS = highest RRF
+    expect(merged[1]!.id).toBe(2);
+    expect(merged[2]!.id).toBe(3);
+  });
+
+  test("merges vec-only results", () => {
+    const fts: FtsMatchRow[] = [];
+    const vec: VecMatchRow[] = [
+      { observation_id: 10, distance: 0.1 },
+      { observation_id: 20, distance: 0.5 },
+    ];
+
+    const merged = mergeResults(fts, vec, 10);
+    expect(merged.length).toBe(2);
+    expect(merged[0]!.id).toBe(10);
+    expect(merged[1]!.id).toBe(20);
+  });
+
+  test("boosts items appearing in both lists", () => {
+    const fts: FtsMatchRow[] = [
+      { id: 1, rank: -5.0 },
+      { id: 2, rank: -3.0 },
+    ];
+    const vec: VecMatchRow[] = [
+      { observation_id: 2, distance: 0.1 },
+      { observation_id: 3, distance: 0.2 },
+    ];
+
+    const merged = mergeResults(fts, vec, 10);
+    // ID 2 appears in both — should be boosted to top
+    expect(merged[0]!.id).toBe(2);
+    expect(merged[0]!.score).toBeGreaterThan(merged[1]!.score);
+  });
+
+  test("respects limit", () => {
+    const fts: FtsMatchRow[] = [
+      { id: 1, rank: -5 },
+      { id: 2, rank: -4 },
+      { id: 3, rank: -3 },
+    ];
+    const vec: VecMatchRow[] = [
+      { observation_id: 4, distance: 0.1 },
+      { observation_id: 5, distance: 0.2 },
+    ];
+
+    const merged = mergeResults(fts, vec, 3);
+    expect(merged.length).toBe(3);
+  });
+
+  test("handles empty inputs", () => {
+    const merged = mergeResults([], [], 10);
+    expect(merged.length).toBe(0);
+  });
+});

package/src/tools/search.ts

@@ -0,0 +1,181 @@
+/**
+ * search_observations MCP tool.
+ *
+ * Hybrid search: FTS5 (keyword) + sqlite-vec (semantic),
+ * merged via Reciprocal Rank Fusion. Falls back to FTS5-only
+ * when embeddings or sqlite-vec are unavailable.
+ */
+
+import { detectProject } from "../storage/projects.js";
+import type {
+  MemDatabase,
+  FtsMatchRow,
+  VecMatchRow,
+} from "../storage/sqlite.js";
+import { embedText } from "../embeddings/embedder.js";
+
+export interface SearchInput {
+  query: string;
+  project_scoped?: boolean;
+  limit?: number;
+  cwd?: string;
+}
+
+export interface SearchResult {
+  observations: SearchResultEntry[];
+  total: number;
+  project?: string;
+}
+
+export interface SearchResultEntry {
+  id: number;
+  type: string;
+  title: string;
+  narrative: string | null;
+  facts: string | null;
+  concepts: string | null;
+  files_modified: string | null;
+  quality: number;
+  lifecycle: string;
+  created_at: string;
+  rank: number;
+}
+
+/**
+ * Hybrid search: FTS5 keywords + sqlite-vec semantic, merged via RRF.
+ */
+export async function searchObservations(
+  db: MemDatabase,
+  input: SearchInput
+): Promise<SearchResult> {
+  const query = input.query.trim();
+  if (!query) {
+    return { observations: [], total: 0 };
+  }
+
+  const limit = input.limit ?? 10;
+  const projectScoped = input.project_scoped !== false;
+
+  let projectId: number | null = null;
+  let projectName: string | undefined;
+
+  if (projectScoped) {
+    const cwd = input.cwd ?? process.cwd();
+    const detected = detectProject(cwd);
+    const project = db.getProjectByCanonicalId(detected.canonical_id);
+    if (project) {
+      projectId = project.id;
+      projectName = project.name;
+    }
+  }
+
+  // FTS5 keyword search
+  const safeQuery = sanitizeFtsQuery(query);
+  const ftsResults = safeQuery
+    ? db.searchFts(safeQuery, projectId, undefined, limit * 2)
+    : [];
+
+  // Vec semantic search (if available)
+  let vecResults: VecMatchRow[] = [];
+  const queryEmbedding = await embedText(query);
+  if (queryEmbedding && db.vecAvailable) {
+    vecResults = db.searchVec(
+      queryEmbedding,
+      projectId,
+      ["active", "aging", "pinned"],
+      limit * 2
+    );
+  }
+
+  // Merge via RRF if we have both, otherwise use whichever is available
+  const merged = mergeResults(ftsResults, vecResults, limit);
+
+  if (merged.length === 0) {
+    return { observations: [], total: 0, project: projectName };
+  }
+
+  const ids = merged.map((r) => r.id);
+  const scoreMap = new Map(merged.map((r) => [r.id, r.score]));
+  const observations = db.getObservationsByIds(ids);
+
+  // Filter out superseded observations
+  const active = observations.filter((obs) => obs.superseded_by === null);
+
+  // Apply lifecycle weighting
+  const entries: SearchResultEntry[] = active.map((obs) => {
+    const baseScore = scoreMap.get(obs.id) ?? 0;
+    const lifecycleWeight = obs.lifecycle === "aging" ? 0.7 : 1.0;
+
+    return {
+      id: obs.id,
+      type: obs.type,
+      title: obs.title,
+      narrative: obs.narrative,
+      facts: obs.facts,
+      concepts: obs.concepts,
+      files_modified: obs.files_modified,
+      quality: obs.quality,
+      lifecycle: obs.lifecycle,
+      created_at: obs.created_at,
+      rank: baseScore * lifecycleWeight,
+    };
+  });
+
+  // Sort by score (higher = better match)
+  entries.sort((a, b) => b.rank - a.rank);
+
+  return {
+    observations: entries,
+    total: entries.length,
+    project: projectName,
+  };
+}
+
+// --- Reciprocal Rank Fusion ---
+
+const RRF_K = 60;
+
+interface ScoredId {
+  id: number;
+  score: number;
+}
+
+/**
+ * Merge FTS5 and vec results using Reciprocal Rank Fusion.
+ * RRF is rank-based, so no score normalization needed.
+ * Items appearing in both lists get boosted.
+ */
+export function mergeResults(
+  ftsResults: FtsMatchRow[],
+  vecResults: VecMatchRow[],
+  limit: number
+): ScoredId[] {
+  const scores = new Map<number, number>();
+
+  // FTS results sorted by rank (more negative = better match)
+  for (let rank = 0; rank < ftsResults.length; rank++) {
+    const id = ftsResults[rank]!.id;
+    scores.set(id, (scores.get(id) ?? 0) + 1 / (RRF_K + rank + 1));
+  }
+
+  // Vec results sorted by distance (lower = closer)
+  for (let rank = 0; rank < vecResults.length; rank++) {
+    const id = vecResults[rank]!.observation_id;
+    scores.set(id, (scores.get(id) ?? 0) + 1 / (RRF_K + rank + 1));
+  }
+
+  return Array.from(scores.entries())
+    .map(([id, score]) => ({ id, score }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit);
+}
+
+/**
+ * Sanitize a query for FTS5.
+ */
+function sanitizeFtsQuery(query: string): string {
+  let safe = query.replace(/[{}()[\]^~*:]/g, " ");
+  safe = safe.replace(/\s+/g, " ").trim();
+  if (!safe) return "";
+  return safe;
+}

package/src/tools/timeline.ts

@@ -0,0 +1,64 @@
+/**
+ * get_timeline MCP tool.
+ *
+ * Returns chronological observations around an anchor point,
+ * providing temporal context for understanding sequences of events.
+ */
+
+import { detectProject } from "../storage/projects.js";
+import type { MemDatabase, ObservationRow } from "../storage/sqlite.js";
+
+export interface TimelineInput {
+  anchor_id: number;
+  depth_before?: number;
+  depth_after?: number;
+  project_scoped?: boolean;
+  cwd?: string;
+}
+
+export interface TimelineResult {
+  observations: ObservationRow[];
+  anchor_index: number;
+  project?: string;
+}
+
+/**
+ * Get a timeline of observations around an anchor.
+ */
+export function getTimeline(
+  db: MemDatabase,
+  input: TimelineInput
+): TimelineResult {
+  const depthBefore = input.depth_before ?? 3;
+  const depthAfter = input.depth_after ?? 3;
+  const projectScoped = input.project_scoped !== false;
+
+  let projectId: number | null = null;
+  let projectName: string | undefined;
+
+  if (projectScoped) {
+    const cwd = input.cwd ?? process.cwd();
+    const detected = detectProject(cwd);
+    const project = db.getProjectByCanonicalId(detected.canonical_id);
+    if (project) {
+      projectId = project.id;
+      projectName = project.name;
+    }
+  }
+
+  const observations = db.getTimeline(
+    input.anchor_id,
+    projectId,
+    depthBefore,
+    depthAfter
+  );
+
+  // Find the anchor's position in the result
+  const anchorIndex = observations.findIndex((o) => o.id === input.anchor_id);
+
+  return {
+    observations,
+    anchor_index: anchorIndex >= 0 ? anchorIndex : 0,
+    project: projectName,
+  };
+}

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "types": ["bun-types"],
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "dist",
+    "rootDir": "src",
+    "lib": ["ESNext"]
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}