engrm 0.1.0

package/src/sync/push.ts

@@ -0,0 +1,230 @@
+/**
+ * Push engine: flush sync outbox to Candengo Vector.
+ *
+ * Reads pending entries from the outbox, builds Vector documents,
+ * and pushes them via the REST client. Supports batch operations.
+ */
+
+import type { MemDatabase, ObservationRow, SessionSummaryRow } from "../storage/sqlite.js";
+import type { Config } from "../config.js";
+import {
+  getPendingEntries,
+  markSyncing,
+  markSynced,
+  markFailed,
+} from "../storage/outbox.js";
+import { VectorClient, type VectorDocument } from "./client.js";
+import { buildSourceId } from "./auth.js";
+
+export interface PushResult {
+  pushed: number;
+  failed: number;
+  skipped: number;
+}
+
+/**
+ * Build a Candengo Vector document from a local observation.
+ */
+export function buildVectorDocument(
+  obs: ObservationRow,
+  config: Config,
+  project: { canonical_id: string; name: string }
+): VectorDocument {
+  // Compose content: title + narrative + facts
+  const parts = [obs.title];
+  if (obs.narrative) parts.push(obs.narrative);
+  if (obs.facts) {
+    try {
+      const facts = JSON.parse(obs.facts) as string[];
+      if (Array.isArray(facts) && facts.length > 0) {
+        parts.push("Facts:\n" + facts.map((f) => `- ${f}`).join("\n"));
+      }
+    } catch {
+      // Not valid JSON — use as-is
+      parts.push(obs.facts);
+    }
+  }
+
+  return {
+    site_id: config.site_id,
+    namespace: config.namespace,
+    source_type: obs.type,
+    source_id: buildSourceId(config, obs.id),
+    content: parts.join("\n\n"),
+    metadata: {
+      project_canonical: project.canonical_id,
+      project_name: project.name,
+      user_id: obs.user_id,
+      device_id: obs.device_id,
+      agent: obs.agent,
+      title: obs.title,
+      type: obs.type,
+      quality: obs.quality,
+      concepts: obs.concepts ? JSON.parse(obs.concepts) : [],
+      files_modified: obs.files_modified
+        ? JSON.parse(obs.files_modified)
+        : [],
+      session_id: obs.session_id,
+      created_at_epoch: obs.created_at_epoch,
+      local_id: obs.id,
+    },
+  };
+}
+
+/**
+ * Build a Candengo Vector document from a session summary.
+ */
+export function buildSummaryVectorDocument(
+  summary: SessionSummaryRow,
+  config: Config,
+  project: { canonical_id: string; name: string }
+): VectorDocument {
+  const parts: string[] = [];
+  if (summary.request) parts.push(`Request: ${summary.request}`);
+  if (summary.learned) parts.push(`Learned: ${summary.learned}`);
+  if (summary.completed) parts.push(`Completed: ${summary.completed}`);
+
+  return {
+    site_id: config.site_id,
+    namespace: config.namespace,
+    source_type: "summary",
+    source_id: buildSourceId(config, summary.id, "summary"),
+    content: parts.join("\n\n"),
+    metadata: {
+      project_canonical: project.canonical_id,
+      project_name: project.name,
+      user_id: summary.user_id,
+      session_id: summary.session_id,
+      created_at_epoch: summary.created_at_epoch,
+      local_id: summary.id,
+    },
+  };
+}
+
+/**
+ * Push pending outbox entries to Candengo Vector.
+ */
+export async function pushOutbox(
+  db: MemDatabase,
+  client: VectorClient,
+  config: Config,
+  batchSize: number = 50
+): Promise<PushResult> {
+  const entries = getPendingEntries(db, batchSize);
+
+  let pushed = 0;
+  let failed = 0;
+  let skipped = 0;
+
+  // Collect documents for batch ingest
+  const batch: { entryId: number; doc: VectorDocument }[] = [];
+
+  for (const entry of entries) {
+    if (entry.record_type === "summary") {
+      const summary = db.getSessionSummary(
+        // record_id is the summary row id — look it up
+        (() => {
+          const row = db.db
+            .query<{ session_id: string }, [number]>(
+              "SELECT session_id FROM session_summaries WHERE id = ?"
+            )
+            .get(entry.record_id);
+          return row?.session_id ?? "";
+        })()
+      );
+
+      if (!summary || !summary.project_id) {
+        markSynced(db, entry.id);
+        skipped++;
+        continue;
+      }
+
+      const project = db.getProjectById(summary.project_id);
+      if (!project) {
+        markSynced(db, entry.id);
+        skipped++;
+        continue;
+      }
+
+      markSyncing(db, entry.id);
+      const doc = buildSummaryVectorDocument(summary, config, {
+        canonical_id: project.canonical_id,
+        name: project.name,
+      });
+      batch.push({ entryId: entry.id, doc });
+      continue;
+    }
+
+    if (entry.record_type !== "observation") {
+      skipped++;
+      continue;
+    }
+
+    const obs = db.getObservationById(entry.record_id);
+    if (!obs) {
+      // Observation was deleted
+      markSynced(db, entry.id);
+      skipped++;
+      continue;
+    }
+
+    // Don't sync secret observations
+    if (obs.sensitivity === "secret") {
+      markSynced(db, entry.id);
+      skipped++;
+      continue;
+    }
+
+    // Don't sync archived/purged observations (they get removed separately)
+    if (obs.lifecycle === "archived" || obs.lifecycle === "purged") {
+      markSynced(db, entry.id);
+      skipped++;
+      continue;
+    }
+
+    const project = db.getProjectById(obs.project_id);
+    if (!project) {
+      markSynced(db, entry.id);
+      skipped++;
+      continue;
+    }
+
+    markSyncing(db, entry.id);
+
+    const doc = buildVectorDocument(obs, config, {
+      canonical_id: project.canonical_id,
+      name: project.name,
+    });
+
+    batch.push({ entryId: entry.id, doc });
+  }
+
+  if (batch.length === 0) return { pushed, failed, skipped };
+
+  // Try batch ingest first
+  try {
+    await client.batchIngest(batch.map((b) => b.doc));
+    for (const { entryId } of batch) {
+      markSynced(db, entryId);
+      pushed++;
+    }
+  } catch {
+    // Batch failed — fall back to individual ingest
+    for (const { entryId, doc } of batch) {
+      try {
+        await client.ingest(doc);
+        markSynced(db, entryId);
+        pushed++;
+      } catch (err) {
+        markFailed(
+          db,
+          entryId,
+          err instanceof Error ? err.message : String(err)
+        );
+        failed++;
+      }
+    }
+  }
+
+  return { pushed, failed, skipped };
+}

package/src/tools/get.ts

@@ -0,0 +1,34 @@
+/**
+ * get_observations MCP tool.
+ *
+ * Retrieve observations by ID(s). Supports single and batch lookups.
+ */
+
+import type { MemDatabase, ObservationRow } from "../storage/sqlite.js";
+
+export interface GetObservationsInput {
+  ids: number[];
+}
+
+export interface GetObservationsResult {
+  observations: ObservationRow[];
+  not_found: number[];
+}
+
+/**
+ * Get observations by their IDs.
+ */
+export function getObservations(
+  db: MemDatabase,
+  input: GetObservationsInput
+): GetObservationsResult {
+  if (input.ids.length === 0) {
+    return { observations: [], not_found: [] };
+  }
+
+  const observations = db.getObservationsByIds(input.ids);
+  const foundIds = new Set(observations.map((o) => o.id));
+  const notFound = input.ids.filter((id) => !foundIds.has(id));
+
+  return { observations, not_found: notFound };
+}

package/src/tools/pin.ts

@@ -0,0 +1,47 @@
+/**
+ * pin_observation MCP tool.
+ *
+ * Pin or unpin an observation. Pinned observations are excluded
+ * from lifecycle aging and archival.
+ */
+
+import type { MemDatabase } from "../storage/sqlite.js";
+
+export interface PinInput {
+  id: number;
+  pinned: boolean;
+}
+
+export interface PinResult {
+  success: boolean;
+  reason?: string;
+}
+
+/**
+ * Pin or unpin an observation.
+ */
+export function pinObservation(
+  db: MemDatabase,
+  input: PinInput
+): PinResult {
+  const success = db.pinObservation(input.id, input.pinned);
+
+  if (!success) {
+    const obs = db.getObservationById(input.id);
+    if (!obs) {
+      return { success: false, reason: `Observation #${input.id} not found` };
+    }
+    if (input.pinned) {
+      return {
+        success: false,
+        reason: `Cannot pin observation in '${obs.lifecycle}' state (must be active or aging)`,
+      };
+    }
+    return {
+      success: false,
+      reason: `Cannot unpin observation in '${obs.lifecycle}' state (must be pinned)`,
+    };
+  }
+
+  return { success: true };
+}

package/src/tools/save.test.ts

@@ -0,0 +1,301 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "../storage/sqlite.js";
+import type { Config } from "../config.js";
+import { saveObservation, type SaveObservationInput } from "./save.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+let config: Config;
+
+function makeConfig(overrides?: Partial<Config>): Config {
+  return {
+    candengo_url: "https://api.candengo.com",
+    candengo_api_key: "test-key",
+    site_id: "test-site",
+    namespace: "test-ns",
+    user_id: "david",
+    device_id: "laptop-abc",
+    user_email: "",
+    teams: [],
+    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
+    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
+    scrubbing: {
+      enabled: true,
+      custom_patterns: [],
+      default_sensitivity: "shared",
+    },
+    ...overrides,
+  };
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "candengo-mem-save-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+  config = makeConfig();
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("saveObservation", () => {
+  test("saves valid observation", async () => {
+    const result = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "Fix authentication bug",
+      narrative: "The token was not being refreshed properly on expiry, causing 401 errors",
+      cwd: "/Volumes/Data/devs/candengo-mem",
+    });
+    expect(result.success).toBe(true);
+    expect(result.observation_id).toBeGreaterThan(0);
+    expect(result.quality_score).toBeGreaterThan(0);
+  });
+
+  test("rejects invalid type", async () => {
+    const result = await saveObservation(db, config, {
+      type: "invalid",
+      title: "Test",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(false);
+    expect(result.reason).toContain("Invalid type");
+  });
+
+  test("rejects empty title", async () => {
+    const result = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(false);
+    expect(result.reason).toContain("Title is required");
+  });
+
+  test("rejects whitespace-only title", async () => {
+    const result = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "   ",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(false);
+    expect(result.reason).toContain("Title is required");
+  });
+
+  test("rejects low-quality observation", async () => {
+    // A bare "change" type with no enrichment scores 0.05 < threshold 0.1
+    const result = await saveObservation(db, config, {
+      type: "change",
+      title: "Minor tweak",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(false);
+    expect(result.reason).toContain("Quality score");
+  });
+
+  test("scrubs secrets from title", async () => {
+    const result = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "Fix sk-abc123def456ghi789jkl012mno in config",
+      narrative: "Found the API key hardcoded, which is a discovery worth noting for the team",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    expect(obs!.title).toContain("[REDACTED_API_KEY]");
+    expect(obs!.title).not.toContain("sk-abc123");
+  });
+
+  test("scrubs secrets from narrative", async () => {
+    const result = await saveObservation(db, config, {
+      type: "discovery",
+      title: "Found hardcoded credentials",
+      narrative: "Database URL was postgresql://admin:secret@db.example.com/prod",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    expect(obs!.narrative).toContain("[REDACTED_DB_URL]");
+  });
+
+  test("skips scrubbing when disabled", async () => {
+    const noScrubConfig = makeConfig({
+      scrubbing: {
+        enabled: false,
+        custom_patterns: [],
+        default_sensitivity: "shared",
+      },
+    });
+
+    const result = await saveObservation(db, noScrubConfig, {
+      type: "bugfix",
+      title: "Fix sk-abc123def456ghi789jkl012mno",
+      narrative: "This is a long narrative about the API key that was found in the config file and needs to be addressed",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    expect(obs!.title).toContain("sk-abc123");
+  });
+
+  test("upgrades sensitivity when secrets detected", async () => {
+    const result = await saveObservation(db, config, {
+      type: "discovery",
+      title: "Found API key sk-abc123def456ghi789jkl012mno in env",
+      narrative: "Important discovery worth documenting for the team context and future reference",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    expect(obs!.sensitivity).toBe("personal");
+  });
+
+  test("detects duplicates within 24h", async () => {
+    // Save first observation
+    const first = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "Fix authentication token refresh",
+      narrative: "Long enough narrative to pass quality: the token refresh mechanism was broken due to race condition",
+      cwd: "/Volumes/Data/devs/candengo-mem",
+    });
+    expect(first.success).toBe(true);
+
+    // Save near-duplicate
+    const second = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "Fix authentication token refresh",
+      narrative: "Same fix described differently but still about the token refresh race condition issue",
+      cwd: "/Volumes/Data/devs/candengo-mem",
+    });
+    expect(second.success).toBe(true);
+    expect(second.merged_into).toBe(first.observation_id);
+  });
+
+  test("adds observation to sync outbox", async () => {
+    const result = await saveObservation(db, config, {
+      type: "decision",
+      title: "Choose PostgreSQL over MySQL",
+      narrative: "PostgreSQL has better JSON support and more advanced indexing capabilities for our use case",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const outbox = db.db
+      .query<{ record_id: number }, [number]>(
+        "SELECT record_id FROM sync_outbox WHERE record_id = ? AND record_type = 'observation'"
+      )
+      .get(result.observation_id!);
+    expect(outbox).not.toBeNull();
+  });
+
+  test("stores facts and concepts as JSON", async () => {
+    const result = await saveObservation(db, config, {
+      type: "decision",
+      title: "Choose PostgreSQL",
+      facts: ["Supports JSONB", "Better indexing"],
+      concepts: ["database", "postgresql"],
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    expect(JSON.parse(obs!.facts!)).toEqual(["Supports JSONB", "Better indexing"]);
+    expect(JSON.parse(obs!.concepts!)).toEqual(["database", "postgresql"]);
+  });
+
+  test("saves with session_id", async () => {
+    const result = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "Fix auth bug",
+      narrative: "Important fix that required significant investigation and multiple file changes",
+      session_id: "sess-123",
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    expect(obs!.session_id).toBe("sess-123");
+  });
+
+  test("converts absolute file paths to project-relative", async () => {
+    const result = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "Fix path handling",
+      narrative: "Important fix to ensure file paths are stored relative to the project root directory",
+      files_read: ["/projects/myapp/src/auth.ts", "/projects/myapp/README.md"],
+      files_modified: ["/projects/myapp/src/auth.ts"],
+      cwd: "/projects/myapp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    const filesRead = JSON.parse(obs!.files_read!);
+    const filesModified = JSON.parse(obs!.files_modified!);
+    expect(filesRead).toEqual(["src/auth.ts", "README.md"]);
+    expect(filesModified).toEqual(["src/auth.ts"]);
+  });
+
+  test("leaves already-relative paths unchanged", async () => {
+    const result = await saveObservation(db, config, {
+      type: "bugfix",
+      title: "Fix with relative paths",
+      narrative: "Another important fix that tests the relative path handling for already relative paths",
+      files_modified: ["src/auth.ts", "lib/utils.ts"],
+      cwd: "/tmp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    const filesModified = JSON.parse(obs!.files_modified!);
+    expect(filesModified).toEqual(["src/auth.ts", "lib/utils.ts"]);
+  });
+
+  test("keeps paths outside project root as-is", async () => {
+    const result = await saveObservation(db, config, {
+      type: "discovery",
+      title: "Found external dependency issue",
+      narrative: "Discovered a problem in an external file outside the project root directory boundary",
+      files_read: ["/usr/local/lib/node_modules/pkg/index.js"],
+      cwd: "/projects/myapp",
+    });
+    expect(result.success).toBe(true);
+
+    const obs = db.getObservationById(result.observation_id!);
+    const filesRead = JSON.parse(obs!.files_read!);
+    expect(filesRead).toEqual(["/usr/local/lib/node_modules/pkg/index.js"]);
+  });
+
+  test("all valid types accepted", async () => {
+    const types = [
+      "bugfix",
+      "discovery",
+      "decision",
+      "pattern",
+      "change",
+      "feature",
+      "refactor",
+      "digest",
+    ];
+
+    for (const type of types) {
+      const result = await saveObservation(db, config, {
+        type,
+        title: `Test ${type} observation`,
+        narrative: "Detailed narrative to ensure quality threshold is met for all observation types",
+        facts: ["fact1", "fact2"],
+        cwd: "/tmp",
+      });
+      // Some low-scoring types may not meet threshold, but they shouldn't fail with "Invalid type"
+      if (!result.success) {
+        expect(result.reason).not.toContain("Invalid type");
+      }
+    }
+  });
+});