engrm 0.1.0 → 0.2.1

package/src/sync/push.ts

@@ -1,230 +0,0 @@
-/**
- * Push engine: flush sync outbox to Candengo Vector.
- *
- * Reads pending entries from the outbox, builds Vector documents,
- * and pushes them via the REST client. Supports batch operations.
- */
-
-import type { MemDatabase, ObservationRow, SessionSummaryRow } from "../storage/sqlite.js";
-import type { Config } from "../config.js";
-import {
-  getPendingEntries,
-  markSyncing,
-  markSynced,
-  markFailed,
-} from "../storage/outbox.js";
-import { VectorClient, type VectorDocument } from "./client.js";
-import { buildSourceId } from "./auth.js";
-
-export interface PushResult {
-  pushed: number;
-  failed: number;
-  skipped: number;
-}
-
-/**
- * Build a Candengo Vector document from a local observation.
- */
-export function buildVectorDocument(
-  obs: ObservationRow,
-  config: Config,
-  project: { canonical_id: string; name: string }
-): VectorDocument {
-  // Compose content: title + narrative + facts
-  const parts = [obs.title];
-  if (obs.narrative) parts.push(obs.narrative);
-  if (obs.facts) {
-    try {
-      const facts = JSON.parse(obs.facts) as string[];
-      if (Array.isArray(facts) && facts.length > 0) {
-        parts.push("Facts:\n" + facts.map((f) => `- ${f}`).join("\n"));
-      }
-    } catch {
-      // Not valid JSON — use as-is
-      parts.push(obs.facts);
-    }
-  }
-
-  return {
-    site_id: config.site_id,
-    namespace: config.namespace,
-    source_type: obs.type,
-    source_id: buildSourceId(config, obs.id),
-    content: parts.join("\n\n"),
-    metadata: {
-      project_canonical: project.canonical_id,
-      project_name: project.name,
-      user_id: obs.user_id,
-      device_id: obs.device_id,
-      agent: obs.agent,
-      title: obs.title,
-      type: obs.type,
-      quality: obs.quality,
-      concepts: obs.concepts ? JSON.parse(obs.concepts) : [],
-      files_modified: obs.files_modified
-        ? JSON.parse(obs.files_modified)
-        : [],
-      session_id: obs.session_id,
-      created_at_epoch: obs.created_at_epoch,
-      local_id: obs.id,
-    },
-  };
-}
-
-/**
- * Build a Candengo Vector document from a session summary.
- */
-export function buildSummaryVectorDocument(
-  summary: SessionSummaryRow,
-  config: Config,
-  project: { canonical_id: string; name: string }
-): VectorDocument {
-  const parts: string[] = [];
-  if (summary.request) parts.push(`Request: ${summary.request}`);
-  if (summary.learned) parts.push(`Learned: ${summary.learned}`);
-  if (summary.completed) parts.push(`Completed: ${summary.completed}`);
-
-  return {
-    site_id: config.site_id,
-    namespace: config.namespace,
-    source_type: "summary",
-    source_id: buildSourceId(config, summary.id, "summary"),
-    content: parts.join("\n\n"),
-    metadata: {
-      project_canonical: project.canonical_id,
-      project_name: project.name,
-      user_id: summary.user_id,
-      session_id: summary.session_id,
-      created_at_epoch: summary.created_at_epoch,
-      local_id: summary.id,
-    },
-  };
-}
-
-/**
- * Push pending outbox entries to Candengo Vector.
- */
-export async function pushOutbox(
-  db: MemDatabase,
-  client: VectorClient,
-  config: Config,
-  batchSize: number = 50
-): Promise<PushResult> {
-  const entries = getPendingEntries(db, batchSize);
-
-  let pushed = 0;
-  let failed = 0;
-  let skipped = 0;
-
-  // Collect documents for batch ingest
-  const batch: { entryId: number; doc: VectorDocument }[] = [];
-
-  for (const entry of entries) {
-    if (entry.record_type === "summary") {
-      const summary = db.getSessionSummary(
-        // record_id is the summary row id — look it up
-        (() => {
-          const row = db.db
-            .query<{ session_id: string }, [number]>(
-              "SELECT session_id FROM session_summaries WHERE id = ?"
-            )
-            .get(entry.record_id);
-          return row?.session_id ?? "";
-        })()
-      );
-
-      if (!summary || !summary.project_id) {
-        markSynced(db, entry.id);
-        skipped++;
-        continue;
-      }
-
-      const project = db.getProjectById(summary.project_id);
-      if (!project) {
-        markSynced(db, entry.id);
-        skipped++;
-        continue;
-      }
-
-      markSyncing(db, entry.id);
-      const doc = buildSummaryVectorDocument(summary, config, {
-        canonical_id: project.canonical_id,
-        name: project.name,
-      });
-      batch.push({ entryId: entry.id, doc });
-      continue;
-    }
-
-    if (entry.record_type !== "observation") {
-      skipped++;
-      continue;
-    }
-
-    const obs = db.getObservationById(entry.record_id);
-    if (!obs) {
-      // Observation was deleted
-      markSynced(db, entry.id);
-      skipped++;
-      continue;
-    }
-
-    // Don't sync secret observations
-    if (obs.sensitivity === "secret") {
-      markSynced(db, entry.id);
-      skipped++;
-      continue;
-    }
-
-    // Don't sync archived/purged observations (they get removed separately)
-    if (obs.lifecycle === "archived" || obs.lifecycle === "purged") {
-      markSynced(db, entry.id);
-      skipped++;
-      continue;
-    }
-
-    const project = db.getProjectById(obs.project_id);
-    if (!project) {
-      markSynced(db, entry.id);
-      skipped++;
-      continue;
-    }
-
-    markSyncing(db, entry.id);
-
-    const doc = buildVectorDocument(obs, config, {
-      canonical_id: project.canonical_id,
-      name: project.name,
-    });
-
-    batch.push({ entryId: entry.id, doc });
-  }
-
-  if (batch.length === 0) return { pushed, failed, skipped };
-
-  // Try batch ingest first
-  try {
-    await client.batchIngest(batch.map((b) => b.doc));
-    for (const { entryId } of batch) {
-      markSynced(db, entryId);
-      pushed++;
-    }
-  } catch {
-    // Batch failed — fall back to individual ingest
-    for (const { entryId, doc } of batch) {
-      try {
-        await client.ingest(doc);
-        markSynced(db, entryId);
-        pushed++;
-      } catch (err) {
-        markFailed(
-          db,
-          entryId,
-          err instanceof Error ? err.message : String(err)
-        );
-        failed++;
-      }
-    }
-  }
-
-  return { pushed, failed, skipped };
-}

package/src/tools/get.ts

@@ -1,34 +0,0 @@
-/**
- * get_observations MCP tool.
- *
- * Retrieve observations by ID(s). Supports single and batch lookups.
- */
-
-import type { MemDatabase, ObservationRow } from "../storage/sqlite.js";
-
-export interface GetObservationsInput {
-  ids: number[];
-}
-
-export interface GetObservationsResult {
-  observations: ObservationRow[];
-  not_found: number[];
-}
-
-/**
- * Get observations by their IDs.
- */
-export function getObservations(
-  db: MemDatabase,
-  input: GetObservationsInput
-): GetObservationsResult {
-  if (input.ids.length === 0) {
-    return { observations: [], not_found: [] };
-  }
-
-  const observations = db.getObservationsByIds(input.ids);
-  const foundIds = new Set(observations.map((o) => o.id));
-  const notFound = input.ids.filter((id) => !foundIds.has(id));
-
-  return { observations, not_found: notFound };
-}

package/src/tools/pin.ts

@@ -1,47 +0,0 @@
-/**
- * pin_observation MCP tool.
- *
- * Pin or unpin an observation. Pinned observations are excluded
- * from lifecycle aging and archival.
- */
-
-import type { MemDatabase } from "../storage/sqlite.js";
-
-export interface PinInput {
-  id: number;
-  pinned: boolean;
-}
-
-export interface PinResult {
-  success: boolean;
-  reason?: string;
-}
-
-/**
- * Pin or unpin an observation.
- */
-export function pinObservation(
-  db: MemDatabase,
-  input: PinInput
-): PinResult {
-  const success = db.pinObservation(input.id, input.pinned);
-
-  if (!success) {
-    const obs = db.getObservationById(input.id);
-    if (!obs) {
-      return { success: false, reason: `Observation #${input.id} not found` };
-    }
-    if (input.pinned) {
-      return {
-        success: false,
-        reason: `Cannot pin observation in '${obs.lifecycle}' state (must be active or aging)`,
-      };
-    }
-    return {
-      success: false,
-      reason: `Cannot unpin observation in '${obs.lifecycle}' state (must be pinned)`,
-    };
-  }
-
-  return { success: true };
-}

package/src/tools/save.test.ts

@@ -1,301 +0,0 @@
-import { describe, expect, test, beforeEach, afterEach } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import { MemDatabase } from "../storage/sqlite.js";
-import type { Config } from "../config.js";
-import { saveObservation, type SaveObservationInput } from "./save.js";
-
-let db: MemDatabase;
-let tmpDir: string;
-let config: Config;
-
-function makeConfig(overrides?: Partial<Config>): Config {
-  return {
-    candengo_url: "https://api.candengo.com",
-    candengo_api_key: "test-key",
-    site_id: "test-site",
-    namespace: "test-ns",
-    user_id: "david",
-    device_id: "laptop-abc",
-    user_email: "",
-    teams: [],
-    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
-    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
-    scrubbing: {
-      enabled: true,
-      custom_patterns: [],
-      default_sensitivity: "shared",
-    },
-    ...overrides,
-  };
-}
-
-beforeEach(() => {
-  tmpDir = mkdtempSync(join(tmpdir(), "candengo-mem-save-test-"));
-  db = new MemDatabase(join(tmpDir, "test.db"));
-  config = makeConfig();
-});
-
-afterEach(() => {
-  db.close();
-  rmSync(tmpDir, { recursive: true, force: true });
-});
-
-describe("saveObservation", () => {
-  test("saves valid observation", async () => {
-    const result = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "Fix authentication bug",
-      narrative: "The token was not being refreshed properly on expiry, causing 401 errors",
-      cwd: "/Volumes/Data/devs/candengo-mem",
-    });
-    expect(result.success).toBe(true);
-    expect(result.observation_id).toBeGreaterThan(0);
-    expect(result.quality_score).toBeGreaterThan(0);
-  });
-
-  test("rejects invalid type", async () => {
-    const result = await saveObservation(db, config, {
-      type: "invalid",
-      title: "Test",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(false);
-    expect(result.reason).toContain("Invalid type");
-  });
-
-  test("rejects empty title", async () => {
-    const result = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(false);
-    expect(result.reason).toContain("Title is required");
-  });
-
-  test("rejects whitespace-only title", async () => {
-    const result = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "   ",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(false);
-    expect(result.reason).toContain("Title is required");
-  });
-
-  test("rejects low-quality observation", async () => {
-    // A bare "change" type with no enrichment scores 0.05 < threshold 0.1
-    const result = await saveObservation(db, config, {
-      type: "change",
-      title: "Minor tweak",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(false);
-    expect(result.reason).toContain("Quality score");
-  });
-
-  test("scrubs secrets from title", async () => {
-    const result = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "Fix sk-abc123def456ghi789jkl012mno in config",
-      narrative: "Found the API key hardcoded, which is a discovery worth noting for the team",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    expect(obs!.title).toContain("[REDACTED_API_KEY]");
-    expect(obs!.title).not.toContain("sk-abc123");
-  });
-
-  test("scrubs secrets from narrative", async () => {
-    const result = await saveObservation(db, config, {
-      type: "discovery",
-      title: "Found hardcoded credentials",
-      narrative: "Database URL was postgresql://admin:secret@db.example.com/prod",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    expect(obs!.narrative).toContain("[REDACTED_DB_URL]");
-  });
-
-  test("skips scrubbing when disabled", async () => {
-    const noScrubConfig = makeConfig({
-      scrubbing: {
-        enabled: false,
-        custom_patterns: [],
-        default_sensitivity: "shared",
-      },
-    });
-
-    const result = await saveObservation(db, noScrubConfig, {
-      type: "bugfix",
-      title: "Fix sk-abc123def456ghi789jkl012mno",
-      narrative: "This is a long narrative about the API key that was found in the config file and needs to be addressed",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    expect(obs!.title).toContain("sk-abc123");
-  });
-
-  test("upgrades sensitivity when secrets detected", async () => {
-    const result = await saveObservation(db, config, {
-      type: "discovery",
-      title: "Found API key sk-abc123def456ghi789jkl012mno in env",
-      narrative: "Important discovery worth documenting for the team context and future reference",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    expect(obs!.sensitivity).toBe("personal");
-  });
-
-  test("detects duplicates within 24h", async () => {
-    // Save first observation
-    const first = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "Fix authentication token refresh",
-      narrative: "Long enough narrative to pass quality: the token refresh mechanism was broken due to race condition",
-      cwd: "/Volumes/Data/devs/candengo-mem",
-    });
-    expect(first.success).toBe(true);
-
-    // Save near-duplicate
-    const second = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "Fix authentication token refresh",
-      narrative: "Same fix described differently but still about the token refresh race condition issue",
-      cwd: "/Volumes/Data/devs/candengo-mem",
-    });
-    expect(second.success).toBe(true);
-    expect(second.merged_into).toBe(first.observation_id);
-  });
-
-  test("adds observation to sync outbox", async () => {
-    const result = await saveObservation(db, config, {
-      type: "decision",
-      title: "Choose PostgreSQL over MySQL",
-      narrative: "PostgreSQL has better JSON support and more advanced indexing capabilities for our use case",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const outbox = db.db
-      .query<{ record_id: number }, [number]>(
-        "SELECT record_id FROM sync_outbox WHERE record_id = ? AND record_type = 'observation'"
-      )
-      .get(result.observation_id!);
-    expect(outbox).not.toBeNull();
-  });
-
-  test("stores facts and concepts as JSON", async () => {
-    const result = await saveObservation(db, config, {
-      type: "decision",
-      title: "Choose PostgreSQL",
-      facts: ["Supports JSONB", "Better indexing"],
-      concepts: ["database", "postgresql"],
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    expect(JSON.parse(obs!.facts!)).toEqual(["Supports JSONB", "Better indexing"]);
-    expect(JSON.parse(obs!.concepts!)).toEqual(["database", "postgresql"]);
-  });
-
-  test("saves with session_id", async () => {
-    const result = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "Fix auth bug",
-      narrative: "Important fix that required significant investigation and multiple file changes",
-      session_id: "sess-123",
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    expect(obs!.session_id).toBe("sess-123");
-  });
-
-  test("converts absolute file paths to project-relative", async () => {
-    const result = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "Fix path handling",
-      narrative: "Important fix to ensure file paths are stored relative to the project root directory",
-      files_read: ["/projects/myapp/src/auth.ts", "/projects/myapp/README.md"],
-      files_modified: ["/projects/myapp/src/auth.ts"],
-      cwd: "/projects/myapp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    const filesRead = JSON.parse(obs!.files_read!);
-    const filesModified = JSON.parse(obs!.files_modified!);
-    expect(filesRead).toEqual(["src/auth.ts", "README.md"]);
-    expect(filesModified).toEqual(["src/auth.ts"]);
-  });
-
-  test("leaves already-relative paths unchanged", async () => {
-    const result = await saveObservation(db, config, {
-      type: "bugfix",
-      title: "Fix with relative paths",
-      narrative: "Another important fix that tests the relative path handling for already relative paths",
-      files_modified: ["src/auth.ts", "lib/utils.ts"],
-      cwd: "/tmp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    const filesModified = JSON.parse(obs!.files_modified!);
-    expect(filesModified).toEqual(["src/auth.ts", "lib/utils.ts"]);
-  });
-
-  test("keeps paths outside project root as-is", async () => {
-    const result = await saveObservation(db, config, {
-      type: "discovery",
-      title: "Found external dependency issue",
-      narrative: "Discovered a problem in an external file outside the project root directory boundary",
-      files_read: ["/usr/local/lib/node_modules/pkg/index.js"],
-      cwd: "/projects/myapp",
-    });
-    expect(result.success).toBe(true);
-
-    const obs = db.getObservationById(result.observation_id!);
-    const filesRead = JSON.parse(obs!.files_read!);
-    expect(filesRead).toEqual(["/usr/local/lib/node_modules/pkg/index.js"]);
-  });
-
-  test("all valid types accepted", async () => {
-    const types = [
-      "bugfix",
-      "discovery",
-      "decision",
-      "pattern",
-      "change",
-      "feature",
-      "refactor",
-      "digest",
-    ];
-
-    for (const type of types) {
-      const result = await saveObservation(db, config, {
-        type,
-        title: `Test ${type} observation`,
-        narrative: "Detailed narrative to ensure quality threshold is met for all observation types",
-        facts: ["fact1", "fact2"],
-        cwd: "/tmp",
-      });
-      // Some low-scoring types may not meet threshold, but they shouldn't fail with "Invalid type"
-      if (!result.success) {
-        expect(result.reason).not.toContain("Invalid type");
-      }
-    }
-  });
-});