engrm 0.1.0

package/src/sync/engine.ts

@@ -0,0 +1,127 @@
+/**
+ * Sync engine orchestrator.
+ *
+ * Manages push/pull timers and startup backfill.
+ * If sync is not configured (no API key), the engine is a no-op.
+ *
+ * Timers:
+ *   - Push: every config.sync.interval_seconds (default 30s)
+ *   - Pull: every 60s
+ */
+
+import type { MemDatabase } from "../storage/sqlite.js";
+import type { Config } from "../config.js";
+import { VectorClient } from "./client.js";
+import { pushOutbox } from "./push.js";
+import { pullFromVector } from "./pull.js";
+
+const DEFAULT_PULL_INTERVAL = 60_000; // 60 seconds
+
+export class SyncEngine {
+  private client: VectorClient | null = null;
+  private pushTimer: ReturnType<typeof setInterval> | null = null;
+  private pullTimer: ReturnType<typeof setInterval> | null = null;
+  private _pushing = false;
+  private _pulling = false;
+  private _running = false;
+
+  constructor(
+    private readonly db: MemDatabase,
+    private readonly config: Config
+  ) {
+    if (VectorClient.isConfigured(config)) {
+      try {
+        this.client = new VectorClient(config);
+      } catch {
+        // Configuration invalid — stay in offline mode
+      }
+    }
+  }
+
+  /**
+   * Start the sync engine.
+   * Runs startup backfill, then sets up push/pull timers.
+   */
+  start(): void {
+    if (!this.client || !this.config.sync.enabled) {
+      this._running = false;
+      return;
+    }
+
+    this._running = true;
+
+    // Startup backfill: push any pending outbox items
+    this.pushNow().catch(() => {});
+
+    // Push timer
+    const pushInterval = this.config.sync.interval_seconds * 1000;
+    this.pushTimer = setInterval(() => {
+      this.pushNow().catch(() => {});
+    }, pushInterval);
+
+    // Pull timer
+    this.pullTimer = setInterval(() => {
+      this.pullNow().catch(() => {});
+    }, DEFAULT_PULL_INTERVAL);
+  }
+
+  /**
+   * Stop all timers and clean up.
+   */
+  stop(): void {
+    if (this.pushTimer) {
+      clearInterval(this.pushTimer);
+      this.pushTimer = null;
+    }
+    if (this.pullTimer) {
+      clearInterval(this.pullTimer);
+      this.pullTimer = null;
+    }
+    this._running = false;
+  }
+
+  /**
+   * Force an immediate push of pending outbox items.
+   */
+  async pushNow(): Promise<void> {
+    if (!this.client || this._pushing) return;
+    this._pushing = true;
+    try {
+      await pushOutbox(
+        this.db,
+        this.client,
+        this.config,
+        this.config.sync.batch_size
+      );
+    } finally {
+      this._pushing = false;
+    }
+  }
+
+  /**
+   * Force an immediate pull from Candengo Vector.
+   */
+  async pullNow(): Promise<void> {
+    if (!this.client || this._pulling) return;
+    this._pulling = true;
+    try {
+      await pullFromVector(this.db, this.client, this.config);
+    } finally {
+      this._pulling = false;
+    }
+  }
+
+  /**
+   * Is the sync engine running?
+   */
+  isRunning(): boolean {
+    return this._running;
+  }
+
+  /**
+   * Is the client configured for remote sync?
+   */
+  isConfigured(): boolean {
+    return this.client !== null;
+  }
+}

package/src/sync/pull.test.ts

@@ -0,0 +1,279 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "../storage/sqlite.js";
+import type { Config } from "../config.js";
+import { pullFromVector } from "./pull.js";
+import type {
+  VectorClient,
+  VectorChangeFeedResponse,
+  VectorSearchResult,
+} from "./client.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+
+function makeConfig(overrides: Partial<Config> = {}): Config {
+  return {
+    candengo_url: "https://candengo.com",
+    candengo_api_key: "cvk_test123",
+    site_id: "test-site",
+    namespace: "dev-memory",
+    user_id: "david",
+    user_email: "",
+    device_id: "laptop-abc",
+    teams: [],
+    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
+    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
+    scrubbing: {
+      enabled: true,
+      custom_patterns: [],
+      default_sensitivity: "shared",
+    },
+  };
+}
+
+function makeChange(
+  id: number,
+  overrides: Partial<VectorSearchResult> = {}
+): VectorSearchResult {
+  return {
+    source_id:
+      overrides.source_id ?? `other-user-other-device-obs-${id}`,
+    content: overrides.content ?? `Title ${id}\n\nNarrative for ${id}`,
+    score: 1.0,
+    metadata: overrides.metadata ?? {
+      project_canonical: "github.com/test/repo",
+      project_name: "repo",
+      type: "discovery",
+      title: `Title ${id}`,
+      user_id: "other-user",
+      device_id: "other-device",
+      agent: "claude-code",
+      quality: 0.7,
+    },
+  };
+}
+
+function mockClient(
+  pages: VectorChangeFeedResponse[]
+): VectorClient {
+  let callCount = 0;
+  return {
+    pullChanges: async (
+      _cursor?: string,
+      _limit?: number
+    ): Promise<VectorChangeFeedResponse> => {
+      const page = pages[callCount] ?? {
+        changes: [],
+        cursor: "end",
+        has_more: false,
+      };
+      callCount++;
+      return page;
+    },
+  } as unknown as VectorClient;
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "engrm-pull-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("pullFromVector", () => {
+  test("merges single page of changes", async () => {
+    const client = mockClient([
+      {
+        changes: [makeChange(1), makeChange(2)],
+        cursor: "cursor-1",
+        has_more: false,
+      },
+    ]);
+
+    const result = await pullFromVector(db, client, makeConfig());
+    expect(result.received).toBe(2);
+    expect(result.merged).toBe(2);
+    expect(result.skipped).toBe(0);
+  });
+
+  test("loops on has_more", async () => {
+    const client = mockClient([
+      {
+        changes: [makeChange(1), makeChange(2)],
+        cursor: "cursor-1",
+        has_more: true,
+      },
+      {
+        changes: [makeChange(3)],
+        cursor: "cursor-2",
+        has_more: false,
+      },
+    ]);
+
+    const result = await pullFromVector(db, client, makeConfig());
+    expect(result.received).toBe(3);
+    expect(result.merged).toBe(3);
+  });
+
+  test("loops through multiple pages", async () => {
+    const client = mockClient([
+      {
+        changes: [makeChange(1)],
+        cursor: "c1",
+        has_more: true,
+      },
+      {
+        changes: [makeChange(2)],
+        cursor: "c2",
+        has_more: true,
+      },
+      {
+        changes: [makeChange(3)],
+        cursor: "c3",
+        has_more: false,
+      },
+    ]);
+
+    const result = await pullFromVector(db, client, makeConfig());
+    expect(result.received).toBe(3);
+    expect(result.merged).toBe(3);
+  });
+
+  test("stops at MAX_PAGES safety limit", async () => {
+    // Create 25 pages (MAX_PAGES = 20)
+    const pages: VectorChangeFeedResponse[] = Array.from(
+      { length: 25 },
+      (_, i) => ({
+        changes: [makeChange(i + 1)],
+        cursor: `cursor-${i + 1}`,
+        has_more: true,
+      })
+    );
+
+    const client = mockClient(pages);
+    const result = await pullFromVector(db, client, makeConfig());
+    // Should stop at 20 pages
+    expect(result.received).toBe(20);
+    expect(result.merged).toBe(20);
+  });
+
+  test("skips observations from own device", async () => {
+    const ownChange: VectorSearchResult = {
+      source_id: "david-laptop-abc-obs-1",
+      content: "Title\n\nNarrative",
+      score: 1.0,
+      metadata: {
+        project_canonical: "github.com/test/repo",
+        type: "discovery",
+        title: "Title",
+        user_id: "david",
+        device_id: "laptop-abc",
+        agent: "claude-code",
+        quality: 0.7,
+      },
+    };
+
+    const client = mockClient([
+      {
+        changes: [ownChange, makeChange(2)],
+        cursor: "c1",
+        has_more: false,
+      },
+    ]);
+
+    const result = await pullFromVector(db, client, makeConfig());
+    expect(result.received).toBe(2);
+    expect(result.merged).toBe(1);
+    expect(result.skipped).toBe(1);
+  });
+
+  test("skips duplicate remote_source_id", async () => {
+    const client = mockClient([
+      {
+        changes: [makeChange(1)],
+        cursor: "c1",
+        has_more: false,
+      },
+    ]);
+
+    // Pull once
+    await pullFromVector(db, client, makeConfig());
+
+    // Pull same change again
+    const client2 = mockClient([
+      {
+        changes: [makeChange(1)],
+        cursor: "c2",
+        has_more: false,
+      },
+    ]);
+
+    const result = await pullFromVector(db, client2, makeConfig());
+    expect(result.merged).toBe(0);
+    expect(result.skipped).toBe(1);
+  });
+
+  test("saves cursor after each page", async () => {
+    const client = mockClient([
+      {
+        changes: [makeChange(1)],
+        cursor: "cursor-page1",
+        has_more: true,
+      },
+      {
+        changes: [makeChange(2)],
+        cursor: "cursor-page2",
+        has_more: false,
+      },
+    ]);
+
+    await pullFromVector(db, client, makeConfig());
+    expect(db.getSyncState("pull_cursor")).toBe("cursor-page2");
+  });
+
+  test("handles empty response", async () => {
+    const client = mockClient([
+      {
+        changes: [],
+        cursor: "",
+        has_more: false,
+      },
+    ]);
+
+    const result = await pullFromVector(db, client, makeConfig());
+    expect(result.received).toBe(0);
+    expect(result.merged).toBe(0);
+  });
+
+  test("skips changes without project_canonical", async () => {
+    const noProject: VectorSearchResult = {
+      source_id: "other-other-obs-1",
+      content: "Title\n\nNarrative",
+      score: 1.0,
+      metadata: {
+        type: "discovery",
+        title: "Title",
+        user_id: "other",
+        device_id: "other",
+      },
+    };
+
+    const client = mockClient([
+      {
+        changes: [noProject],
+        cursor: "c1",
+        has_more: false,
+      },
+    ]);
+
+    const result = await pullFromVector(db, client, makeConfig());
+    expect(result.skipped).toBe(1);
+    expect(result.merged).toBe(0);
+  });
+});

package/src/sync/pull.ts

@@ -0,0 +1,170 @@
+/**
+ * Pull engine: fetch observations from Candengo Vector change feed
+ * and merge into local SQLite.
+ *
+ * Uses server-side cursors to track position in the change feed.
+ * Skips observations from the current device (already local).
+ */
+
+import type { MemDatabase } from "../storage/sqlite.js";
+import type { Config } from "../config.js";
+import { VectorClient, type VectorSearchResult } from "./client.js";
+import { parseSourceId } from "./auth.js";
+import { composeEmbeddingText, embedText } from "../embeddings/embedder.js";
+
+const PULL_CURSOR_KEY = "pull_cursor";
+
+export interface PullResult {
+  received: number;
+  merged: number;
+  skipped: number;
+}
+
+/** Safety limit to prevent infinite loops if server keeps returning has_more. */
+const MAX_PAGES = 20;
+
+/**
+ * Pull changes from Candengo Vector and merge into local SQLite.
+ * Loops on has_more until all pages are consumed (up to MAX_PAGES).
+ */
+export async function pullFromVector(
+  db: MemDatabase,
+  client: VectorClient,
+  config: Config,
+  limit: number = 50
+): Promise<PullResult> {
+  let cursor = db.getSyncState(PULL_CURSOR_KEY) ?? undefined;
+  let totalReceived = 0;
+  let totalMerged = 0;
+  let totalSkipped = 0;
+
+  for (let page = 0; page < MAX_PAGES; page++) {
+    const response = await client.pullChanges(cursor, limit);
+    const { merged, skipped } = mergeChanges(db, config, response.changes);
+
+    totalReceived += response.changes.length;
+    totalMerged += merged;
+    totalSkipped += skipped;
+
+    // Update cursor after each page so progress is saved even if we crash
+    if (response.cursor) {
+      db.setSyncState(PULL_CURSOR_KEY, response.cursor);
+      cursor = response.cursor;
+    }
+
+    if (!response.has_more || response.changes.length === 0) break;
+  }
+
+  return { received: totalReceived, merged: totalMerged, skipped: totalSkipped };
+}
+
+/**
+ * Merge a batch of changes into local SQLite. Returns merged/skipped counts.
+ * Embedding is intentionally synchronous-per-change to avoid overwhelming the model.
+ */
+function mergeChanges(
+  db: MemDatabase,
+  config: Config,
+  changes: VectorSearchResult[]
+): { merged: number; skipped: number } {
+  let merged = 0;
+  let skipped = 0;
+
+  for (const change of changes) {
+    const parsed = parseSourceId(change.source_id);
+
+    // Skip observations from own device
+    if (parsed && parsed.deviceId === config.device_id) {
+      skipped++;
+      continue;
+    }
+
+    // Check if already imported (by remote_source_id)
+    const existing = db.db
+      .query<{ id: number }, [string]>(
+        "SELECT id FROM observations WHERE remote_source_id = ?"
+      )
+      .get(change.source_id);
+
+    if (existing) {
+      skipped++;
+      continue;
+    }
+
+    // Find or create the project
+    const projectCanonical =
+      (change.metadata?.project_canonical as string) ?? null;
+    if (!projectCanonical) {
+      skipped++;
+      continue;
+    }
+
+    let project = db.getProjectByCanonicalId(projectCanonical);
+    if (!project) {
+      project = db.upsertProject({
+        canonical_id: projectCanonical,
+        name:
+          (change.metadata?.project_name as string) ??
+          projectCanonical.split("/").pop() ??
+          "unknown",
+      });
+    }
+
+    // Insert the observation
+    const obs = db.insertObservation({
+      session_id: (change.metadata?.session_id as string) ?? null,
+      project_id: project.id,
+      type: (change.metadata?.type as string) ?? "discovery",
+      title: (change.metadata?.title as string) ?? change.content.split("\n")[0] ?? "Untitled",
+      narrative: extractNarrative(change.content),
+      facts: change.metadata?.facts
+        ? JSON.stringify(change.metadata.facts)
+        : null,
+      concepts: change.metadata?.concepts
+        ? JSON.stringify(change.metadata.concepts)
+        : null,
+      quality: (change.metadata?.quality as number) ?? 0.5,
+      lifecycle: "active",
+      sensitivity: "shared",
+      user_id: (change.metadata?.user_id as string) ?? "unknown",
+      device_id: (change.metadata?.device_id as string) ?? "unknown",
+      agent: (change.metadata?.agent as string) ?? "unknown",
+    });
+
+    // Mark with remote source ID for dedup
+    db.db
+      .query("UPDATE observations SET remote_source_id = ? WHERE id = ?")
+      .run(change.source_id, obs.id);
+
+    // Embed for local vector search (fire-and-forget — don't block pull loop)
+    if (db.vecAvailable) {
+      embedAndInsert(db, obs).catch(() => {});
+    }
+
+    merged++;
+  }
+
+  return { merged, skipped };
+}
+
+/**
+ * Embed an observation and insert into vec_observations.
+ */
+async function embedAndInsert(
+  db: MemDatabase,
+  obs: { id: number; title: string; narrative: string | null; facts: string | null; concepts: string | null }
+): Promise<void> {
+  const text = composeEmbeddingText(obs);
+  const embedding = await embedText(text);
+  if (embedding) db.vecInsert(obs.id, embedding);
+}
+
+/**
+ * Extract narrative from Vector content (everything after the title line).
+ */
+function extractNarrative(content: string): string | null {
+  const lines = content.split("\n");
+  if (lines.length <= 1) return null;
+  const narrative = lines.slice(1).join("\n").trim();
+  return narrative.length > 0 ? narrative : null;
+}

package/src/sync/push.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "../storage/sqlite.js";
+import type { Config } from "../config.js";
+import { buildVectorDocument } from "./push.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+let projectId: number;
+
+function makeConfig(): Config {
+  return {
+    candengo_url: "https://candengo.com",
+    candengo_api_key: "cvk_test123",
+    site_id: "test-site",
+    namespace: "dev-memory",
+    user_id: "david",
+    device_id: "laptop-abc",
+    user_email: "",
+    teams: [],
+    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
+    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
+    scrubbing: { enabled: true, custom_patterns: [], default_sensitivity: "shared" },
+  };
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "candengo-push-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+  const project = db.upsertProject({
+    canonical_id: "github.com/test/repo",
+    name: "repo",
+  });
+  projectId = project.id;
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("buildVectorDocument", () => {
+  test("produces correct structure", () => {
+    const obs = db.insertObservation({
+      project_id: projectId,
+      type: "bugfix",
+      title: "Fixed auth bug",
+      narrative: "The auth was broken due to a type mismatch",
+      facts: '["fact1", "fact2"]',
+      concepts: '["auth", "debugging"]',
+      quality: 0.8,
+      user_id: "david",
+      device_id: "laptop-abc",
+      agent: "claude-code",
+    });
+
+    const doc = buildVectorDocument(obs, makeConfig(), {
+      canonical_id: "github.com/test/repo",
+      name: "repo",
+    });
+
+    expect(doc.site_id).toBe("test-site");
+    expect(doc.namespace).toBe("dev-memory");
+    expect(doc.source_type).toBe("bugfix");
+    expect(doc.source_id).toBe(`david-laptop-abc-obs-${obs.id}`);
+    expect(doc.content).toContain("Fixed auth bug");
+    expect(doc.content).toContain("The auth was broken");
+    expect(doc.content).toContain("- fact1");
+    expect(doc.content).toContain("- fact2");
+    expect(doc.metadata.project_canonical).toBe("github.com/test/repo");
+    expect(doc.metadata.quality).toBe(0.8);
+    expect(doc.metadata.user_id).toBe("david");
+  });
+
+  test("handles observation with no narrative or facts", () => {
+    const obs = db.insertObservation({
+      project_id: projectId,
+      type: "change",
+      title: "Simple change",
+      quality: 0.3,
+      user_id: "david",
+      device_id: "laptop-abc",
+    });
+
+    const doc = buildVectorDocument(obs, makeConfig(), {
+      canonical_id: "github.com/test/repo",
+      name: "repo",
+    });
+
+    expect(doc.content).toBe("Simple change");
+    expect(doc.metadata.title).toBe("Simple change");
+  });
+
+  test("includes files_modified in metadata", () => {
+    const obs = db.insertObservation({
+      project_id: projectId,
+      type: "change",
+      title: "Edit file",
+      files_modified: '["src/main.ts", "src/utils.ts"]',
+      quality: 0.5,
+      user_id: "david",
+      device_id: "laptop-abc",
+    });
+
+    const doc = buildVectorDocument(obs, makeConfig(), {
+      canonical_id: "github.com/test/repo",
+      name: "repo",
+    });
+
+    expect(doc.metadata.files_modified).toEqual([
+      "src/main.ts",
+      "src/utils.ts",
+    ]);
+  });
+});