engrm 0.1.0 → 0.2.0

package/src/sync/engine.ts

@@ -1,127 +0,0 @@
-/**
- * Sync engine orchestrator.
- *
- * Manages push/pull timers and startup backfill.
- * If sync is not configured (no API key), the engine is a no-op.
- *
- * Timers:
- *   - Push: every config.sync.interval_seconds (default 30s)
- *   - Pull: every 60s
- */
-
-import type { MemDatabase } from "../storage/sqlite.js";
-import type { Config } from "../config.js";
-import { VectorClient } from "./client.js";
-import { pushOutbox } from "./push.js";
-import { pullFromVector } from "./pull.js";
-
-const DEFAULT_PULL_INTERVAL = 60_000; // 60 seconds
-
-export class SyncEngine {
-  private client: VectorClient | null = null;
-  private pushTimer: ReturnType<typeof setInterval> | null = null;
-  private pullTimer: ReturnType<typeof setInterval> | null = null;
-  private _pushing = false;
-  private _pulling = false;
-  private _running = false;
-
-  constructor(
-    private readonly db: MemDatabase,
-    private readonly config: Config
-  ) {
-    if (VectorClient.isConfigured(config)) {
-      try {
-        this.client = new VectorClient(config);
-      } catch {
-        // Configuration invalid — stay in offline mode
-      }
-    }
-  }
-
-  /**
-   * Start the sync engine.
-   * Runs startup backfill, then sets up push/pull timers.
-   */
-  start(): void {
-    if (!this.client || !this.config.sync.enabled) {
-      this._running = false;
-      return;
-    }
-
-    this._running = true;
-
-    // Startup backfill: push any pending outbox items
-    this.pushNow().catch(() => {});
-
-    // Push timer
-    const pushInterval = this.config.sync.interval_seconds * 1000;
-    this.pushTimer = setInterval(() => {
-      this.pushNow().catch(() => {});
-    }, pushInterval);
-
-    // Pull timer
-    this.pullTimer = setInterval(() => {
-      this.pullNow().catch(() => {});
-    }, DEFAULT_PULL_INTERVAL);
-  }
-
-  /**
-   * Stop all timers and clean up.
-   */
-  stop(): void {
-    if (this.pushTimer) {
-      clearInterval(this.pushTimer);
-      this.pushTimer = null;
-    }
-    if (this.pullTimer) {
-      clearInterval(this.pullTimer);
-      this.pullTimer = null;
-    }
-    this._running = false;
-  }
-
-  /**
-   * Force an immediate push of pending outbox items.
-   */
-  async pushNow(): Promise<void> {
-    if (!this.client || this._pushing) return;
-    this._pushing = true;
-    try {
-      await pushOutbox(
-        this.db,
-        this.client,
-        this.config,
-        this.config.sync.batch_size
-      );
-    } finally {
-      this._pushing = false;
-    }
-  }
-
-  /**
-   * Force an immediate pull from Candengo Vector.
-   */
-  async pullNow(): Promise<void> {
-    if (!this.client || this._pulling) return;
-    this._pulling = true;
-    try {
-      await pullFromVector(this.db, this.client, this.config);
-    } finally {
-      this._pulling = false;
-    }
-  }
-
-  /**
-   * Is the sync engine running?
-   */
-  isRunning(): boolean {
-    return this._running;
-  }
-
-  /**
-   * Is the client configured for remote sync?
-   */
-  isConfigured(): boolean {
-    return this.client !== null;
-  }
-}

package/src/sync/pull.test.ts

@@ -1,279 +0,0 @@
-import { describe, expect, test, beforeEach, afterEach } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import { MemDatabase } from "../storage/sqlite.js";
-import type { Config } from "../config.js";
-import { pullFromVector } from "./pull.js";
-import type {
-  VectorClient,
-  VectorChangeFeedResponse,
-  VectorSearchResult,
-} from "./client.js";
-
-let db: MemDatabase;
-let tmpDir: string;
-
-function makeConfig(overrides: Partial<Config> = {}): Config {
-  return {
-    candengo_url: "https://candengo.com",
-    candengo_api_key: "cvk_test123",
-    site_id: "test-site",
-    namespace: "dev-memory",
-    user_id: "david",
-    user_email: "",
-    device_id: "laptop-abc",
-    teams: [],
-    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
-    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
-    scrubbing: {
-      enabled: true,
-      custom_patterns: [],
-      default_sensitivity: "shared",
-    },
-  };
-}
-
-function makeChange(
-  id: number,
-  overrides: Partial<VectorSearchResult> = {}
-): VectorSearchResult {
-  return {
-    source_id:
-      overrides.source_id ?? `other-user-other-device-obs-${id}`,
-    content: overrides.content ?? `Title ${id}\n\nNarrative for ${id}`,
-    score: 1.0,
-    metadata: overrides.metadata ?? {
-      project_canonical: "github.com/test/repo",
-      project_name: "repo",
-      type: "discovery",
-      title: `Title ${id}`,
-      user_id: "other-user",
-      device_id: "other-device",
-      agent: "claude-code",
-      quality: 0.7,
-    },
-  };
-}
-
-function mockClient(
-  pages: VectorChangeFeedResponse[]
-): VectorClient {
-  let callCount = 0;
-  return {
-    pullChanges: async (
-      _cursor?: string,
-      _limit?: number
-    ): Promise<VectorChangeFeedResponse> => {
-      const page = pages[callCount] ?? {
-        changes: [],
-        cursor: "end",
-        has_more: false,
-      };
-      callCount++;
-      return page;
-    },
-  } as unknown as VectorClient;
-}
-
-beforeEach(() => {
-  tmpDir = mkdtempSync(join(tmpdir(), "engrm-pull-test-"));
-  db = new MemDatabase(join(tmpDir, "test.db"));
-});
-
-afterEach(() => {
-  db.close();
-  rmSync(tmpDir, { recursive: true, force: true });
-});
-
-describe("pullFromVector", () => {
-  test("merges single page of changes", async () => {
-    const client = mockClient([
-      {
-        changes: [makeChange(1), makeChange(2)],
-        cursor: "cursor-1",
-        has_more: false,
-      },
-    ]);
-
-    const result = await pullFromVector(db, client, makeConfig());
-    expect(result.received).toBe(2);
-    expect(result.merged).toBe(2);
-    expect(result.skipped).toBe(0);
-  });
-
-  test("loops on has_more", async () => {
-    const client = mockClient([
-      {
-        changes: [makeChange(1), makeChange(2)],
-        cursor: "cursor-1",
-        has_more: true,
-      },
-      {
-        changes: [makeChange(3)],
-        cursor: "cursor-2",
-        has_more: false,
-      },
-    ]);
-
-    const result = await pullFromVector(db, client, makeConfig());
-    expect(result.received).toBe(3);
-    expect(result.merged).toBe(3);
-  });
-
-  test("loops through multiple pages", async () => {
-    const client = mockClient([
-      {
-        changes: [makeChange(1)],
-        cursor: "c1",
-        has_more: true,
-      },
-      {
-        changes: [makeChange(2)],
-        cursor: "c2",
-        has_more: true,
-      },
-      {
-        changes: [makeChange(3)],
-        cursor: "c3",
-        has_more: false,
-      },
-    ]);
-
-    const result = await pullFromVector(db, client, makeConfig());
-    expect(result.received).toBe(3);
-    expect(result.merged).toBe(3);
-  });
-
-  test("stops at MAX_PAGES safety limit", async () => {
-    // Create 25 pages (MAX_PAGES = 20)
-    const pages: VectorChangeFeedResponse[] = Array.from(
-      { length: 25 },
-      (_, i) => ({
-        changes: [makeChange(i + 1)],
-        cursor: `cursor-${i + 1}`,
-        has_more: true,
-      })
-    );
-
-    const client = mockClient(pages);
-    const result = await pullFromVector(db, client, makeConfig());
-    // Should stop at 20 pages
-    expect(result.received).toBe(20);
-    expect(result.merged).toBe(20);
-  });
-
-  test("skips observations from own device", async () => {
-    const ownChange: VectorSearchResult = {
-      source_id: "david-laptop-abc-obs-1",
-      content: "Title\n\nNarrative",
-      score: 1.0,
-      metadata: {
-        project_canonical: "github.com/test/repo",
-        type: "discovery",
-        title: "Title",
-        user_id: "david",
-        device_id: "laptop-abc",
-        agent: "claude-code",
-        quality: 0.7,
-      },
-    };
-
-    const client = mockClient([
-      {
-        changes: [ownChange, makeChange(2)],
-        cursor: "c1",
-        has_more: false,
-      },
-    ]);
-
-    const result = await pullFromVector(db, client, makeConfig());
-    expect(result.received).toBe(2);
-    expect(result.merged).toBe(1);
-    expect(result.skipped).toBe(1);
-  });
-
-  test("skips duplicate remote_source_id", async () => {
-    const client = mockClient([
-      {
-        changes: [makeChange(1)],
-        cursor: "c1",
-        has_more: false,
-      },
-    ]);
-
-    // Pull once
-    await pullFromVector(db, client, makeConfig());
-
-    // Pull same change again
-    const client2 = mockClient([
-      {
-        changes: [makeChange(1)],
-        cursor: "c2",
-        has_more: false,
-      },
-    ]);
-
-    const result = await pullFromVector(db, client2, makeConfig());
-    expect(result.merged).toBe(0);
-    expect(result.skipped).toBe(1);
-  });
-
-  test("saves cursor after each page", async () => {
-    const client = mockClient([
-      {
-        changes: [makeChange(1)],
-        cursor: "cursor-page1",
-        has_more: true,
-      },
-      {
-        changes: [makeChange(2)],
-        cursor: "cursor-page2",
-        has_more: false,
-      },
-    ]);
-
-    await pullFromVector(db, client, makeConfig());
-    expect(db.getSyncState("pull_cursor")).toBe("cursor-page2");
-  });
-
-  test("handles empty response", async () => {
-    const client = mockClient([
-      {
-        changes: [],
-        cursor: "",
-        has_more: false,
-      },
-    ]);
-
-    const result = await pullFromVector(db, client, makeConfig());
-    expect(result.received).toBe(0);
-    expect(result.merged).toBe(0);
-  });
-
-  test("skips changes without project_canonical", async () => {
-    const noProject: VectorSearchResult = {
-      source_id: "other-other-obs-1",
-      content: "Title\n\nNarrative",
-      score: 1.0,
-      metadata: {
-        type: "discovery",
-        title: "Title",
-        user_id: "other",
-        device_id: "other",
-      },
-    };
-
-    const client = mockClient([
-      {
-        changes: [noProject],
-        cursor: "c1",
-        has_more: false,
-      },
-    ]);
-
-    const result = await pullFromVector(db, client, makeConfig());
-    expect(result.skipped).toBe(1);
-    expect(result.merged).toBe(0);
-  });
-});

package/src/sync/pull.ts

@@ -1,170 +0,0 @@
-/**
- * Pull engine: fetch observations from Candengo Vector change feed
- * and merge into local SQLite.
- *
- * Uses server-side cursors to track position in the change feed.
- * Skips observations from the current device (already local).
- */
-
-import type { MemDatabase } from "../storage/sqlite.js";
-import type { Config } from "../config.js";
-import { VectorClient, type VectorSearchResult } from "./client.js";
-import { parseSourceId } from "./auth.js";
-import { composeEmbeddingText, embedText } from "../embeddings/embedder.js";
-
-const PULL_CURSOR_KEY = "pull_cursor";
-
-export interface PullResult {
-  received: number;
-  merged: number;
-  skipped: number;
-}
-
-/** Safety limit to prevent infinite loops if server keeps returning has_more. */
-const MAX_PAGES = 20;
-
-/**
- * Pull changes from Candengo Vector and merge into local SQLite.
- * Loops on has_more until all pages are consumed (up to MAX_PAGES).
- */
-export async function pullFromVector(
-  db: MemDatabase,
-  client: VectorClient,
-  config: Config,
-  limit: number = 50
-): Promise<PullResult> {
-  let cursor = db.getSyncState(PULL_CURSOR_KEY) ?? undefined;
-  let totalReceived = 0;
-  let totalMerged = 0;
-  let totalSkipped = 0;
-
-  for (let page = 0; page < MAX_PAGES; page++) {
-    const response = await client.pullChanges(cursor, limit);
-    const { merged, skipped } = mergeChanges(db, config, response.changes);
-
-    totalReceived += response.changes.length;
-    totalMerged += merged;
-    totalSkipped += skipped;
-
-    // Update cursor after each page so progress is saved even if we crash
-    if (response.cursor) {
-      db.setSyncState(PULL_CURSOR_KEY, response.cursor);
-      cursor = response.cursor;
-    }
-
-    if (!response.has_more || response.changes.length === 0) break;
-  }
-
-  return { received: totalReceived, merged: totalMerged, skipped: totalSkipped };
-}
-
-/**
- * Merge a batch of changes into local SQLite. Returns merged/skipped counts.
- * Embedding is intentionally synchronous-per-change to avoid overwhelming the model.
- */
-function mergeChanges(
-  db: MemDatabase,
-  config: Config,
-  changes: VectorSearchResult[]
-): { merged: number; skipped: number } {
-  let merged = 0;
-  let skipped = 0;
-
-  for (const change of changes) {
-    const parsed = parseSourceId(change.source_id);
-
-    // Skip observations from own device
-    if (parsed && parsed.deviceId === config.device_id) {
-      skipped++;
-      continue;
-    }
-
-    // Check if already imported (by remote_source_id)
-    const existing = db.db
-      .query<{ id: number }, [string]>(
-        "SELECT id FROM observations WHERE remote_source_id = ?"
-      )
-      .get(change.source_id);
-
-    if (existing) {
-      skipped++;
-      continue;
-    }
-
-    // Find or create the project
-    const projectCanonical =
-      (change.metadata?.project_canonical as string) ?? null;
-    if (!projectCanonical) {
-      skipped++;
-      continue;
-    }
-
-    let project = db.getProjectByCanonicalId(projectCanonical);
-    if (!project) {
-      project = db.upsertProject({
-        canonical_id: projectCanonical,
-        name:
-          (change.metadata?.project_name as string) ??
-          projectCanonical.split("/").pop() ??
-          "unknown",
-      });
-    }
-
-    // Insert the observation
-    const obs = db.insertObservation({
-      session_id: (change.metadata?.session_id as string) ?? null,
-      project_id: project.id,
-      type: (change.metadata?.type as string) ?? "discovery",
-      title: (change.metadata?.title as string) ?? change.content.split("\n")[0] ?? "Untitled",
-      narrative: extractNarrative(change.content),
-      facts: change.metadata?.facts
-        ? JSON.stringify(change.metadata.facts)
-        : null,
-      concepts: change.metadata?.concepts
-        ? JSON.stringify(change.metadata.concepts)
-        : null,
-      quality: (change.metadata?.quality as number) ?? 0.5,
-      lifecycle: "active",
-      sensitivity: "shared",
-      user_id: (change.metadata?.user_id as string) ?? "unknown",
-      device_id: (change.metadata?.device_id as string) ?? "unknown",
-      agent: (change.metadata?.agent as string) ?? "unknown",
-    });
-
-    // Mark with remote source ID for dedup
-    db.db
-      .query("UPDATE observations SET remote_source_id = ? WHERE id = ?")
-      .run(change.source_id, obs.id);
-
-    // Embed for local vector search (fire-and-forget — don't block pull loop)
-    if (db.vecAvailable) {
-      embedAndInsert(db, obs).catch(() => {});
-    }
-
-    merged++;
-  }
-
-  return { merged, skipped };
-}
-
-/**
- * Embed an observation and insert into vec_observations.
- */
-async function embedAndInsert(
-  db: MemDatabase,
-  obs: { id: number; title: string; narrative: string | null; facts: string | null; concepts: string | null }
-): Promise<void> {
-  const text = composeEmbeddingText(obs);
-  const embedding = await embedText(text);
-  if (embedding) db.vecInsert(obs.id, embedding);
-}
-
-/**
- * Extract narrative from Vector content (everything after the title line).
- */
-function extractNarrative(content: string): string | null {
-  const lines = content.split("\n");
-  if (lines.length <= 1) return null;
-  const narrative = lines.slice(1).join("\n").trim();
-  return narrative.length > 0 ? narrative : null;
-}

package/src/sync/push.test.ts

@@ -1,117 +0,0 @@
-import { describe, expect, test, beforeEach, afterEach } from "bun:test";
-import { mkdtempSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import { MemDatabase } from "../storage/sqlite.js";
-import type { Config } from "../config.js";
-import { buildVectorDocument } from "./push.js";
-
-let db: MemDatabase;
-let tmpDir: string;
-let projectId: number;
-
-function makeConfig(): Config {
-  return {
-    candengo_url: "https://candengo.com",
-    candengo_api_key: "cvk_test123",
-    site_id: "test-site",
-    namespace: "dev-memory",
-    user_id: "david",
-    device_id: "laptop-abc",
-    user_email: "",
-    teams: [],
-    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
-    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
-    scrubbing: { enabled: true, custom_patterns: [], default_sensitivity: "shared" },
-  };
-}
-
-beforeEach(() => {
-  tmpDir = mkdtempSync(join(tmpdir(), "candengo-push-test-"));
-  db = new MemDatabase(join(tmpDir, "test.db"));
-  const project = db.upsertProject({
-    canonical_id: "github.com/test/repo",
-    name: "repo",
-  });
-  projectId = project.id;
-});
-
-afterEach(() => {
-  db.close();
-  rmSync(tmpDir, { recursive: true, force: true });
-});
-
-describe("buildVectorDocument", () => {
-  test("produces correct structure", () => {
-    const obs = db.insertObservation({
-      project_id: projectId,
-      type: "bugfix",
-      title: "Fixed auth bug",
-      narrative: "The auth was broken due to a type mismatch",
-      facts: '["fact1", "fact2"]',
-      concepts: '["auth", "debugging"]',
-      quality: 0.8,
-      user_id: "david",
-      device_id: "laptop-abc",
-      agent: "claude-code",
-    });
-
-    const doc = buildVectorDocument(obs, makeConfig(), {
-      canonical_id: "github.com/test/repo",
-      name: "repo",
-    });
-
-    expect(doc.site_id).toBe("test-site");
-    expect(doc.namespace).toBe("dev-memory");
-    expect(doc.source_type).toBe("bugfix");
-    expect(doc.source_id).toBe(`david-laptop-abc-obs-${obs.id}`);
-    expect(doc.content).toContain("Fixed auth bug");
-    expect(doc.content).toContain("The auth was broken");
-    expect(doc.content).toContain("- fact1");
-    expect(doc.content).toContain("- fact2");
-    expect(doc.metadata.project_canonical).toBe("github.com/test/repo");
-    expect(doc.metadata.quality).toBe(0.8);
-    expect(doc.metadata.user_id).toBe("david");
-  });
-
-  test("handles observation with no narrative or facts", () => {
-    const obs = db.insertObservation({
-      project_id: projectId,
-      type: "change",
-      title: "Simple change",
-      quality: 0.3,
-      user_id: "david",
-      device_id: "laptop-abc",
-    });
-
-    const doc = buildVectorDocument(obs, makeConfig(), {
-      canonical_id: "github.com/test/repo",
-      name: "repo",
-    });
-
-    expect(doc.content).toBe("Simple change");
-    expect(doc.metadata.title).toBe("Simple change");
-  });
-
-  test("includes files_modified in metadata", () => {
-    const obs = db.insertObservation({
-      project_id: projectId,
-      type: "change",
-      title: "Edit file",
-      files_modified: '["src/main.ts", "src/utils.ts"]',
-      quality: 0.5,
-      user_id: "david",
-      device_id: "laptop-abc",
-    });
-
-    const doc = buildVectorDocument(obs, makeConfig(), {
-      canonical_id: "github.com/test/repo",
-      name: "repo",
-    });
-
-    expect(doc.metadata.files_modified).toEqual([
-      "src/main.ts",
-      "src/utils.ts",
-    ]);
-  });
-});