engrm 0.1.0

package/src/storage/vec.test.ts

@@ -0,0 +1,198 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "./sqlite.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "engrm-vec-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+// All tests conditional on sqlite-vec being available
+describe("sqlite-vec integration", () => {
+  test("vecAvailable is true when extension loads", () => {
+    expect(db.vecAvailable).toBe(true);
+  });
+
+  test("vec_observations table exists", () => {
+    if (!db.vecAvailable) return;
+    const tables = db.db
+      .query<{ name: string }, []>(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='vec_observations'"
+      )
+      .all();
+    expect(tables.length).toBe(1);
+  });
+
+  test("vecInsert and searchVec round-trip", () => {
+    if (!db.vecAvailable) return;
+
+    // Create a project + observation
+    const project = db.upsertProject({
+      canonical_id: "test/project",
+      name: "test",
+    });
+
+    const obs = db.insertObservation({
+      project_id: project.id,
+      type: "bugfix",
+      title: "Fix auth bug",
+      quality: 0.8,
+      user_id: "david",
+      device_id: "laptop",
+    });
+
+    // Insert embedding
+    const vec = new Float32Array(384);
+    vec[0] = 1.0;
+    vec[1] = 0.5;
+    db.vecInsert(obs.id, vec);
+
+    // Search with same vector — should find it
+    const results = db.searchVec(vec, project.id, ["active"], 5);
+    expect(results.length).toBe(1);
+    expect(results[0]!.observation_id).toBe(obs.id);
+    expect(results[0]!.distance).toBe(0); // exact match
+  });
+
+  test("searchVec filters by project", () => {
+    if (!db.vecAvailable) return;
+
+    const proj1 = db.upsertProject({ canonical_id: "p1", name: "p1" });
+    const proj2 = db.upsertProject({ canonical_id: "p2", name: "p2" });
+
+    const obs1 = db.insertObservation({
+      project_id: proj1.id,
+      type: "bugfix",
+      title: "P1 fix",
+      quality: 0.8,
+      user_id: "david",
+      device_id: "laptop",
+    });
+    const obs2 = db.insertObservation({
+      project_id: proj2.id,
+      type: "bugfix",
+      title: "P2 fix",
+      quality: 0.8,
+      user_id: "david",
+      device_id: "laptop",
+    });
+
+    const vec = new Float32Array(384);
+    vec[0] = 1.0;
+    db.vecInsert(obs1.id, vec);
+    db.vecInsert(obs2.id, vec);
+
+    // Search scoped to project 1
+    const results = db.searchVec(vec, proj1.id, ["active"], 5);
+    expect(results.length).toBe(1);
+    expect(results[0]!.observation_id).toBe(obs1.id);
+  });
+
+  test("searchVec excludes superseded observations", () => {
+    if (!db.vecAvailable) return;
+
+    const project = db.upsertProject({ canonical_id: "test/proj", name: "test" });
+
+    const obs1 = db.insertObservation({
+      project_id: project.id,
+      type: "decision",
+      title: "Old decision",
+      quality: 0.7,
+      user_id: "david",
+      device_id: "laptop",
+    });
+    const obs2 = db.insertObservation({
+      project_id: project.id,
+      type: "decision",
+      title: "New decision",
+      quality: 0.9,
+      user_id: "david",
+      device_id: "laptop",
+    });
+
+    const vec = new Float32Array(384);
+    vec[0] = 1.0;
+    db.vecInsert(obs1.id, vec);
+    db.vecInsert(obs2.id, vec);
+
+    // Supersede obs1 with obs2
+    db.supersedeObservation(obs1.id, obs2.id);
+
+    // Search should only return obs2
+    const results = db.searchVec(vec, null, ["active"], 5);
+    expect(results.length).toBe(1);
+    expect(results[0]!.observation_id).toBe(obs2.id);
+  });
+
+  test("vecDelete removes embedding", () => {
+    if (!db.vecAvailable) return;
+
+    const project = db.upsertProject({ canonical_id: "t/p", name: "t" });
+    const obs = db.insertObservation({
+      project_id: project.id,
+      type: "bugfix",
+      title: "Test",
+      quality: 0.8,
+      user_id: "u",
+      device_id: "d",
+    });
+
+    const vec = new Float32Array(384);
+    vec[0] = 1.0;
+    db.vecInsert(obs.id, vec);
+    expect(db.searchVec(vec, null, ["active"], 5).length).toBe(1);
+
+    db.vecDelete(obs.id);
+    expect(db.searchVec(vec, null, ["active"], 5).length).toBe(0);
+  });
+
+  test("getUnembeddedObservations returns unembedded only", () => {
+    if (!db.vecAvailable) return;
+
+    const project = db.upsertProject({ canonical_id: "t/p", name: "t" });
+
+    const obs1 = db.insertObservation({
+      project_id: project.id,
+      type: "bugfix",
+      title: "Embedded",
+      quality: 0.8,
+      user_id: "u",
+      device_id: "d",
+    });
+    db.insertObservation({
+      project_id: project.id,
+      type: "bugfix",
+      title: "Not embedded",
+      quality: 0.8,
+      user_id: "u",
+      device_id: "d",
+    });
+
+    // Embed only obs1
+    const vec = new Float32Array(384);
+    db.vecInsert(obs1.id, vec);
+
+    const unembedded = db.getUnembeddedObservations(10);
+    expect(unembedded.length).toBe(1);
+    expect(unembedded[0]!.title).toBe("Not embedded");
+
+    expect(db.getUnembeddedCount()).toBe(1);
+  });
+
+  test("searchVec returns empty when vecAvailable but no embeddings", () => {
+    if (!db.vecAvailable) return;
+    const vec = new Float32Array(384);
+    const results = db.searchVec(vec, null, ["active"], 5);
+    expect(results.length).toBe(0);
+  });
+});

package/src/sync/auth.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, test, afterEach } from "bun:test";
+import { getApiKey, getBaseUrl, buildSourceId, parseSourceId } from "./auth.js";
+import type { Config } from "../config.js";
+
+function makeConfig(overrides: Partial<Config> = {}): Config {
+  return {
+    candengo_url: overrides.candengo_url ?? "https://candengo.com",
+    candengo_api_key: overrides.candengo_api_key ?? "cvk_test123",
+    site_id: overrides.site_id ?? "test-site",
+    namespace: overrides.namespace ?? "dev-memory",
+    user_id: overrides.user_id ?? "david",
+    device_id: overrides.device_id ?? "laptop-abc",
+    user_email: "",
+    teams: [],
+    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
+    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
+    scrubbing: { enabled: true, custom_patterns: [], default_sensitivity: "shared" },
+  };
+}
+
+afterEach(() => {
+  delete process.env.ENGRM_TOKEN;
+});
+
+describe("getApiKey", () => {
+  test("returns env var when set", () => {
+    process.env.ENGRM_TOKEN = "cvk_from_env";
+    expect(getApiKey(makeConfig())).toBe("cvk_from_env");
+  });
+
+  test("falls back to config when env var not set", () => {
+    expect(getApiKey(makeConfig())).toBe("cvk_test123");
+  });
+
+  test("returns null when both empty", () => {
+    expect(getApiKey(makeConfig({ candengo_api_key: "" }))).toBeNull();
+  });
+
+  test("ignores env var without cvk_ prefix", () => {
+    process.env.ENGRM_TOKEN = "not_a_valid_key";
+    expect(getApiKey(makeConfig())).toBe("cvk_test123");
+  });
+});
+
+describe("getBaseUrl", () => {
+  test("returns config URL", () => {
+    expect(getBaseUrl(makeConfig())).toBe("https://candengo.com");
+  });
+
+  test("returns null for empty URL", () => {
+    expect(getBaseUrl(makeConfig({ candengo_url: "" }))).toBeNull();
+  });
+});
+
+describe("buildSourceId", () => {
+  test("produces correct format", () => {
+    const config = makeConfig({ user_id: "david", device_id: "laptop-abc" });
+    expect(buildSourceId(config, 42)).toBe("david-laptop-abc-obs-42");
+  });
+});
+
+describe("parseSourceId", () => {
+  test("parses valid source ID", () => {
+    const result = parseSourceId("david-laptop-abc-obs-42");
+    expect(result).toEqual({
+      userId: "david",
+      deviceId: "laptop-abc",
+      localId: 42,
+    });
+  });
+
+  test("returns null for invalid format", () => {
+    expect(parseSourceId("invalid")).toBeNull();
+    expect(parseSourceId("")).toBeNull();
+  });
+});

package/src/sync/auth.ts

@@ -0,0 +1,68 @@
+/**
+ * Sync authentication utilities.
+ *
+ * Resolves API credentials from environment variables or config.
+ * Environment variable takes precedence for CI/CD use.
+ */
+
+import type { Config } from "../config.js";
+
+/**
+ * Get API key for Candengo Vector.
+ * Priority: ENGRM_TOKEN env var → config.candengo_api_key
+ */
+export function getApiKey(config: Config): string | null {
+  const envKey = process.env.ENGRM_TOKEN;
+  if (envKey && envKey.startsWith("cvk_")) return envKey;
+  if (config.candengo_api_key && config.candengo_api_key.length > 0) {
+    return config.candengo_api_key;
+  }
+  return null;
+}
+
+/**
+ * Get base URL for Candengo Vector API.
+ */
+export function getBaseUrl(config: Config): string | null {
+  if (config.candengo_url && config.candengo_url.length > 0) {
+    return config.candengo_url;
+  }
+  return null;
+}
+
+/**
+ * Build a globally unique source ID for a record.
+ * Format: {user_id}-{device_id}-{type}-{local_id}
+ * Default type is "obs" for backwards compatibility.
+ */
+export function buildSourceId(config: Config, localId: number, type: string = "obs"): string {
+  return `${config.user_id}-${config.device_id}-${type}-${localId}`;
+}
+
+/**
+ * Parse a source ID back into its components.
+ * Returns null if the format doesn't match.
+ */
+export function parseSourceId(
+  sourceId: string
+): { userId: string; deviceId: string; localId: number } | null {
+  // Format: {user_id}-{device_id}-obs-{local_id}
+  // Split on "-obs-" which is our guaranteed delimiter
+  const obsIndex = sourceId.lastIndexOf("-obs-");
+  if (obsIndex === -1) return null;
+
+  const prefix = sourceId.slice(0, obsIndex);
+  const localIdStr = sourceId.slice(obsIndex + 5); // skip "-obs-"
+  const localId = parseInt(localIdStr, 10);
+  if (isNaN(localId)) return null;
+
+  // Split prefix on first "-" to get userId and deviceId
+  const firstDash = prefix.indexOf("-");
+  if (firstDash === -1) return null;
+
+  return {
+    userId: prefix.slice(0, firstDash),
+    deviceId: prefix.slice(firstDash + 1),
+    localId,
+  };
+}

package/src/sync/client.ts

@@ -0,0 +1,183 @@
+/**
+ * Candengo Vector REST client.
+ *
+ * Wraps the Vector API for ingest, search, and delete operations.
+ * Uses Bun's built-in fetch — no external HTTP dependencies.
+ */
+
+import type { Config } from "../config.js";
+import { getApiKey, getBaseUrl } from "./auth.js";
+
+// --- Types ---
+
+export interface VectorDocument {
+  site_id: string;
+  namespace: string;
+  source_type: string;
+  source_id: string;
+  content: string;
+  metadata: Record<string, unknown>;
+}
+
+export interface VectorSearchResult {
+  source_id: string;
+  content: string;
+  score: number;
+  metadata: Record<string, unknown>;
+}
+
+export interface VectorSearchResponse {
+  results: VectorSearchResult[];
+  total?: number;
+}
+
+export interface VectorChangeFeedResponse {
+  changes: VectorSearchResult[];
+  cursor: string;
+  has_more: boolean;
+}
+
+// --- Client ---
+
+export class VectorClient {
+  private readonly baseUrl: string;
+  private readonly apiKey: string;
+  readonly siteId: string;
+  readonly namespace: string;
+
+  constructor(config: Config) {
+    const baseUrl = getBaseUrl(config);
+    const apiKey = getApiKey(config);
+
+    if (!baseUrl || !apiKey) {
+      throw new Error(
+        "VectorClient requires candengo_url and candengo_api_key"
+      );
+    }
+
+    this.baseUrl = baseUrl.replace(/\/$/, ""); // strip trailing slash
+    this.apiKey = apiKey;
+    this.siteId = config.site_id;
+    this.namespace = config.namespace;
+  }
+
+  /**
+   * Check if the client has valid configuration.
+   */
+  static isConfigured(config: Config): boolean {
+    return getApiKey(config) !== null && getBaseUrl(config) !== null;
+  }
+
+  /**
+   * Ingest a single document.
+   */
+  async ingest(doc: VectorDocument): Promise<void> {
+    await this.request("POST", "/v1/ingest", doc);
+  }
+
+  /**
+   * Batch ingest multiple documents.
+   */
+  async batchIngest(docs: VectorDocument[]): Promise<void> {
+    if (docs.length === 0) return;
+    await this.request("POST", "/v1/ingest/batch", { documents: docs });
+  }
+
+  /**
+   * Semantic search with optional metadata filters.
+   */
+  async search(
+    query: string,
+    metadataFilter?: Record<string, string>,
+    limit: number = 10
+  ): Promise<VectorSearchResponse> {
+    const body: Record<string, unknown> = { query: query, limit: limit };
+    if (metadataFilter) {
+      body.metadata_filter = metadataFilter;
+    }
+    return this.request("POST", "/v1/search", body);
+  }
+
+  /**
+   * Delete documents by source IDs.
+   */
+  async deleteBySourceIds(sourceIds: string[]): Promise<void> {
+    if (sourceIds.length === 0) return;
+    await this.request("POST", "/v1/documents/delete", {
+      source_ids: sourceIds,
+    });
+  }
+
+  /**
+   * Pull changes from the sync change feed.
+   */
+  async pullChanges(
+    cursor?: string,
+    limit: number = 50
+  ): Promise<VectorChangeFeedResponse> {
+    const params = new URLSearchParams();
+    if (cursor) params.set("cursor", cursor);
+    params.set("namespace", this.namespace);
+    params.set("limit", String(limit));
+    return this.request("GET", `/v1/sync/changes?${params.toString()}`);
+  }
+
+  /**
+   * Health check.
+   */
+  async health(): Promise<boolean> {
+    try {
+      await this.request("GET", "/health");
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  // --- Internal ---
+
+  private async request<T>(
+    method: string,
+    path: string,
+    body?: unknown
+  ): Promise<T> {
+    const url = `${this.baseUrl}${path}`;
+    const headers: Record<string, string> = {
+      Authorization: `Bearer ${this.apiKey}`,
+      "Content-Type": "application/json",
+    };
+
+    const init: RequestInit = { method, headers };
+    if (body && method !== "GET") {
+      init.body = JSON.stringify(body);
+    }
+
+    const response = await fetch(url, init);
+
+    if (!response.ok) {
+      const text = await response.text().catch(() => "");
+      throw new VectorApiError(response.status, text, path);
+    }
+
+    // Some endpoints (DELETE, ingest) may return 204 with no body
+    if (
+      response.status === 204 ||
+      response.headers.get("content-length") === "0"
+    ) {
+      return undefined as T;
+    }
+
+    return response.json() as Promise<T>;
+  }
+}
+
+export class VectorApiError extends Error {
+  constructor(
+    readonly status: number,
+    readonly body: string,
+    readonly path: string
+  ) {
+    super(`Vector API error ${status} on ${path}: ${body}`);
+    this.name = "VectorApiError";
+  }
+}

package/src/sync/engine.test.ts

@@ -0,0 +1,94 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "../storage/sqlite.js";
+import type { Config } from "../config.js";
+import { SyncEngine } from "./engine.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+
+function makeConfig(overrides: Partial<Config> = {}): Config {
+  return {
+    candengo_url: overrides.candengo_url ?? "",
+    candengo_api_key: overrides.candengo_api_key ?? "",
+    site_id: "test-site",
+    namespace: "dev-memory",
+    user_id: "david",
+    device_id: "laptop-abc",
+    user_email: "",
+    teams: [],
+    sync: { enabled: true, interval_seconds: 30, batch_size: 50 },
+    search: { default_limit: 10, local_boost: 1.2, scope: "all" },
+    scrubbing: { enabled: true, custom_patterns: [], default_sensitivity: "shared" },
+  };
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "candengo-engine-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("SyncEngine", () => {
+  test("is not configured when no API key", () => {
+    const engine = new SyncEngine(db, makeConfig());
+    expect(engine.isConfigured()).toBe(false);
+  });
+
+  test("is configured when API key and URL provided", () => {
+    const engine = new SyncEngine(
+      db,
+      makeConfig({
+        candengo_url: "https://candengo.com",
+        candengo_api_key: "cvk_test123",
+      })
+    );
+    expect(engine.isConfigured()).toBe(true);
+  });
+
+  test("start is no-op when not configured", () => {
+    const engine = new SyncEngine(db, makeConfig());
+    engine.start();
+    expect(engine.isRunning()).toBe(false);
+    engine.stop(); // should not throw
+  });
+
+  test("start and stop work cleanly when configured", () => {
+    const engine = new SyncEngine(
+      db,
+      makeConfig({
+        candengo_url: "https://candengo.com",
+        candengo_api_key: "cvk_test123",
+      })
+    );
+    engine.start();
+    expect(engine.isRunning()).toBe(true);
+    engine.stop();
+    expect(engine.isRunning()).toBe(false);
+  });
+
+  test("stop is idempotent", () => {
+    const engine = new SyncEngine(db, makeConfig());
+    engine.stop();
+    engine.stop();
+    // Should not throw
+  });
+
+  test("start with sync disabled is no-op", () => {
+    const config = makeConfig({
+      candengo_url: "https://candengo.com",
+      candengo_api_key: "cvk_test123",
+    });
+    config.sync.enabled = false;
+    const engine = new SyncEngine(db, config);
+    engine.start();
+    expect(engine.isRunning()).toBe(false);
+    engine.stop();
+  });
+});