@rce-mcp/data-plane 0.1.0

package/src/sqlite-runtime.ts

@@ -0,0 +1,137 @@
+import { createRequire } from "node:module";
+
+export type SqliteBindValue = string | number | bigint | Uint8Array | null;
+
+export interface SqliteRunResult {
+  changes?: number;
+}
+
+export interface SqliteStatement {
+  run(...params: SqliteBindValue[]): SqliteRunResult;
+  get(...params: SqliteBindValue[]): unknown;
+  all(...params: SqliteBindValue[]): unknown[];
+}
+
+export interface SqliteDatabase {
+  exec(sql: string): void;
+  prepare(sql: string): SqliteStatement;
+  close(): void;
+}
+
+type SqliteDriverName = "node:sqlite" | "bun:sqlite";
+
+interface SqliteRuntimeDriver {
+  name: SqliteDriverName;
+  open(dbPath: string): SqliteDatabase;
+}
+
+const require = createRequire(import.meta.url);
+
+function loadNodeSqliteDriver(): SqliteRuntimeDriver {
+  const mod = require("node:sqlite") as {
+    DatabaseSync: new (path: string) => {
+      exec(sql: string): void;
+      prepare(sql: string): {
+        run(...params: SqliteBindValue[]): SqliteRunResult;
+        get(...params: SqliteBindValue[]): unknown;
+        all(...params: SqliteBindValue[]): unknown[];
+      };
+      close(): void;
+    };
+  };
+
+  return {
+    name: "node:sqlite",
+    open(dbPath: string): SqliteDatabase {
+      const db = new mod.DatabaseSync(dbPath);
+      return {
+        exec(sql) {
+          db.exec(sql);
+        },
+        prepare(sql) {
+          return db.prepare(sql);
+        },
+        close() {
+          db.close();
+        }
+      };
+    }
+  };
+}
+
+function loadBunSqliteDriver(): SqliteRuntimeDriver {
+  const mod = require("bun:sqlite") as {
+    Database?: new (path: string) => {
+      exec(sql: string): void;
+      query(sql: string): {
+        run(...params: SqliteBindValue[]): SqliteRunResult;
+        get(...params: SqliteBindValue[]): unknown;
+        all(...params: SqliteBindValue[]): unknown[];
+      };
+      close(): void;
+    };
+    default?: {
+      Database?: new (path: string) => {
+        exec(sql: string): void;
+        query(sql: string): {
+          run(...params: SqliteBindValue[]): SqliteRunResult;
+          get(...params: SqliteBindValue[]): unknown;
+          all(...params: SqliteBindValue[]): unknown[];
+        };
+        close(): void;
+      };
+    };
+  };
+
+  const BunDatabase = mod.Database ?? mod.default?.Database;
+  if (!BunDatabase) {
+    throw new Error("bun:sqlite Database export is unavailable");
+  }
+
+  return {
+    name: "bun:sqlite",
+    open(dbPath: string): SqliteDatabase {
+      const db = new BunDatabase(dbPath);
+      return {
+        exec(sql) {
+          db.exec(sql);
+        },
+        prepare(sql) {
+          const statement = db.query(sql);
+          return {
+            run(...params) {
+              return statement.run(...params);
+            },
+            get(...params) {
+              return statement.get(...params);
+            },
+            all(...params) {
+              return statement.all(...params);
+            }
+          };
+        },
+        close() {
+          db.close();
+        }
+      };
+    }
+  };
+}
+
+function loadSqliteRuntimeDriver(): SqliteRuntimeDriver {
+  const isBun = typeof process.versions.bun === "string" && process.versions.bun.length > 0;
+  if (isBun) {
+    return loadBunSqliteDriver();
+  }
+  return loadNodeSqliteDriver();
+}
+
+const sqliteRuntimeDriver = loadSqliteRuntimeDriver();
+
+export function sqliteRuntimeDriverName(): SqliteDriverName {
+  return sqliteRuntimeDriver.name;
+}
+
+export function openSqliteDatabase(dbPath: string): SqliteDatabase {
+  return sqliteRuntimeDriver.open(dbPath);
+}

package/test/ioredis-mock.d.ts

@@ -0,0 +1 @@
+declare module "ioredis-mock";

package/test/queue.integration.test.ts

@@ -0,0 +1,129 @@
+import { afterEach, describe, expect, it } from "vitest";
+import Redis from "ioredis-mock";
+import { RedisIndexJobQueue } from "../src/index.js";
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+describe("redis queue integration", () => {
+  const clients: Array<{ quit(): Promise<string> }> = [];
+
+  afterEach(async () => {
+    while (clients.length > 0) {
+      const client = clients.pop();
+      if (client) {
+        await client.quit();
+      }
+    }
+  });
+
+  it("retries jobs and then dead-letters after max attempts", async () => {
+    const redis = new Redis();
+    clients.push(redis);
+
+    const queue = new RedisIndexJobQueue(redis as unknown as any, {
+      keyPrefix: `test:index_jobs:${Date.now()}`,
+      maxAttempts: 2
+    });
+
+    await queue.enqueue({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-1",
+      manifest_key: "tenant-a/ws-a/idx-1/manifest.json"
+    });
+
+    const firstClaim = await queue.claimNext(0);
+    expect(firstClaim).toBeDefined();
+    await queue.retryOrDeadLetter(firstClaim!, "upstream timeout");
+
+    expect(await queue.pendingCount()).toBe(1);
+    expect(await queue.deadLetterCount()).toBe(0);
+
+    const secondClaim = await queue.claimNext(0);
+    expect(secondClaim?.payload.attempts).toBe(1);
+    await queue.retryOrDeadLetter(secondClaim!, "upstream timeout again");
+
+    expect(await queue.pendingCount()).toBe(0);
+    expect(await queue.deadLetterCount()).toBe(1);
+
+    const deadLetters = await queue.listDeadLetters();
+    expect(deadLetters[0]?.attempts).toBe(2);
+    expect(deadLetters[0]?.last_error).toContain("upstream timeout again");
+  });
+
+  it("reclaims orphaned processing jobs after lease expiry", async () => {
+    const redis = new Redis();
+    clients.push(redis);
+
+    const queue = new RedisIndexJobQueue(redis as unknown as any, {
+      keyPrefix: `test:index_jobs:reclaim:${Date.now()}`,
+      maxAttempts: 2,
+      claimLeaseSeconds: 1
+    });
+
+    await queue.enqueue({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-lease-1",
+      manifest_key: "tenant-a/ws-a/idx-lease-1/manifest.json"
+    });
+
+    const claimed = await queue.claimNext(0);
+    expect(claimed).toBeDefined();
+    expect(await queue.processingCount()).toBe(1);
+
+    expect(await queue.reclaimOrphaned(1)).toBe(0);
+    await delay(1_100);
+
+    expect(await queue.reclaimOrphaned(1)).toBe(1);
+    expect(await queue.pendingCount()).toBe(1);
+    expect(await queue.processingCount()).toBe(0);
+  });
+
+  it("reconnects and retries queue operations after transient redis disconnects", async () => {
+    const redis = new Redis();
+    clients.push(redis);
+
+    const queue = new RedisIndexJobQueue(redis as unknown as any, {
+      keyPrefix: `test:index_jobs:reconnect:${Date.now()}`,
+      maxAttempts: 2,
+      reconnectRetries: 2,
+      reconnectDelayMs: 0
+    });
+
+    let failRpush = true;
+    const originalRpush = redis.rpush.bind(redis);
+    redis.rpush = (async (...args: [string, ...string[]]) => {
+      if (failRpush) {
+        failRpush = false;
+        throw new Error("Connection is closed.");
+      }
+      return originalRpush(...args);
+    }) as unknown as typeof redis.rpush;
+
+    let failClaim = true;
+    const originalBrpoplpush = redis.brpoplpush.bind(redis);
+    redis.brpoplpush = (async (...args: [string, string, number]) => {
+      if (failClaim) {
+        failClaim = false;
+        throw new Error("read ECONNRESET");
+      }
+      return originalBrpoplpush(...args);
+    }) as unknown as typeof redis.brpoplpush;
+
+    await queue.enqueue({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-reconnect",
+      manifest_key: "tenant-a/ws-a/idx-reconnect/manifest.json"
+    });
+
+    const claimed = await queue.claimNext(0);
+    expect(claimed).toBeDefined();
+    await queue.ack(claimed!);
+    expect(await queue.pendingCount()).toBe(0);
+    expect(await queue.processingCount()).toBe(0);
+  });
+});

package/test/runtime-mode.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+import { resolveRuntimeMode, sqliteRuntimeDriverName } from "../src/index.js";
+
+describe("runtime mode resolution", () => {
+  it("defaults to hybrid and falls back to local when cloud env is incomplete", () => {
+    const resolved = resolveRuntimeMode({});
+    expect(resolved.requested_mode).toBe("hybrid");
+    expect(resolved.effective_mode).toBe("local");
+    expect(resolved.cloud_configured).toBe(false);
+    expect(resolved.missing_cloud_vars.length).toBeGreaterThan(0);
+  });
+
+  it("selects cloud in hybrid mode when all cloud env vars are present", () => {
+    const resolved = resolveRuntimeMode({
+      RCE_RUNTIME_MODE: "hybrid",
+      DATABASE_URL: "postgres://localhost/db",
+      REDIS_URL: "redis://localhost:6379",
+      S3_BUCKET: "bucket",
+      S3_REGION: "us-east-1",
+      S3_ACCESS_KEY_ID: "key",
+      S3_SECRET_ACCESS_KEY: "secret"
+    });
+
+    expect(resolved.requested_mode).toBe("hybrid");
+    expect(resolved.effective_mode).toBe("cloud");
+    expect(resolved.cloud_configured).toBe(true);
+    expect(resolved.missing_cloud_vars).toHaveLength(0);
+  });
+
+  it("forces local mode without requiring cloud env", () => {
+    const resolved = resolveRuntimeMode({
+      RCE_RUNTIME_MODE: "local"
+    });
+
+    expect(resolved.requested_mode).toBe("local");
+    expect(resolved.effective_mode).toBe("local");
+  });
+
+  it("throws for cloud mode when required cloud env vars are missing", () => {
+    expect(() =>
+      resolveRuntimeMode({
+        RCE_RUNTIME_MODE: "cloud",
+        DATABASE_URL: "postgres://localhost/db"
+      })
+    ).toThrow(/RCE_RUNTIME_MODE=cloud requires/i);
+  });
+
+  it("selects a runtime-compatible sqlite driver", () => {
+    const driver = sqliteRuntimeDriverName();
+    if (process.versions.bun) {
+      expect(driver).toBe("bun:sqlite");
+      return;
+    }
+    expect(driver).toBe("node:sqlite");
+  });
+});

package/test/sqlite-queue.integration.test.ts

@@ -0,0 +1,54 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { SqliteIndexJobQueue } from "../src/index.js";
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+describe("sqlite queue integration", () => {
+  const dirs: string[] = [];
+
+  afterEach(async () => {
+    while (dirs.length > 0) {
+      const dir = dirs.pop();
+      if (dir) {
+        await rm(dir, { recursive: true, force: true });
+      }
+    }
+  });
+
+  it("reclaims stale processing jobs", async () => {
+    const root = await mkdtemp(join(tmpdir(), "rce-sqlite-queue-"));
+    dirs.push(root);
+    const sqlitePath = join(root, "jobs.sqlite");
+    const queue = new SqliteIndexJobQueue(sqlitePath, {
+      maxAttempts: 2,
+      claimTtlSeconds: 1
+    });
+
+    try {
+      await queue.enqueue({
+        tenant_id: "tenant-a",
+        workspace_id: "ws-a",
+        index_version: "idx-1",
+        manifest_key: "tenant-a/ws-a/idx-1/manifest.json"
+      });
+
+      const claimed = await queue.claimNext(0);
+      expect(claimed).toBeDefined();
+      expect(await queue.processingCount()).toBe(1);
+
+      expect(await queue.reclaimOrphaned(1)).toBe(0);
+      await delay(1_100);
+
+      expect(await queue.reclaimOrphaned(1)).toBe(1);
+      expect(await queue.pendingCount()).toBe(1);
+      expect(await queue.processingCount()).toBe(0);
+    } finally {
+      queue.close();
+    }
+  });
+});

package/test/usage-metering.integration.test.ts

@@ -0,0 +1,71 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { SqliteUsageMeterStore } from "../src/index.js";
+
+describe("usage metering store", () => {
+  const dirs: string[] = [];
+
+  afterEach(async () => {
+    while (dirs.length > 0) {
+      const dir = dirs.pop();
+      if (dir) {
+        await rm(dir, { recursive: true, force: true });
+      }
+    }
+  });
+
+  it("persists usage summaries and audit events", async () => {
+    const root = await mkdtemp(join(tmpdir(), "rce-usage-metering-"));
+    dirs.push(root);
+    const sqlitePath = join(root, "metering.sqlite");
+
+    const store = new SqliteUsageMeterStore(sqlitePath);
+    await store.migrate();
+
+    try {
+      await store.recordUsage({
+        tenant_id: "tenant-a",
+        workspace_id: "ws-a",
+        tool_name: "search_context",
+        trace_id: "trc-1",
+        status: "success",
+        latency_ms: 42,
+        result_count: 3,
+        units: 3
+      });
+      await store.recordUsage({
+        tenant_id: "tenant-a",
+        workspace_id: "ws-a",
+        tool_name: "search_context",
+        trace_id: "trc-2",
+        status: "error",
+        latency_ms: 84,
+        result_count: 0,
+        units: 1
+      });
+      await store.recordAuditEvent({
+        tenant_id: "tenant-a",
+        subject: "user-1",
+        action: "search_context",
+        resource: "workspace:ws-a",
+        status: "success",
+        trace_id: "trc-1",
+        details: { statusCode: 200 }
+      });
+
+      const usage = await store.listUsageSummary({ tenant_id: "tenant-a" });
+      expect(usage.length).toBe(1);
+      expect(usage[0]?.request_count).toBe(2);
+      expect(usage[0]?.error_count).toBe(1);
+      expect(usage[0]?.total_units).toBe(4);
+
+      const audit = await store.listAuditEvents({ tenant_id: "tenant-a", limit: 10 });
+      expect(audit.length).toBe(1);
+      expect(audit[0]?.action).toBe("search_context");
+    } finally {
+      store.close();
+    }
+  });
+});

package/tsconfig.build.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "declaration": true,
+    "declarationMap": false,
+    "rootDir": "src",
+    "outDir": "dist",
+    "tsBuildInfoFile": "dist/.tsbuildinfo"
+  },
+  "include": ["src/**/*.ts", "src/**/*.d.ts"],
+  "references": [{ "path": "../contracts/tsconfig.build.json" }]
+}

package/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": ["src/**/*.ts", "src/**/*.d.ts", "test/**/*.ts", "test/**/*.d.ts"]
+}