@stigmer/server 3.14.0-dev.20260910084630

package/src/store/sqlite/__tests__/migrations.test.ts

@@ -0,0 +1,308 @@
+/**
+ * Pins the schema-continuity contract (D2 §3): a fresh database replays
+ * v1–v7; a REAL Go-created v6 database (the DD-002 fixture) adopts to v7
+ * with every row preserved — including the three out-of-chain tables Go's
+ * consumer stores created lazily; a pending_oauth_state table that
+ * predates Go's idempotent ALTERs gains its columns; a legacy pre-v2
+ * database gets its prefix-based audit rows migrated. Rollback safety
+ * (DD-006): a v7 database re-opened by Go-shaped version checks
+ * (< 6) runs nothing.
+ */
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { SqliteStore } from "../store.js";
+import {
+  CURRENT_SCHEMA_VERSION,
+  getSchemaVersion,
+  runMigrations,
+} from "../migrations.js";
+import { materializeGoFixture } from "./support.js";
+
+const cleanups: Array<() => void | Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()!();
+  }
+});
+
+function tempDbPath(): string {
+  const dir = mkdtempSync(path.join(tmpdir(), "stigmer-migrations-"));
+  cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
+  return path.join(dir, "stigmer.db");
+}
+
+function tableNames(db: DatabaseSync): string[] {
+  const rows = db
+    .prepare(
+      `SELECT name FROM sqlite_master WHERE type IN ('table') ORDER BY name`,
+    )
+    .all() as Array<{ name: string }>;
+  return rows.map((row) => row.name);
+}
+
+describe("fresh database", () => {
+  it("replays the full chain to the current version with every table present", async () => {
+    const dbPath = tempDbPath();
+    const store = SqliteStore.open(dbPath);
+    cleanups.push(() => store.close());
+
+    const db = new DatabaseSync(dbPath);
+    cleanups.push(() => db.close());
+
+    expect(getSchemaVersion(db)).toBe(CURRENT_SCHEMA_VERSION);
+    const tables = tableNames(db);
+    for (const expected of [
+      "resources",
+      "resource_audit",
+      "search_index",
+      "bootstrap_state",
+      "workflow_execution_events",
+      "schedule_runs",
+      "signal_dedupe",
+      "oauth_grant",
+      "pending_oauth_state",
+    ]) {
+      expect(tables, `table ${expected} should exist`).toContain(expected);
+    }
+  });
+
+  it("records one schema_version row per migration (MAX semantics, as Go computes it)", () => {
+    const dbPath = tempDbPath();
+    const db = new DatabaseSync(dbPath);
+    cleanups.push(() => db.close());
+    runMigrations(db);
+
+    const rows = db
+      .prepare(`SELECT version FROM schema_version ORDER BY version`)
+      .all() as Array<{ version: number }>;
+    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]);
+  });
+
+  it("re-opening an already-migrated database is a no-op", async () => {
+    const dbPath = tempDbPath();
+    const first = SqliteStore.open(dbPath);
+    await first.close();
+    const second = SqliteStore.open(dbPath);
+    cleanups.push(() => second.close());
+
+    const db = new DatabaseSync(dbPath);
+    cleanups.push(() => db.close());
+    const count = db
+      .prepare(`SELECT COUNT(*) AS count FROM schema_version`)
+      .get() as { count: number };
+    expect(count.count).toBe(CURRENT_SCHEMA_VERSION);
+  });
+});
+
+describe("Go-created v6 database adoption (DD-002 fixture)", () => {
+  it("migrates 6 → current preserving every row, including the out-of-chain tables", async () => {
+    const fixture = materializeGoFixture();
+    cleanups.push(() => fixture.cleanup());
+
+    // Preconditions: the fixture really is a v6 database with live data.
+    const before = new DatabaseSync(fixture.dbPath);
+    expect(getSchemaVersion(before)).toBe(6);
+    before.close();
+
+    const store = SqliteStore.open(fixture.dbPath);
+    cleanups.push(() => store.close());
+
+    const db = new DatabaseSync(fixture.dbPath);
+    cleanups.push(() => db.close());
+
+    expect(getSchemaVersion(db)).toBe(CURRENT_SCHEMA_VERSION);
+
+    // Every Go-written row survives adoption untouched.
+    const org = db
+      .prepare(`SELECT id FROM resources WHERE kind = 'organization'`)
+      .get() as { id: string };
+    expect(org.id).toBe("acme");
+
+    const audit = db
+      .prepare(`SELECT version_hash, tag FROM resource_audit`)
+      .get() as { version_hash: string; tag: string };
+    expect(audit).toEqual({ version_hash: "hash-v1", tag: "stable" });
+
+    const bootstrap = db
+      .prepare(
+        `SELECT value FROM bootstrap_state WHERE key = 'seedpack_version'`,
+      )
+      .get() as { value: string };
+    expect(bootstrap.value).toBe("1.1.0");
+
+    const events = db
+      .prepare(
+        `SELECT COUNT(*) AS count FROM workflow_execution_events WHERE execution_id = 'wfe_fixture'`,
+      )
+      .get() as { count: number };
+    expect(events.count).toBe(2);
+
+    const run = db
+      .prepare(`SELECT outcome, completed_at FROM schedule_runs`)
+      .get() as { outcome: string; completed_at: string };
+    expect(run.outcome).toBe("completed");
+    expect(run.completed_at).toBe("2026-08-20T00:00:05Z");
+
+    // The consolidated tables: Go-written rows adopted, not shadowed.
+    const dedupe = db
+      .prepare(`SELECT status FROM signal_dedupe WHERE id = 'acme:fixture-key'`)
+      .get() as { status: string };
+    expect(dedupe.status).toBe("DELIVERED");
+
+    const grant = db
+      .prepare(
+        `SELECT client_id FROM oauth_grant WHERE identity_account_id = 'ida_fixture'`,
+      )
+      .get() as { client_id: string };
+    expect(grant.client_id).toBe("client-1");
+
+    const pending = db
+      .prepare(
+        `SELECT code_verifier, org FROM pending_oauth_state WHERE state = 'state-fixture'`,
+      )
+      .get() as { code_verifier: string; org: string };
+    expect(pending.code_verifier).toBe("enc:v1:sealed");
+    expect(pending.org).toBe("acme");
+
+    // The Go-written FTS5 index stays queryable through the TS driver's
+    // connection — SP-C on real Go-built index data.
+    const hits = db
+      .prepare(
+        `SELECT resource_id FROM search_index WHERE search_index MATCH 'fixture'`,
+      )
+      .all() as Array<{ resource_id: string }>;
+    expect(hits).toEqual([{ resource_id: "acme" }]);
+  });
+
+  it("reads a Go-marshaled resource blob through the TS driver (wire-format continuity)", async () => {
+    const fixture = materializeGoFixture();
+    cleanups.push(() => fixture.cleanup());
+
+    const store = SqliteStore.open(fixture.dbPath);
+    cleanups.push(() => store.close());
+
+    const { OrganizationSchema } =
+      await import("@stigmer/protos/ai/stigmer/tenancy/organization/v1/api_pb");
+    const { ApiResourceKind } =
+      await import("@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb");
+    const org = await store.getResource(
+      ApiResourceKind.organization,
+      "acme",
+      OrganizationSchema,
+    );
+    expect(org.metadata?.slug).toBe("acme");
+    expect(org.spec?.description).toBe("fixture org");
+  });
+});
+
+describe("v7 column reconciliation", () => {
+  it("adds org/token_auth_method to a pending_oauth_state table that predates Go's ALTERs", () => {
+    const dbPath = tempDbPath();
+    const setup = new DatabaseSync(dbPath);
+
+    // A v6 database whose pending_oauth_state was created by an OLD Go
+    // build — before pending_state_store.go gained the idempotent ALTERs
+    // for `org` and `token_auth_method`. v7 must reconcile the columns
+    // without touching the row.
+    setup.exec(`
+      CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (datetime('now')));
+      INSERT INTO schema_version (version) VALUES (1),(2),(3),(4),(5),(6);
+      CREATE TABLE resources (kind TEXT NOT NULL, id TEXT NOT NULL, data BLOB NOT NULL, updated_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (kind, id)) WITHOUT ROWID;
+      CREATE TABLE pending_oauth_state (
+        state               TEXT PRIMARY KEY,
+        code_verifier       TEXT NOT NULL,
+        client_id           TEXT NOT NULL DEFAULT '',
+        client_secret       TEXT NOT NULL DEFAULT '',
+        token_endpoint      TEXT NOT NULL DEFAULT '',
+        mcp_server_id       TEXT NOT NULL,
+        identity_account_id TEXT NOT NULL,
+        target_env_var      TEXT NOT NULL DEFAULT '',
+        auth_method         TEXT NOT NULL DEFAULT '',
+        redirect_uri        TEXT NOT NULL DEFAULT '',
+        created_at          INTEGER NOT NULL
+      );
+      INSERT INTO pending_oauth_state (state, code_verifier, mcp_server_id, identity_account_id, created_at)
+        VALUES ('old-state', 'verifier', 'mcp_1', 'ida_1', 1700000000);
+    `);
+    setup.close();
+
+    const migrating = new DatabaseSync(dbPath);
+    runMigrations(migrating);
+    migrating.close();
+
+    const db = new DatabaseSync(dbPath);
+    cleanups.push(() => db.close());
+    const row = db
+      .prepare(
+        `SELECT state, org, token_auth_method FROM pending_oauth_state WHERE state = 'old-state'`,
+      )
+      .get() as { state: string; org: string; token_auth_method: string };
+    expect(row).toEqual({ state: "old-state", org: "", token_auth_method: "" });
+    expect(getSchemaVersion(db)).toBe(CURRENT_SCHEMA_VERSION);
+  });
+});
+
+describe("legacy pre-v2 database", () => {
+  it("moves prefix-based audit rows out of resources into resource_audit (Go migrateAuditRecords)", () => {
+    const dbPath = tempDbPath();
+    const setup = new DatabaseSync(dbPath);
+
+    // A v1-era database: audit snapshots lived in `resources` under
+    // "<type>_audit/<resource_id>/<timestamp>" ids.
+    setup.exec(`
+      CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (datetime('now')));
+      INSERT INTO schema_version (version) VALUES (1);
+      CREATE TABLE resources (kind TEXT NOT NULL, id TEXT NOT NULL, data BLOB NOT NULL, updated_at TEXT NOT NULL DEFAULT (datetime('now')), PRIMARY KEY (kind, id)) WITHOUT ROWID;
+      INSERT INTO resources (kind, id, data) VALUES ('skill', 'skl_1', X'0a01aa');
+      INSERT INTO resources (kind, id, data) VALUES ('skill', 'skill_audit/skl_1/1706123456789', X'0a01bb');
+    `);
+    setup.close();
+
+    const migrating = new DatabaseSync(dbPath);
+    runMigrations(migrating);
+    migrating.close();
+
+    const db = new DatabaseSync(dbPath);
+    cleanups.push(() => db.close());
+
+    const liveIds = (
+      db.prepare(`SELECT id FROM resources ORDER BY id`).all() as Array<{
+        id: string;
+      }>
+    ).map((row) => row.id);
+    expect(liveIds, "the legacy audit row leaves resources").toEqual(["skl_1"]);
+
+    const audit = db
+      .prepare(`SELECT resource_id, version_hash, tag FROM resource_audit`)
+      .get() as { resource_id: string; version_hash: string; tag: string };
+    expect(audit.resource_id).toBe("skl_1");
+    // Hash/tag stay empty for migrated rows (unknowable without the type).
+    expect(audit.version_hash).toBe("");
+    expect(audit.tag).toBe("");
+    expect(getSchemaVersion(db)).toBe(CURRENT_SCHEMA_VERSION);
+  });
+});
+
+describe("rollback safety (DD-006)", () => {
+  it("a v7 database passes Go-shaped 'currentVersion < 6' checks untouched", () => {
+    const dbPath = tempDbPath();
+    const migrating = new DatabaseSync(dbPath);
+    runMigrations(migrating);
+    migrating.close();
+
+    const db = new DatabaseSync(dbPath);
+    cleanups.push(() => db.close());
+    // Go's runMigrations gates every step on currentVersion < N with
+    // N <= 6; at 7 nothing runs. The MAX query is exactly Go's read.
+    const version = db
+      .prepare(`SELECT COALESCE(MAX(version), 0) AS v FROM schema_version`)
+      .get() as { v: number };
+    expect(version.v).toBeGreaterThanOrEqual(6);
+  });
+});

package/src/store/sqlite/__tests__/store-contract.test.ts

@@ -0,0 +1,41 @@
+/**
+ * Runs the driver-agnostic Store contract suite (../../__tests__/
+ * store-contract.ts) against the sqlite driver — the extraction target of
+ * the Phase-1 store.test.ts/substores.test.ts interface-shaped cases (T01
+ * D-4). sqlite-physical pins stay in store.test.ts; the engine-specific
+ * search half stays in fts5.test.ts.
+ */
+import { DatabaseSync } from "node:sqlite";
+
+import { describe } from "vitest";
+
+import type { StoreContractFixture } from "../../__tests__/store-contract.js";
+import { describeStoreContract } from "../../__tests__/store-contract.js";
+import { tempStore } from "./support.js";
+
+describe("sqlite store contract", () => {
+  describeStoreContract(async (): Promise<StoreContractFixture> => {
+    const temp = tempStore();
+    return {
+      store: temp.store,
+      async forceDedupeExpiry(id, expiresAtIso) {
+        // A second connection: the store's own is busy being the subject.
+        const db = new DatabaseSync(temp.dbPath);
+        db.prepare(`UPDATE signal_dedupe SET expires_at = ? WHERE id = ?`).run(
+          expiresAtIso,
+          id,
+        );
+        db.close();
+      },
+      async countPendingOAuthStates() {
+        const db = new DatabaseSync(temp.dbPath);
+        const row = db
+          .prepare(`SELECT COUNT(*) AS count FROM pending_oauth_state`)
+          .get() as { count: number };
+        db.close();
+        return row.count;
+      },
+      cleanup: () => temp.cleanup(),
+    };
+  });
+});

package/src/store/sqlite/__tests__/store.test.ts

@@ -0,0 +1,105 @@
+/**
+ * Pins the sqlite driver's PHYSICAL contracts — the parts a Go-era
+ * database on disk depends on and the contract suite deliberately cannot
+ * express: the resources column layout (kind = the proto enum name,
+ * updated_at stamped), and the SP-C FTS5 probe (bm25 weights, porter
+ * stemming, DELETE+INSERT upsert), kept permanently so a Node build that
+ * drops FTS5 fails CI here, not at a user's laptop.
+ *
+ * The driver's interface-shaped behavior is pinned by the shared contract
+ * suite — see store-contract.test.ts (T01 D-4 extraction).
+ */
+import { DatabaseSync } from "node:sqlite";
+
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+import { OrganizationSchema } from "@stigmer/protos/ai/stigmer/tenancy/organization/v1/api_pb";
+
+import { makeOrganization, tempStore, type TempStore } from "./support.js";
+
+const KIND = ApiResourceKind.organization;
+
+let temp: TempStore;
+
+beforeEach(() => {
+  temp = tempStore();
+});
+
+afterEach(async () => {
+  await temp.cleanup();
+});
+
+describe("physical layout", () => {
+  it("stores kind as the proto name and stamps updated_at (Go reads these exact columns)", async () => {
+    const org = makeOrganization({ id: "acme" });
+    await temp.store.saveResource(KIND, "acme", OrganizationSchema, org);
+
+    const db = new DatabaseSync(temp.dbPath);
+    const row = db
+      .prepare(`SELECT kind, updated_at FROM resources WHERE id = 'acme'`)
+      .get() as { kind: string; updated_at: string };
+    db.close();
+    expect(row.kind).toBe("organization");
+    expect(row.updated_at).not.toBe("");
+  });
+});
+
+describe("search index (SP-C, permanent)", () => {
+  it("upserts, ranks with Go's bm25 weights, stems with porter, and deletes", async () => {
+    await temp.store.upsertSearchIndex(KIND, "acme", {
+      name: "Billing Reconciler",
+      description: "reconciles billing runs nightly",
+      tags: "billing finance",
+      org: "acme",
+      visibility: "visibility_org",
+      createdAt: 1,
+    });
+    await temp.store.upsertSearchIndex(KIND, "beta", {
+      name: "Invoice Sync",
+      description: "synchronizes invoices with the billing ledger",
+      tags: "",
+      org: "acme",
+      visibility: "visibility_org",
+      createdAt: 2,
+    });
+
+    // Read through a second connection — WAL allows concurrent readers,
+    // and the query store (#13) will read exactly this way.
+    const db = new DatabaseSync(temp.dbPath);
+    const ranked = db
+      .prepare(
+        `SELECT resource_id, bm25(search_index, 1.0, 0, 10.0, 5.0, 5.0) AS score
+         FROM search_index WHERE search_index MATCH ? ORDER BY score`,
+      )
+      .all('"billing"') as Array<{ resource_id: string }>;
+    // name(10) + tags(5) hits outrank a description(5) hit.
+    expect(ranked.map((row) => row.resource_id)).toEqual(["acme", "beta"]);
+
+    const stemmed = db
+      .prepare(`SELECT resource_id FROM search_index WHERE search_index MATCH ?`)
+      .all('"reconciling"') as Array<{ resource_id: string }>;
+    expect(stemmed, "porter stems reconciling→reconcil").toEqual([
+      { resource_id: "acme" },
+    ]);
+    db.close();
+
+    // Upsert replaces (DELETE + INSERT — FTS5 has no UPDATE).
+    await temp.store.upsertSearchIndex(KIND, "acme", {
+      name: "Renamed",
+      description: "",
+      tags: "",
+      org: "acme",
+      visibility: "visibility_org",
+      createdAt: 1,
+    });
+    await temp.store.deleteSearchIndex(KIND, "beta");
+
+    const remaining = new DatabaseSync(temp.dbPath);
+    const rows = remaining
+      .prepare(`SELECT resource_id, name FROM search_index`)
+      .all() as Array<{ resource_id: string; name: string }>;
+    remaining.close();
+    expect(rows).toEqual([{ resource_id: "acme", name: "Renamed" }]);
+  });
+});

package/src/store/sqlite/__tests__/support.ts

@@ -0,0 +1,62 @@
+/**
+ * Shared helpers for the sqlite driver tests: temp-dir stores and the
+ * DD-002 Go-database fixture loader. The organization factory moved to
+ * ../../__tests__/support.ts with the contract-suite extraction (T01
+ * D-4); re-exported here so existing driver tests keep their import path.
+ */
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+import { SqliteStore } from "../store.js";
+
+export { makeOrganization } from "../../__tests__/support.js";
+
+export interface TempStore {
+  store: SqliteStore;
+  dbPath: string;
+  cleanup(): Promise<void>;
+}
+
+/** A fresh store on a throwaway database file. */
+export function tempStore(): TempStore {
+  const dir = mkdtempSync(path.join(tmpdir(), "stigmer-store-test-"));
+  const dbPath = path.join(dir, "stigmer.db");
+  const store = SqliteStore.open(dbPath);
+  return {
+    store,
+    dbPath,
+    async cleanup() {
+      await store.close();
+      rmSync(dir, { recursive: true, force: true });
+    },
+  };
+}
+
+const FIXTURE_PATH = path.join(
+  import.meta.dirname,
+  "fixtures",
+  "go-v6-database.sql",
+);
+
+/**
+ * Reconstructs the Go-created v6 database from its committed SQL dump into
+ * a throwaway file and returns the path. Write-then-reopen is required:
+ * the dump instantiates the FTS5 virtual table through writable_schema,
+ * which only takes effect on the next connection — the same shape as real
+ * adoption, where the driver always opens an existing file fresh.
+ */
+export function materializeGoFixture(): { dbPath: string; cleanup(): void } {
+  const dir = mkdtempSync(path.join(tmpdir(), "stigmer-go-fixture-"));
+  const dbPath = path.join(dir, "stigmer.db");
+  const writer = new DatabaseSync(dbPath);
+  writer.exec(readFileSync(FIXTURE_PATH, "utf8"));
+  writer.close();
+  return {
+    dbPath,
+    cleanup() {
+      rmSync(dir, { recursive: true, force: true });
+    },
+  };
+}

package/src/store/sqlite/fts5.ts

@@ -0,0 +1,70 @@
+/**
+ * FTS5 query rendering and score normalization — the sqlite driver's
+ * engine-specific half of the search read contract. The Store interface
+ * carries engine-neutral terms and wire-ready scores (DD-009: engine
+ * syntax lives inside each driver; sub-project DD-001: each driver
+ * normalizes its own engine's ranking); this module renders FTS5's.
+ *
+ * Ports the retired Go server's EscapeFTS5Query and NormalizeScore
+ * (pkg/query/search/store/sqlite_search_query_store.go, git history),
+ * which the D4 #14 port first placed in the search service — the Phase-2
+ * seam redraw moved them here so the interface could stay engine-neutral.
+ * The rendering and normalization bytes are unchanged by that move.
+ *
+ * Proven by __tests__/fts5.test.ts (Go's escaping and normalization
+ * tables case-for-case, plus the driver-level read pins) and
+ * search.conformance.test.ts on local.
+ */
+
+/**
+ * Renders the FTS5 MATCH expression for engine-neutral search terms:
+ * every term individually double-quoted (operator syntax — NOT/NEAR,
+ * column filters — becomes literal text); embedded double quotes
+ * stripped; a term left empty by stripping is dropped. A SINGLE
+ * surviving term gets a trailing `*` for prefix matching (valid on
+ * quoted terms) — the prefix decision counts terms AFTER sanitization,
+ * Go's exact order. Multi-term expressions use FTS5's implicit AND. The
+ * porter unicode61 tokenizer still stems inside quotes.
+ *
+ * Terms that sanitize to nothing render "" — FTS5 rejects an empty MATCH,
+ * which surfaces as the same store failure the retired Go server produced
+ * for such queries (behavior preserved, not beautified).
+ */
+export function renderFts5MatchExpression(terms: readonly string[]): string {
+  const quoted: string[] = [];
+  for (const term of terms) {
+    const clean = term.replaceAll('"', "");
+    if (clean === "") {
+      continue;
+    }
+    quoted.push(`"${clean}"`);
+  }
+
+  if (quoted.length === 0) {
+    return "";
+  }
+  if (quoted.length === 1) {
+    return `${quoted[0]}*`;
+  }
+  return quoted.join(" ");
+}
+
+/**
+ * Go NormalizeScore: FTS5 bm25() is negative (lower = better); map to the
+ * wire's 0–1 (higher = better). Non-negative input (list mode's pinned
+ * 1.0) → exactly 1.0; else 1 + bm25/10, clamped to [0, 1] — the linear
+ * mapping for bm25's typical −5..0 range.
+ */
+export function normalizeBm25Score(bm25Score: number): number {
+  if (bm25Score >= 0) {
+    return 1.0;
+  }
+  const score = 1.0 + bm25Score / 10.0;
+  if (score < 0) {
+    return 0;
+  }
+  if (score > 1) {
+    return 1;
+  }
+  return score;
+}