@stigmer/server 3.24.0 → 3.26.0

package/src/store/sqlite/__tests__/migrations.test.ts

@@ -9,7 +9,9 @@
  * (< 6) runs nothing. v10, the chain's first row-decoding step, moves
  * every row of the seven kinds that held the retired public level to org
  * and leaves every other row's bytes as they were; a row it cannot decode
- * fails the step and leaves the database at v9.
+ * fails the step and leaves the database at v9. v11 adds the list index's
+ * schema only, and a Go-written row of a declared kind is derived by the
+ * store's reconciliation at open and read through the index.
  */
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -22,8 +24,13 @@ import { create, fromBinary, fromJson, toBinary } from "@bufbuild/protobuf";
 import type { DescMessage } from "@bufbuild/protobuf";
 
 import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb";
+import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
 import { ApiResourceVisibility } from "@stigmer/protos/ai/stigmer/commons/apiresource/enum_pb";
 
+import {
+  CONTRACT_SESSION_INDEX,
+  CONTRACT_STORE_OPTIONS,
+} from "../../__tests__/store-contract.js";
 import { SqliteStore } from "../store.js";
 import {
   CURRENT_SCHEMA_VERSION,
@@ -93,7 +100,7 @@ describe("fresh database", () => {
       .prepare(`SELECT version FROM schema_version ORDER BY version`)
       .all() as Array<{ version: number }>;
     expect(rows.map((row) => row.version)).toEqual([
-      1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+      1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
     ]);
   });
 
@@ -218,6 +225,46 @@ describe("Go-created v6 database adoption (DD-002 fixture)", () => {
     expect(hits).toEqual([{ resource_id: "acme" }]);
   });
 
+  it("derives the list facts of a Go-written row of a declared kind at open", async () => {
+    const fixture = materializeGoFixture();
+    cleanups.push(() => fixture.cleanup());
+
+    // A session as the Go server wrote it: four columns, no list facts.
+    const session = create(SessionSchema, {
+      metadata: { id: "ses_go", org: "acme" },
+      spec: { agentInstanceId: "ain_go" },
+    });
+    const before = new DatabaseSync(fixture.dbPath);
+    before
+      .prepare(
+        `INSERT INTO resources VALUES ('session', 'ses_go', ?, '2026-08-23 11:43:54')`,
+      )
+      .run(toBinary(SessionSchema, session));
+    before.close();
+
+    const store = SqliteStore.open(
+      fixture.dbPath,
+      undefined,
+      CONTRACT_STORE_OPTIONS,
+    );
+    cleanups.push(() => store.close());
+
+    const db = new DatabaseSync(fixture.dbPath);
+    cleanups.push(() => db.close());
+    const unproven = db
+      .prepare(
+        `SELECT count(*) AS count FROM resources WHERE list_indexed_at IS NOT updated_at`,
+      )
+      .get() as { count: number };
+    expect(unproven.count, "every adopted row derived or stamped at open").toBe(
+      0,
+    );
+    const rows = await store.queryResources(CONTRACT_SESSION_INDEX, {
+      anyKey: [{ name: "agent_instance", value: "ain_go" }],
+    });
+    expect(rows.map((row) => row.id)).toEqual(["ses_go"]);
+  });
+
   it("reads a Go-marshaled resource blob through the TS driver (wire-format continuity)", async () => {
     const fixture = materializeGoFixture();
     cleanups.push(() => fixture.cleanup());
@@ -349,14 +396,11 @@ describe("rollback safety (DD-006)", () => {
 });
 
 describe("v10: the retired public level leaves every row", () => {
-  /** A v9 database: the chain replayed, then v10's version row removed. */
+  /** A v9 database: the chain replayed up to the step before v10. */
   function v9Database(): { dbPath: string; db: DatabaseSync } {
     const dbPath = tempDbPath();
     const setup = new DatabaseSync(dbPath);
-    runMigrations(setup);
-    setup
-      .prepare(`DELETE FROM schema_version WHERE version = ?`)
-      .run(SCHEMA_VERSION_10);
+    runMigrations(setup, SCHEMA_VERSION_10 - 1);
     expect(getSchemaVersion(setup)).toBe(SCHEMA_VERSION_10 - 1);
     return { dbPath, db: setup };
   }

package/src/store/sqlite/__tests__/store-contract.test.ts

@@ -3,39 +3,93 @@
  * store-contract.ts) against the sqlite driver — the extraction target of
  * the Phase-1 store.test.ts/substores.test.ts interface-shaped cases (T01
  * D-4). sqlite-physical pins stay in store.test.ts; the engine-specific
- * search half stays in fts5.test.ts.
+ * search half stays in fts5.test.ts. The escape hatches ride a second
+ * connection: the store's own is busy being the subject.
  */
 import { DatabaseSync } from "node:sqlite";
 
 import { describe } from "vitest";
 
+import type { Store } from "../../interface.js";
+import { apiResourceKindName } from "../../proto-fields.js";
 import type { StoreContractFixture } from "../../__tests__/store-contract.js";
-import { describeStoreContract } from "../../__tests__/store-contract.js";
+import {
+  CONTRACT_STORE_OPTIONS,
+  describeStoreContract,
+} from "../../__tests__/store-contract.js";
+import { SqliteStore } from "../store.js";
 import { tempStore } from "./support.js";
 
 describe("sqlite store contract", () => {
   describeStoreContract(async (): Promise<StoreContractFixture> => {
-    const temp = tempStore();
+    const temp = tempStore(CONTRACT_STORE_OPTIONS);
+    const others: Store[] = [];
+    const withConnection = <T>(work: (db: DatabaseSync) => T): T => {
+      const db = new DatabaseSync(temp.dbPath);
+      try {
+        return work(db);
+      } finally {
+        db.close();
+      }
+    };
     return {
       store: temp.store,
       async forceDedupeExpiry(id, expiresAtIso) {
-        // A second connection: the store's own is busy being the subject.
-        const db = new DatabaseSync(temp.dbPath);
-        db.prepare(`UPDATE signal_dedupe SET expires_at = ? WHERE id = ?`).run(
-          expiresAtIso,
-          id,
+        withConnection((db) =>
+          db
+            .prepare(`UPDATE signal_dedupe SET expires_at = ? WHERE id = ?`)
+            .run(expiresAtIso, id),
         );
-        db.close();
       },
       async countPendingOAuthStates() {
-        const db = new DatabaseSync(temp.dbPath);
-        const row = db
-          .prepare(`SELECT COUNT(*) AS count FROM pending_oauth_state`)
-          .get() as { count: number };
-        db.close();
-        return row.count;
+        return withConnection(
+          (db) =>
+            (
+              db
+                .prepare(`SELECT COUNT(*) AS count FROM pending_oauth_state`)
+                .get() as {
+                count: number;
+              }
+            ).count,
+        );
+      },
+      async writeAsOlderBinary(kind, id, data) {
+        // The statement this driver ran before the list index existed.
+        withConnection((db) =>
+          db
+            .prepare(
+              `INSERT OR REPLACE INTO resources (kind, id, data, updated_at) VALUES (?, ?, ?, datetime('now'))`,
+            )
+            .run(apiResourceKindName(kind), id, data),
+        );
+      },
+      async countUnproven(kind, revision) {
+        return withConnection((db) => {
+          const revisionArm =
+            revision === undefined
+              ? ""
+              : ` OR list_index_revision IS NULL OR list_index_revision <> ${revision}`;
+          return (
+            db
+              .prepare(
+                `SELECT COUNT(*) AS count FROM resources
+                 WHERE kind = ? AND (list_indexed_at IS NOT updated_at${revisionArm})`,
+              )
+              .get(apiResourceKindName(kind)) as { count: number }
+          ).count;
+        });
+      },
+      async openAnother(options) {
+        const other = SqliteStore.open(temp.dbPath, undefined, options);
+        others.push(other);
+        return other;
+      },
+      async cleanup() {
+        for (const other of others) {
+          await other.close();
+        }
+        await temp.cleanup();
       },
-      cleanup: () => temp.cleanup(),
     };
   });
 });

package/src/store/sqlite/__tests__/support.ts

@@ -9,6 +9,8 @@ import { tmpdir } from "node:os";
 import path from "node:path";
 import { DatabaseSync } from "node:sqlite";
 
+import { LIST_INDEXES } from "../../../boot/list-indexes.js";
+import type { StoreOpenOptions } from "../../interface.js";
 import { SqliteStore } from "../store.js";
 
 export { makeOrganization } from "../../__tests__/support.js";
@@ -19,11 +21,18 @@ export interface TempStore {
   cleanup(): Promise<void>;
 }
 
-/** A fresh store on a throwaway database file. */
-export function tempStore(): TempStore {
+/**
+ * A fresh store on a throwaway database file, opened with the server's
+ * list indexes (boot/list-indexes.ts) the way the composition root opens
+ * one, so a lane under test reads through the index it reads in
+ * production. `options` replaces them for a test that declares its own.
+ */
+export function tempStore(
+  options: StoreOpenOptions = { listIndexes: LIST_INDEXES },
+): TempStore {
   const dir = mkdtempSync(path.join(tmpdir(), "stigmer-store-test-"));
   const dbPath = path.join(dir, "stigmer.db");
-  const store = SqliteStore.open(dbPath);
+  const store = SqliteStore.open(dbPath, undefined, options);
   return {
     store,
     dbPath,

package/src/store/sqlite/migrations.ts

@@ -1,8 +1,9 @@
 /**
  * Versioned schema migrations — ports the inline chain in
  * backend/libs/go/store/sqlite/store.go (v1–v6, DDL character-faithful),
- * adds v7, the OD-3 consolidation (D2 §3), and v8, the by-resource
- * oauth_grant index (the channel teardown's query pattern, C3 Stage 6).
+ * adds v7, the OD-3 consolidation (D2 §3), v8, the by-resource
+ * oauth_grant index (the channel teardown's query pattern, C3 Stage 6),
+ * and the later steps each named at its constant below.
  *
  * Schema continuity across cutover is the design point: a database the Go
  * server created at any version migrates forward through the SAME steps Go
@@ -51,12 +52,20 @@ export const SCHEMA_VERSION_8 = 8;
 export const SCHEMA_VERSION_9 = 9;
 /** v10: every row holding the retired public visibility level moved to org. */
 export const SCHEMA_VERSION_10 = 10;
+/** v11: the list index's columns, key table and indexes (DDL only). */
+export const SCHEMA_VERSION_11 = 11;
 
 /** Target version for new databases. */
-export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_10;
+export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_11;
 
-/** Applies all pending migrations in order. */
-export function runMigrations(db: DatabaseSync): void {
+/**
+ * Applies every pending migration up to `targetVersion` in order — all of
+ * them unless a test builds the database a given step starts from.
+ */
+export function runMigrations(
+  db: DatabaseSync,
+  targetVersion: number = CURRENT_SCHEMA_VERSION,
+): void {
   db.exec(`
     CREATE TABLE IF NOT EXISTS schema_version (
       version INTEGER PRIMARY KEY,
@@ -77,10 +86,11 @@ export function runMigrations(db: DatabaseSync): void {
     [SCHEMA_VERSION_8, migrateToV8],
     [SCHEMA_VERSION_9, migrateToV9],
     [SCHEMA_VERSION_10, migrateToV10],
+    [SCHEMA_VERSION_11, migrateToV11],
   ];
 
   for (const [version, migrate] of chain) {
-    if (currentVersion < version) {
+    if (currentVersion < version && version <= targetVersion) {
       applyInTransaction(db, version, migrate);
     }
   }
@@ -423,3 +433,44 @@ function migrateToV10(db: DatabaseSync): void {
     }
   }
 }
+
+/**
+ * v11: the list index (../list-index.ts) — schema only, the Postgres
+ * driver's v6 in this engine's terms (postgres/migrations.ts gives the
+ * reasons for every column and index). Existing rows, a Go-era database's
+ * included, arrive unproven and are derived by the store's reconciliation
+ * at open. sqlite compares text as bytes (BINARY) by default, the order
+ * list-index.ts merges in, so no collation is spelled here. The store now
+ * writes `updated_at` to the millisecond and the stamp from the same
+ * expression in the same statement; a writer that does not know the index
+ * writes `datetime('now')`, whole seconds, which can never equal a stamp.
+ */
+function migrateToV11(db: DatabaseSync): void {
+  db.exec(`
+    ALTER TABLE resources ADD COLUMN list_org TEXT;
+    ALTER TABLE resources ADD COLUMN list_created_at TEXT;
+    ALTER TABLE resources ADD COLUMN list_index_revision INTEGER;
+    ALTER TABLE resources ADD COLUMN list_indexed_at TEXT;
+
+    CREATE INDEX idx_resources_list_org
+      ON resources (kind, list_org, list_created_at, id);
+    CREATE INDEX idx_resources_list_created
+      ON resources (kind, list_created_at, id);
+    CREATE INDEX idx_resources_list_revision
+      ON resources (kind, list_index_revision);
+    CREATE INDEX idx_resources_list_unproven
+      ON resources (kind, id) WHERE list_indexed_at IS NOT updated_at;
+
+    CREATE TABLE resource_list_keys (
+      kind TEXT NOT NULL,
+      id TEXT NOT NULL,
+      key TEXT NOT NULL,
+      value TEXT NOT NULL,
+      created_at TEXT NOT NULL,
+      PRIMARY KEY (kind, id, key)
+    ) WITHOUT ROWID;
+
+    CREATE INDEX idx_resource_list_keys_lookup
+      ON resource_list_keys (kind, key, value, created_at, id);
+  `);
+}