@stigmer/server 3.25.0 → 3.27.0

package/src/store/postgres/__tests__/migrations.test.ts

@@ -7,7 +7,9 @@
  * single-file lock never had. v5, the chain's first row-decoding step,
  * moves every row of the seven kinds that held the retired public level to
  * org and leaves every other row's bytes as they were; a row it cannot
- * decode fails the step and leaves the database at v4.
+ * decode fails the step and leaves the database at v4. v6 adds the list
+ * index's table (its behaviour is the store contract's). A step's starting
+ * database is built by running the chain up to the step before it.
  *
  * Gated on TEST_DATABASE_URL (see support.ts): visible skips without a
  * database, always exercised in CI via the ci.stigmer-server service
@@ -24,7 +26,12 @@ import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb"
 import { ApiResourceVisibility } from "@stigmer/protos/ai/stigmer/commons/apiresource/enum_pb";
 
 import { PUBLIC_ROW_KINDS_AT_RETIREMENT } from "../../public-visibility-retired.js";
-import { CURRENT_SCHEMA_VERSION, SCHEMA_VERSION_5 } from "../migrations.js";
+import {
+  CURRENT_SCHEMA_VERSION,
+  SCHEMA_VERSION_1,
+  SCHEMA_VERSION_5,
+  runMigrations,
+} from "../migrations.js";
 import { PostgresStore } from "../store.js";
 import {
   createTestDatabase,
@@ -32,6 +39,18 @@ import {
   type TestDatabase,
 } from "./support.js";
 
+/** Builds the database a step starts from: the chain run up to `version`. */
+async function migrateTo(databaseUrl: string, version: number): Promise<void> {
+  const pool = new pg.Pool({ connectionString: databaseUrl, max: 1 });
+  const client = await pool.connect();
+  try {
+    await runMigrations(client, version);
+  } finally {
+    client.release();
+    await pool.end();
+  }
+}
+
 describe.skipIf(testDatabaseAdminUrl() === undefined)(
   "postgres migrations",
   () => {
@@ -71,6 +90,7 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
           "oauth_grant",
           "pending_oauth_state",
           "resource_audit",
+          "resource_list_keys",
           "resources",
           "schedule_runs",
           "schema_version",
@@ -102,21 +122,13 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
     });
 
     it("a v1 database resumes the chain on reopen: the v2/v3 indexes arrive and v4 removes the Project rows", async () => {
-      const store = await PostgresStore.open(db.databaseUrl);
-      await store.close();
-
-      // Rewind to a v1 database: drop exactly what v2+v4 created and their
-      // version rows — the state any pre-v2 deployment is actually in —
-      // and seed one row of the Project kind an earlier release wrote
+      // A v1 database — the state any pre-v2 deployment is actually in —
+      // seeded with one row of the Project kind an earlier release wrote
       // (the kind column holds the enum NAME) beside a row v4 must keep.
+      await migrateTo(db.databaseUrl, SCHEMA_VERSION_1);
       const client = new pg.Client({ connectionString: db.databaseUrl });
       await client.connect();
       try {
-        await client.query(`DROP INDEX idx_wfee_created_at`);
-        await client.query(`DROP INDEX idx_oauth_grant_resource`);
-        await client.query(
-          `DELETE FROM schema_version WHERE version IN (2, 3, 4, 5)`,
-        );
         await client.query(
           `INSERT INTO resources (kind, id, data) VALUES ('project', 'prj_seeded', '\\x00'), ('organization', 'acme', '\\x00')`,
         );
@@ -242,15 +254,11 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
         );
       }
 
-      /** Opens the store once (the chain replays), then rewinds to v4 for the seeding. */
+      /** A v4 database, the one v5 starts from, and a client on it for the seeding. */
       async function v4Client(): Promise<pg.Client> {
-        const store = await PostgresStore.open(db.databaseUrl);
-        await store.close();
+        await migrateTo(db.databaseUrl, SCHEMA_VERSION_5 - 1);
         const client = new pg.Client({ connectionString: db.databaseUrl });
         await client.connect();
-        await client.query(`DELETE FROM schema_version WHERE version = $1`, [
-          SCHEMA_VERSION_5,
-        ]);
         return client;
       }
 

package/src/store/postgres/__tests__/store-contract.test.ts

@@ -15,8 +15,13 @@ import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/
 
 import pg from "pg";
 
+import type { Store } from "../../interface.js";
+import { apiResourceKindName } from "../../proto-fields.js";
 import type { StoreContractFixture } from "../../__tests__/store-contract.js";
-import { describeStoreContract } from "../../__tests__/store-contract.js";
+import {
+  CONTRACT_STORE_OPTIONS,
+  describeStoreContract,
+} from "../../__tests__/store-contract.js";
 import { PostgresStore } from "../store.js";
 import {
   createTestDatabase,
@@ -26,6 +31,7 @@ import {
 
 const ALL_TABLES = [
   "resources",
+  "resource_list_keys",
   "resource_audit",
   "search_index",
   "bootstrap_state",
@@ -64,7 +70,12 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
       await hooks.query(
         `TRUNCATE ${ALL_TABLES.join(", ")} RESTART IDENTITY CASCADE`,
       );
-      const store = await PostgresStore.open(db.databaseUrl);
+      const store = await PostgresStore.open(
+        db.databaseUrl,
+        undefined,
+        CONTRACT_STORE_OPTIONS,
+      );
+      const others: Store[] = [];
       return {
         store,
         async forceDedupeExpiry(id, expiresAtIso) {
@@ -79,7 +90,43 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
           );
           return Number((result.rows[0] as { count: string }).count);
         },
-        cleanup: () => store.close(),
+        async writeAsOlderBinary(kind, id, data) {
+          // The statement this driver ran before the list index existed.
+          await hooks.query(
+            `INSERT INTO resources (kind, id, data, updated_at) VALUES ($1, $2, $3, now())
+             ON CONFLICT (kind, id) DO UPDATE SET data = excluded.data, updated_at = now()`,
+            [apiResourceKindName(kind), id, Buffer.from(data)],
+          );
+        },
+        async countUnproven(kind, revision) {
+          const revisionArm =
+            revision === undefined
+              ? ""
+              : ` OR list_index_revision IS NULL OR list_index_revision <> $2`;
+          const result = await hooks.query(
+            `SELECT COUNT(*) AS count FROM resources
+             WHERE kind = $1 AND (list_indexed_at IS DISTINCT FROM updated_at${revisionArm})`,
+            revision === undefined
+              ? [apiResourceKindName(kind)]
+              : [apiResourceKindName(kind), revision],
+          );
+          return Number((result.rows[0] as { count: string }).count);
+        },
+        async openAnother(options) {
+          const other = await PostgresStore.open(
+            db.databaseUrl,
+            undefined,
+            options,
+          );
+          others.push(other);
+          return other;
+        },
+        async cleanup() {
+          for (const other of others) {
+            await other.close();
+          }
+          await store.close();
+        },
       };
     });
 

package/src/store/postgres/migrations.ts

@@ -1,7 +1,7 @@
 /**
  * Versioned schema migrations for the Postgres driver — an INDEPENDENT
  * chain starting at its own v1 (DD-010 §3). It deliberately does NOT
- * mirror sqlite's v1–v7: that chain's value is Go-DDL fidelity for adopted
+ * mirror sqlite's chain: that chain's value is Go-DDL fidelity for adopted
  * laptop databases, a concern Postgres cannot have (no Postgres database
  * predates this driver). Same runner discipline as sqlite/migrations.ts —
  * deliberate, versioned, each step in its own transaction, `schema_version`
@@ -50,9 +50,11 @@ export const SCHEMA_VERSION_3 = 3;
 export const SCHEMA_VERSION_4 = 4;
 /** v5: every row holding the retired public visibility level moved to org. */
 export const SCHEMA_VERSION_5 = 5;
+/** v6: the list index's columns, key table and indexes (DDL only). */
+export const SCHEMA_VERSION_6 = 6;
 
 /** Target version for new databases. */
-export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_5;
+export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_6;
 
 /**
  * Advisory lock key for the migration chain. Arbitrary but stable 64-bit
@@ -63,11 +65,16 @@ export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_5;
 export const MIGRATION_LOCK_KEY = 0x5354474d5231n;
 
 /**
- * Applies all pending migrations in order, serialized across instances by
- * the advisory lock. Runs on a dedicated client (locks are session-scoped;
- * a pool query could release on a different session).
+ * Applies every pending migration up to `targetVersion` in order — all of
+ * them unless a test builds the database a given step starts from —
+ * serialized across instances by the advisory lock. Runs on a dedicated
+ * client (locks are session-scoped; a pool query could release on a
+ * different session).
  */
-export async function runMigrations(client: PoolClient): Promise<void> {
+export async function runMigrations(
+  client: PoolClient,
+  targetVersion: number = CURRENT_SCHEMA_VERSION,
+): Promise<void> {
   await client.query("SELECT pg_advisory_lock($1)", [MIGRATION_LOCK_KEY]);
   try {
     await client.query(`
@@ -87,10 +94,11 @@ export async function runMigrations(client: PoolClient): Promise<void> {
       [SCHEMA_VERSION_3, migrateToV3],
       [SCHEMA_VERSION_4, migrateToV4],
       [SCHEMA_VERSION_5, migrateToV5],
+      [SCHEMA_VERSION_6, migrateToV6],
     ];
 
     for (const [version, migrate] of chain) {
-      if (currentVersion < version) {
+      if (currentVersion < version && version <= targetVersion) {
         await applyInTransaction(client, version, migrate);
       }
     }
@@ -362,3 +370,57 @@ async function migrateToV5(client: PoolClient): Promise<void> {
     }
   }
 }
+
+/**
+ * v6: the list index (../list-index.ts) — schema only. Every existing row
+ * arrives with its stamp NULL, which makes it unproven, so the store's
+ * reconciliation at open derives its facts with the code that runs and
+ * reads stay exact until it has; a migration that decoded rows here would
+ * freeze today's declarations into the chain (the v5 step freezes its
+ * kind table for the same reason, public-visibility-retired.ts).
+ *
+ * - `list_org` and `list_created_at` are the order's inputs, byte-collated
+ *   (`COLLATE "C"`) so the database sorts them the way list-index.ts
+ *   compares them when it merges unproven rows in; `id` is compared under
+ *   the same collation in the index expressions and the queries.
+ * - `list_index_revision` is the declaration revision the facts were
+ *   derived under; `list_indexed_at` is the `updated_at` they were derived
+ *   from. A writer that does not know these columns leaves them behind as
+ *   it bumps `updated_at`, which is exactly what marks its row unproven.
+ * - `resource_list_keys` holds one row per (resource, key) with the
+ *   creation instant copied in, so a parent's rows are one range of one
+ *   index. It carries no foreign key: every indexed read joins
+ *   `resources`, so a key row whose resource is gone matches nothing.
+ * - The partial index holds only unproven rows (none in steady state), so
+ *   finding them is never a scan of the kind.
+ */
+async function migrateToV6(client: PoolClient): Promise<void> {
+  await client.query(`
+    ALTER TABLE resources
+      ADD COLUMN list_org TEXT COLLATE "C",
+      ADD COLUMN list_created_at TEXT COLLATE "C",
+      ADD COLUMN list_index_revision INTEGER,
+      ADD COLUMN list_indexed_at timestamptz;
+
+    CREATE INDEX idx_resources_list_org
+      ON resources (kind, list_org, list_created_at, (id COLLATE "C"));
+    CREATE INDEX idx_resources_list_created
+      ON resources (kind, list_created_at, (id COLLATE "C"));
+    CREATE INDEX idx_resources_list_revision
+      ON resources (kind, list_index_revision);
+    CREATE INDEX idx_resources_list_unproven
+      ON resources (kind, id) WHERE list_indexed_at IS DISTINCT FROM updated_at;
+
+    CREATE TABLE resource_list_keys (
+      kind TEXT NOT NULL,
+      id TEXT NOT NULL,
+      key TEXT NOT NULL,
+      value TEXT NOT NULL,
+      created_at TEXT COLLATE "C" NOT NULL,
+      PRIMARY KEY (kind, id, key)
+    );
+
+    CREATE INDEX idx_resource_list_keys_lookup
+      ON resource_list_keys (kind, key, value, created_at, (id COLLATE "C"));
+  `);
+}