@stigmer/server 3.23.0 → 3.23.1-dev.20260923075205

package/src/store/postgres/__tests__/migrations.test.ts

@@ -4,7 +4,10 @@
  * idempotent, a mid-chain database resumes (v1 → v2 picks up the sweep
  * index), and concurrent first boots serialize on the advisory lock
  * instead of racing the chain — the multi-instance failure class sqlite's
- * single-file lock never had.
+ * single-file lock never had. v5, the chain's first row-decoding step,
+ * moves every row of the seven kinds that held the retired public level to
+ * org and leaves every other row's bytes as they were; a row it cannot
+ * decode fails the step and leaves the database at v4.
  *
  * Gated on TEST_DATABASE_URL (see support.ts): visible skips without a
  * database, always exercised in CI via the ci.stigmer-server service
@@ -14,7 +17,14 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
 
 import pg from "pg";
 
-import { CURRENT_SCHEMA_VERSION } from "../migrations.js";
+import { create, fromJson, toBinary } from "@bufbuild/protobuf";
+import type { DescMessage } from "@bufbuild/protobuf";
+
+import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb";
+import { ApiResourceVisibility } from "@stigmer/protos/ai/stigmer/commons/apiresource/enum_pb";
+
+import { PUBLIC_ROW_KINDS_AT_RETIREMENT } from "../../public-visibility-retired.js";
+import { CURRENT_SCHEMA_VERSION, SCHEMA_VERSION_5 } from "../migrations.js";
 import { PostgresStore } from "../store.js";
 import {
   createTestDatabase,
@@ -105,7 +115,7 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
         await client.query(`DROP INDEX idx_wfee_created_at`);
         await client.query(`DROP INDEX idx_oauth_grant_resource`);
         await client.query(
-          `DELETE FROM schema_version WHERE version IN (2, 3, 4)`,
+          `DELETE FROM schema_version WHERE version IN (2, 3, 4, 5)`,
         );
         await client.query(
           `INSERT INTO resources (kind, id, data) VALUES ('project', 'prj_seeded', '\\x00'), ('organization', 'acme', '\\x00')`,
@@ -120,9 +130,10 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
         const remaining = await client.query(
           `SELECT kind, id FROM resources ORDER BY kind`,
         );
-        expect(remaining.rows, "v4 removed the Project row and kept the rest").toEqual([
-          { kind: "organization", id: "acme" },
-        ]);
+        expect(
+          remaining.rows,
+          "v4 removed the Project row and kept the rest",
+        ).toEqual([{ kind: "organization", id: "acme" }]);
         const projectAudit = await client.query(
           `SELECT count(*)::int AS n FROM resource_audit WHERE kind = 'project'`,
         );
@@ -190,5 +201,175 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
         await client.end();
       }
     });
+
+    describe("v5: the retired public level leaves every row", () => {
+      /** A row of `schema` carrying only the envelope every kind shares, at `visibility`. */
+      function envelopeBytes(
+        schema: DescMessage,
+        id: string,
+        visibility: ApiResourceVisibility,
+      ): Buffer {
+        return Buffer.from(
+          toBinary(
+            schema,
+            fromJson(schema, {
+              metadata: {
+                id,
+                name: id,
+                slug: id,
+                org: "acme",
+                visibility: ApiResourceVisibility[visibility],
+              },
+            }),
+          ),
+        );
+      }
+
+      function agentBytes(
+        id: string,
+        visibility: ApiResourceVisibility,
+      ): Buffer {
+        return Buffer.from(
+          toBinary(
+            AgentSchema,
+            create(AgentSchema, {
+              apiVersion: "agentic.stigmer.ai/v1",
+              kind: "Agent",
+              metadata: { id, name: id, slug: id, org: "acme", visibility },
+              spec: { instructions: "a conformant instruction body" },
+            }),
+          ),
+        );
+      }
+
+      /** Opens the store once (the chain replays), then rewinds to v4 for the seeding. */
+      async function v4Client(): Promise<pg.Client> {
+        const store = await PostgresStore.open(db.databaseUrl);
+        await store.close();
+        const client = new pg.Client({ connectionString: db.databaseUrl });
+        await client.connect();
+        await client.query(`DELETE FROM schema_version WHERE version = $1`, [
+          SCHEMA_VERSION_5,
+        ]);
+        return client;
+      }
+
+      async function row(
+        client: pg.Client,
+        kind: string,
+        id: string,
+      ): Promise<{ data: Buffer; updated_at: Date }> {
+        const result = await client.query<{ data: Buffer; updated_at: Date }>(
+          `SELECT data, updated_at FROM resources WHERE kind = $1 AND id = $2`,
+          [kind, id],
+        );
+        return result.rows[0]!;
+      }
+
+      it("moves a public row of every kind in the frozen table to org, and leaves an org row and a private row byte-for-byte", async () => {
+        const client = await v4Client();
+        const seededAt = new Date("2026-09-01T00:00:00Z");
+        try {
+          for (const entry of PUBLIC_ROW_KINDS_AT_RETIREMENT) {
+            await client.query(
+              `INSERT INTO resources (kind, id, data, updated_at) VALUES ($1, $2, $3, $4)`,
+              [
+                entry.kind,
+                `${entry.kind}_public`,
+                envelopeBytes(
+                  entry.schema,
+                  `${entry.kind}_public`,
+                  ApiResourceVisibility.visibility_public,
+                ),
+                seededAt,
+              ],
+            );
+          }
+          const orgBytes = agentBytes(
+            "agt_org",
+            ApiResourceVisibility.visibility_org,
+          );
+          const privateBytes = agentBytes(
+            "agt_private",
+            ApiResourceVisibility.visibility_private,
+          );
+          await client.query(
+            `INSERT INTO resources (kind, id, data, updated_at) VALUES ('agent', 'agt_org', $1, $3), ('agent', 'agt_private', $2, $3)`,
+            [orgBytes, privateBytes, seededAt],
+          );
+
+          const reopened = await PostgresStore.open(db.databaseUrl);
+          await reopened.close();
+
+          const version = await client.query(
+            `SELECT COALESCE(MAX(version), 0) AS version FROM schema_version`,
+          );
+          expect(Number((version.rows[0] as { version: string }).version)).toBe(
+            CURRENT_SCHEMA_VERSION,
+          );
+          for (const entry of PUBLIC_ROW_KINDS_AT_RETIREMENT) {
+            const moved = await row(client, entry.kind, `${entry.kind}_public`);
+            expect(
+              moved.data.equals(
+                envelopeBytes(
+                  entry.schema,
+                  `${entry.kind}_public`,
+                  ApiResourceVisibility.visibility_org,
+                ),
+              ),
+              entry.kind,
+            ).toBe(true);
+            expect(
+              moved.updated_at.getTime(),
+              `${entry.kind} updated_at bumped`,
+            ).toBeGreaterThan(seededAt.getTime());
+          }
+          const org = await row(client, "agent", "agt_org");
+          expect(org.data.equals(orgBytes)).toBe(true);
+          expect(org.updated_at.getTime()).toBe(seededAt.getTime());
+          const priv = await row(client, "agent", "agt_private");
+          expect(priv.data.equals(privateBytes)).toBe(true);
+          expect(priv.updated_at.getTime()).toBe(seededAt.getTime());
+        } finally {
+          await client.end();
+        }
+      });
+
+      it("a row of a table kind that does not decode fails the step, names the row, and leaves the database at v4 with every row untouched", async () => {
+        const client = await v4Client();
+        try {
+          const publicBytes = agentBytes(
+            "agt_public",
+            ApiResourceVisibility.visibility_public,
+          );
+          await client.query(
+            `INSERT INTO resources (kind, id, data) VALUES ('agent', 'agt_public', $1), ('agent', 'agt_broken', '\\xffffff'), ('session', 'ses_opaque', '\\xffff')`,
+            [publicBytes],
+          );
+
+          await expect(PostgresStore.open(db.databaseUrl)).rejects.toThrow(
+            /migrate to v5: .*agent 'agt_broken' cannot be moved off the retired public level/,
+          );
+
+          const version = await client.query(
+            `SELECT COALESCE(MAX(version), 0) AS version FROM schema_version`,
+          );
+          expect(Number((version.rows[0] as { version: string }).version)).toBe(
+            SCHEMA_VERSION_5 - 1,
+          );
+          expect(
+            (await row(client, "agent", "agt_public")).data.equals(publicBytes),
+          ).toBe(true);
+          // A kind outside the frozen table is never decoded, whatever it holds.
+          expect(
+            (await row(client, "session", "ses_opaque")).data.equals(
+              Buffer.from([0xff, 0xff]),
+            ),
+          ).toBe(true);
+        } finally {
+          await client.end();
+        }
+      });
+    });
   },
 );

package/src/store/postgres/__tests__/store-contract.test.ts

@@ -111,8 +111,6 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
             kinds: ["agent"],
             terms: ["billing"],
             orgFilter: "",
-            crossOrgPublic: false,
-            excludePublic: false,
             limit: 20,
             offset: 0,
           });

package/src/store/postgres/migrations.ts

@@ -38,14 +38,21 @@
  */
 import type { PoolClient } from "pg";
 
+import {
+  PUBLIC_ROW_KINDS_AT_RETIREMENT,
+  movePublicRowToOrg,
+} from "../public-visibility-retired.js";
+
 export const SCHEMA_VERSION_1 = 1;
 export const SCHEMA_VERSION_2 = 2;
 export const SCHEMA_VERSION_3 = 3;
 /** v4: the rows of the removed Project kind deleted. */
 export const SCHEMA_VERSION_4 = 4;
+/** v5: every row holding the retired public visibility level moved to org. */
+export const SCHEMA_VERSION_5 = 5;
 
 /** Target version for new databases. */
-export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_4;
+export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_5;
 
 /**
  * Advisory lock key for the migration chain. Arbitrary but stable 64-bit
@@ -79,6 +86,7 @@ export async function runMigrations(client: PoolClient): Promise<void> {
       [SCHEMA_VERSION_2, migrateToV2],
       [SCHEMA_VERSION_3, migrateToV3],
       [SCHEMA_VERSION_4, migrateToV4],
+      [SCHEMA_VERSION_5, migrateToV5],
     ];
 
     for (const [version, migrate] of chain) {
@@ -315,3 +323,42 @@ async function migrateToV4(client: PoolClient): Promise<void> {
   await client.query(`DELETE FROM resources WHERE kind = 'project'`);
   await client.query(`DELETE FROM resource_audit WHERE kind = 'project'`);
 }
+
+/**
+ * v5: the public visibility level is retired, so every row that still
+ * holds it is moved to org visibility — the same step as the sqlite
+ * driver's v10 (public-visibility-retired.ts says why a migration, why the
+ * kind table is frozen there, and why an undecodable row fails the step).
+ * The rows are read FOR UPDATE inside applyInTransaction's transaction,
+ * under the chain's advisory lock, so a second instance booting against
+ * the same database waits rather than moving a row twice; `updated_at` is
+ * bumped the way saveResource bumps it; rows that hold any other level are
+ * left byte-for-byte as they are. On the hosted edition every such row was
+ * moved through the API before this release deployed, so this step finds
+ * none and records its version.
+ */
+async function migrateToV5(client: PoolClient): Promise<void> {
+  for (const entry of PUBLIC_ROW_KINDS_AT_RETIREMENT) {
+    const result = await client.query<{ id: string; data: Buffer }>(
+      `SELECT id, data FROM resources WHERE kind = $1 FOR UPDATE`,
+      [entry.kind],
+    );
+    for (const row of result.rows) {
+      let moved: Uint8Array | undefined;
+      try {
+        moved = movePublicRowToOrg(entry, new Uint8Array(row.data));
+      } catch (error) {
+        throw new Error(
+          `${entry.kind} '${row.id}' cannot be moved off the retired public level: ${String(error)}`,
+          { cause: error },
+        );
+      }
+      if (moved !== undefined) {
+        await client.query(
+          `UPDATE resources SET data = $1, updated_at = now() WHERE kind = $2 AND id = $3`,
+          [Buffer.from(moved), entry.kind, row.id],
+        );
+      }
+    }
+  }
+}

package/src/store/postgres/store.ts

@@ -814,20 +814,13 @@ export class PostgresStore implements Store {
     // right side").
     const kindParam = `${push(query.kinds as string[])}::text[]`;
 
-    // The scope-filter fragments — same composition as the sqlite driver's
-    // port of Go's buildScopeFilter: org (strict or public-widened) and
-    // the independent public subtraction.
+    // The scope-filter fragments — same composition as the sqlite driver:
+    // the org filter, strict (visibility never widens or narrows the
+    // scope), then the authorized-id allowlist.
     const scopeClauses: string[] = [];
     if (query.orgFilter !== "") {
       const orgParam = push(query.orgFilter);
-      scopeClauses.push(
-        query.crossOrgPublic
-          ? `AND (org = ${orgParam} OR visibility = 'visibility_public')`
-          : `AND org = ${orgParam}`,
-      );
-    }
-    if (query.excludePublic) {
-      scopeClauses.push(`AND visibility != 'visibility_public'`);
+      scopeClauses.push(`AND org = ${orgParam}`);
     }
     if (query.authorizedIdsByKind !== undefined) {
       // The 20260830.01 scoping arm: per-kind resource_id allowlists.

package/src/store/public-visibility-retired.ts

@@ -0,0 +1,119 @@
+/**
+ * The one data migration both store drivers run when the public visibility
+ * level is retired: every stored row that holds `visibility_public` is moved
+ * to `visibility_org`, once, in place. The drivers own the SQL (which rows
+ * to read, how to write them back, the transaction); this module owns what
+ * is edition- and driver-neutral: WHICH kinds could hold the level and HOW
+ * one row's bytes are moved.
+ *
+ * Why a migration and not a sweep. The migration chain is the one place
+ * that runs exactly once per database, before any request is served. A
+ * public row nobody can set again would otherwise stay readable by every
+ * account on a self-hosted install forever, because the level's grant was
+ * derived from the row itself; on the hosted edition the rows are moved
+ * through the API before this release deploys (the tuple lifecycle is
+ * row-driven, and only that path deletes the grant's stored tuple), so the
+ * migration there finds none and is the safety net, never the mechanism.
+ *
+ * Why the kind table is frozen here and not read from the live kind
+ * configuration. A migration is a statement about the store AS IT WAS when
+ * the level was retired: the seven kinds whose VisibilityConfig declared
+ * `supports_public` at that moment are the only kinds a public row can
+ * exist under, and a later release that adds or removes a kind must not
+ * change what this step does. The list is spelled out with its schemas, the
+ * per-consumer kind-to-schema idiom the tree already uses; nothing here
+ * reads `kind_meta`. The `kind` strings are the enum NAMES the drivers'
+ * `kind` column holds (proto-fields.ts `apiResourceKindName`).
+ *
+ * Why decode and re-encode rather than patch bytes. `resources.data` is the
+ * marshaled proto message; the level is one enum field on the metadata
+ * message, and only a decode knows where it is. protobuf-es keeps unknown
+ * fields through `fromBinary` and writes them back in `toBinary`, so a row
+ * written by a newer or older release than this one loses nothing it
+ * carried; every other field is re-encoded from the same values. A row this
+ * step cannot decode is a fault that fails the whole transaction (the
+ * driver's `applyInTransaction` rolls back and the boot stops): skipping it
+ * would leave the one row the migration exists for readable by everyone,
+ * silently.
+ *
+ * `resource_audit` is deliberately untouched: it is the version history,
+ * and a snapshot that said "public" at the time is true history. The search
+ * index is left to boot's rebuild, which re-derives it from the rows.
+ */
+import type { DescMessage } from "@bufbuild/protobuf";
+import { fromBinary, toBinary } from "@bufbuild/protobuf";
+import { reflect } from "@bufbuild/protobuf/reflect";
+
+import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb";
+import { AgentInstanceSchema } from "@stigmer/protos/ai/stigmer/agentic/agentinstance/v1/api_pb";
+import { McpServerSchema } from "@stigmer/protos/ai/stigmer/agentic/mcpserver/v1/api_pb";
+import { PluginSchema } from "@stigmer/protos/ai/stigmer/agentic/plugin/v1/api_pb";
+import { SkillSchema } from "@stigmer/protos/ai/stigmer/agentic/skill/v1/api_pb";
+import { WorkflowSchema } from "@stigmer/protos/ai/stigmer/agentic/workflow/v1/api_pb";
+import { WorkflowInstanceSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowinstance/v1/api_pb";
+import { ApiResourceVisibility } from "@stigmer/protos/ai/stigmer/commons/apiresource/enum_pb";
+
+/** One kind that could hold the public level, with the schema its rows decode through. */
+export interface PublicRowKind {
+  /** The enum NAME the drivers' `kind` column holds. */
+  readonly kind: string;
+  readonly schema: DescMessage;
+}
+
+/**
+ * The seven kinds whose VisibilityConfig declared `supports_public` when the
+ * level was retired — frozen (the module header). Order is the order the
+ * drivers walk the kinds; it has no other meaning.
+ */
+export const PUBLIC_ROW_KINDS_AT_RETIREMENT: ReadonlyArray<PublicRowKind> = [
+  { kind: "agent", schema: AgentSchema },
+  { kind: "skill", schema: SkillSchema },
+  { kind: "mcp_server", schema: McpServerSchema },
+  { kind: "agent_instance", schema: AgentInstanceSchema },
+  { kind: "workflow", schema: WorkflowSchema },
+  { kind: "workflow_instance", schema: WorkflowInstanceSchema },
+  { kind: "plugin", schema: PluginSchema },
+];
+
+/** The level every moved row takes: the widest level that stays inside the owning organization. */
+export const LEVEL_AFTER_RETIREMENT = ApiResourceVisibility.visibility_org;
+
+/**
+ * The moved bytes of one row when it holds the public level, or undefined
+ * when it does not (the driver then leaves the row's bytes exactly as they
+ * are; a row with no metadata holds no level and is one of these). Throws
+ * when the bytes do not decode as the kind's message: a row this migration
+ * cannot read is a row it must not pass over (the module header).
+ */
+export function movePublicRowToOrg(
+  entry: PublicRowKind,
+  data: Uint8Array,
+): Uint8Array | undefined {
+  const message = fromBinary(entry.schema, data);
+  const root = reflect(entry.schema, message);
+  const metadataField = root.fields.find((field) => field.name === "metadata");
+  if (metadataField === undefined || metadataField.fieldKind !== "message") {
+    throw new Error(
+      `${entry.kind}: the schema declares no metadata message field`,
+    );
+  }
+  if (!root.isSet(metadataField)) {
+    return undefined;
+  }
+  const metadata = root.get(metadataField);
+  const visibilityField = metadata.fields.find(
+    (field) => field.name === "visibility",
+  );
+  if (visibilityField === undefined || visibilityField.fieldKind !== "enum") {
+    throw new Error(
+      `${entry.kind}: the metadata message declares no visibility enum field`,
+    );
+  }
+  if (
+    metadata.get(visibilityField) !== ApiResourceVisibility.visibility_public
+  ) {
+    return undefined;
+  }
+  metadata.set(visibilityField, LEVEL_AFTER_RETIREMENT);
+  return toBinary(entry.schema, message);
+}