@stigmer/server 3.38.3 → 3.39.0

package/src/store/organization-slug-history.ts

@@ -0,0 +1,159 @@
+/**
+ * The one data migration both store drivers run when the organization-slug
+ * ledger arrives: every slug an organization held before the ledger existed
+ * is recorded in it, once, before any request is served. The drivers own
+ * the SQL (which rows to read, in what pages, how to insert, the
+ * transaction); this module owns what is edition- and driver-neutral: WHICH
+ * kinds can name an organization and HOW one row's organization is read.
+ *
+ * What the step records. Two kinds of slug were taken before the ledger:
+ *
+ *   - every live organization's, copied from the organization rows by the
+ *     driver's SQL alone (an organization's id is its slug), unretired;
+ *   - every slug a surviving organization-scoped row still names in
+ *     `metadata.org` while no organization holds it, retired. Such a row
+ *     outlived an organization the store deleted before the ledger could
+ *     record it, and it is exactly what a new organization of that slug
+ *     would inherit: under the built-in authorizer a row's organization link
+ *     is derived from `metadata.org`, with no creation times compared.
+ *
+ * A slug whose organization left no row anywhere is not recorded, because
+ * nothing in the store remembers it; re-creating that slug inherits nothing.
+ *
+ * Why a migration and not a boot sweep. The migration chain is the one
+ * place that runs exactly once per database, before any request is served
+ * (public-visibility-retired.ts makes the same case). A sweep would need a
+ * marker, a fault posture, and either a window on its first boot or a boot
+ * that waits on it anyway.
+ *
+ * Why the kind table is frozen here and not read from the live kind
+ * configuration. The step is a statement about the store AS IT WAS when the
+ * ledger arrived: these are the kinds whose `kind_meta` declared
+ * AUTHORIZATION_SCOPE_TYPE_ORGANIZATION at that moment, and a later release
+ * that adds or removes a kind must not change what this step does. The list
+ * is spelled out with its schemas, the per-consumer kind-to-schema idiom the
+ * tree already uses. The `kind` strings are the enum NAMES the drivers'
+ * `kind` column holds (proto-fields.ts `apiResourceKindName`).
+ *
+ * Why an undecodable row fails the step. The row may name a deleted
+ * organization, and skipping it would leave that slug free for anyone to
+ * take along with everything the row grants. So the driver's transaction
+ * rolls back and the boot stops on the row this names, the rule
+ * public-visibility-retired.ts keeps for the same reason. A row with no
+ * metadata, or an empty organization, names no slug and is not a fault.
+ */
+import type { DescMessage } from "@bufbuild/protobuf";
+import { fromBinary } from "@bufbuild/protobuf";
+import { reflect } from "@bufbuild/protobuf/reflect";
+
+import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb";
+import { AgentChannelSchema } from "@stigmer/protos/ai/stigmer/agentic/agentchannel/v1/api_pb";
+import { AgentInstanceSchema } from "@stigmer/protos/ai/stigmer/agentic/agentinstance/v1/api_pb";
+import { AgentShareSchema } from "@stigmer/protos/ai/stigmer/agentic/agentshare/v1/api_pb";
+import { ArtifactSchema } from "@stigmer/protos/ai/stigmer/agentic/artifact/v1/api_pb";
+import { ChannelAppSchema } from "@stigmer/protos/ai/stigmer/agentic/channelapp/v1/api_pb";
+import { EnvironmentSchema } from "@stigmer/protos/ai/stigmer/agentic/environment/v1/api_pb";
+import { McpServerSchema } from "@stigmer/protos/ai/stigmer/agentic/mcpserver/v1/api_pb";
+import { MemorySchema } from "@stigmer/protos/ai/stigmer/agentic/memory/v1/api_pb";
+import { PluginSchema } from "@stigmer/protos/ai/stigmer/agentic/plugin/v1/api_pb";
+import { ScheduleSchema } from "@stigmer/protos/ai/stigmer/agentic/schedule/v1/api_pb";
+import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+import { SkillSchema } from "@stigmer/protos/ai/stigmer/agentic/skill/v1/api_pb";
+import { WorkflowSchema } from "@stigmer/protos/ai/stigmer/agentic/workflow/v1/api_pb";
+import { WorkflowExecutionSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowexecution/v1/api_pb";
+import { WorkflowInstanceSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowinstance/v1/api_pb";
+import { SubscriptionSchema } from "@stigmer/protos/ai/stigmer/billing/subscription/v1/api_pb";
+import { IamPolicySchema } from "@stigmer/protos/ai/stigmer/iam/iampolicy/v1/api_pb";
+import { IdentityProviderSchema } from "@stigmer/protos/ai/stigmer/iam/identityprovider/v1/api_pb";
+import { InvitationSchema } from "@stigmer/protos/ai/stigmer/iam/invitation/v1/api_pb";
+import { OAuthAppSchema } from "@stigmer/protos/ai/stigmer/iam/oauthapp/v1/api_pb";
+import { PlatformClientSchema } from "@stigmer/protos/ai/stigmer/iam/platformclient/v1/api_pb";
+import { TeamSchema } from "@stigmer/protos/ai/stigmer/iam/team/v1/api_pb";
+
+/** One kind whose rows name their organization, with the schema its rows decode through. */
+export interface OrganizationScopedKind {
+  /** The enum NAME the drivers' `kind` column holds. */
+  readonly kind: string;
+  readonly schema: DescMessage;
+}
+
+/**
+ * The organization-scoped kinds when the ledger arrived, in the order
+ * `api_resource_kind.proto` declares them. Frozen: see the module header.
+ */
+export const ORGANIZATION_SCOPED_KINDS_AT_LEDGER: ReadonlyArray<OrganizationScopedKind> =
+  [
+    { kind: "iam_policy", schema: IamPolicySchema },
+    { kind: "invitation", schema: InvitationSchema },
+    { kind: "identity_provider", schema: IdentityProviderSchema },
+    { kind: "oauth_app", schema: OAuthAppSchema },
+    { kind: "platform_client", schema: PlatformClientSchema },
+    { kind: "team", schema: TeamSchema },
+    { kind: "agent", schema: AgentSchema },
+    { kind: "session", schema: SessionSchema },
+    { kind: "skill", schema: SkillSchema },
+    { kind: "mcp_server", schema: McpServerSchema },
+    { kind: "agent_instance", schema: AgentInstanceSchema },
+    { kind: "agent_share", schema: AgentShareSchema },
+    { kind: "agent_channel", schema: AgentChannelSchema },
+    { kind: "channel_app", schema: ChannelAppSchema },
+    { kind: "workflow", schema: WorkflowSchema },
+    { kind: "workflow_instance", schema: WorkflowInstanceSchema },
+    { kind: "workflow_execution", schema: WorkflowExecutionSchema },
+    { kind: "environment", schema: EnvironmentSchema },
+    { kind: "artifact", schema: ArtifactSchema },
+    { kind: "schedule", schema: ScheduleSchema },
+    { kind: "memory", schema: MemorySchema },
+    { kind: "plugin", schema: PluginSchema },
+    { kind: "subscription", schema: SubscriptionSchema },
+  ];
+
+/** How many rows of a kind the step decodes per page, so no kind is ever read whole into memory. */
+export const HISTORY_PAGE_SIZE = 500;
+
+/**
+ * The organization a stored row names in `metadata.org`, or "" when it names
+ * none (no metadata, or an empty organization). Throws when the bytes do not
+ * decode through the kind's schema, or when the schema lacks the metadata
+ * shape every resource shares: both fail the step (see the header).
+ */
+export function organizationNamedBy(
+  entry: OrganizationScopedKind,
+  data: Uint8Array,
+): string {
+  const message = fromBinary(entry.schema, data);
+  const root = reflect(entry.schema, message);
+  const metadataField = root.fields.find((field) => field.name === "metadata");
+  if (metadataField === undefined || metadataField.fieldKind !== "message") {
+    throw new Error(
+      `${entry.kind}: the schema declares no metadata message field`,
+    );
+  }
+  if (!root.isSet(metadataField)) {
+    return "";
+  }
+  const metadata = root.get(metadataField);
+  const orgField = metadata.fields.find((field) => field.name === "org");
+  if (orgField === undefined || orgField.fieldKind !== "scalar") {
+    throw new Error(
+      `${entry.kind}: the metadata message declares no org field`,
+    );
+  }
+  const org = metadata.get(orgField);
+  return typeof org === "string" ? org : "";
+}
+
+/**
+ * The error a driver throws for a row that does not decode, naming the row,
+ * so both drivers stop the boot with the same sentence.
+ */
+export function undecodableRowError(
+  entry: OrganizationScopedKind,
+  id: string,
+  cause: unknown,
+): Error {
+  return new Error(
+    `${entry.kind} '${id}' cannot be read for the organization it names: ${String(cause)}`,
+    { cause },
+  );
+}

package/src/store/postgres/__tests__/migrations.test.ts

@@ -8,8 +8,12 @@
  * moves every row of the seven kinds that held the retired public level to
  * org and leaves every other row's bytes as they were; a row it cannot
  * decode fails the step and leaves the database at v4. v6 adds the list
- * index's table (its behaviour is the store contract's). A step's starting
- * database is built by running the chain up to the step before it.
+ * index's table (its behaviour is the store contract's). v7 creates the
+ * organization-slug ledger and fills it: every live organization unretired,
+ * every organization a surviving scoped row names with none live retired,
+ * across keyset pages; a scoped row it cannot decode fails the step and
+ * leaves the database at v6 with no ledger. A step's starting database is
+ * built by running the chain up to the step before it.
  *
  * Gated on TEST_DATABASE_URL (see support.ts): visible skips without a
  * database, always exercised in CI via the ci.stigmer-server service
@@ -25,11 +29,15 @@ import type { DescMessage } from "@bufbuild/protobuf";
 import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb";
 import { ApiResourceVisibility } from "@stigmer/protos/ai/stigmer/commons/apiresource/enum_pb";
 
+import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+
+import { HISTORY_PAGE_SIZE } from "../../organization-slug-history.js";
 import { PUBLIC_ROW_KINDS_AT_RETIREMENT } from "../../public-visibility-retired.js";
 import {
   CURRENT_SCHEMA_VERSION,
   SCHEMA_VERSION_1,
   SCHEMA_VERSION_5,
+  SCHEMA_VERSION_7,
   runMigrations,
 } from "../migrations.js";
 import { PostgresStore } from "../store.js";
@@ -88,6 +96,7 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
         expect(names).toEqual([
           "bootstrap_state",
           "oauth_grant",
+          "organization_slugs",
           "pending_oauth_state",
           "resource_audit",
           "resource_list_keys",
@@ -379,5 +388,132 @@ describe.skipIf(testDatabaseAdminUrl() === undefined)(
         }
       });
     });
+    describe("v7: the organization-slug ledger records every slug taken before it", () => {
+      /** An agent row naming `org`, encoded through its own schema. */
+      function agentNaming(id: string, org: string): Buffer {
+        return Buffer.from(
+          toBinary(
+            AgentSchema,
+            create(AgentSchema, {
+              metadata: { id, name: id, slug: id, org },
+              spec: { instructions: "a conformant instruction body" },
+            }),
+          ),
+        );
+      }
+
+      function sessionNaming(id: string, org: string): Buffer {
+        return Buffer.from(
+          toBinary(
+            SessionSchema,
+            create(SessionSchema, {
+              metadata: { id, name: id, slug: id, org },
+            }),
+          ),
+        );
+      }
+
+      /** A v6 database, the one v7 starts from, and a client on it for the seeding. */
+      async function v6Client(): Promise<pg.Client> {
+        await migrateTo(db.databaseUrl, SCHEMA_VERSION_7 - 1);
+        const client = new pg.Client({ connectionString: db.databaseUrl });
+        await client.connect();
+        return client;
+      }
+
+      async function ledger(
+        client: pg.Client,
+      ): Promise<Array<{ slug: string; retired: boolean }>> {
+        const result = await client.query<{ slug: string; retired: boolean }>(
+          `SELECT slug, retired_at IS NOT NULL AS retired FROM organization_slugs ORDER BY slug`,
+        );
+        return result.rows;
+      }
+
+      it("records every live organization unretired and every organization a surviving row names with none live retired", async () => {
+        const client = await v6Client();
+        try {
+          await client.query(
+            `INSERT INTO resources (kind, id, data) VALUES
+               ('organization', 'acme', '\\x00'),
+               ('agent', 'agt_live', $1),
+               ('agent', 'agt_gone', $2),
+               ('session', 'ses_gone', $3),
+               ('agent', 'agt_orphan', $4),
+               ('identity_account', 'ida_opaque', '\\xffff')`,
+            [
+              agentNaming("agt_live", "acme"),
+              agentNaming("agt_gone", "deleted-one"),
+              sessionNaming("ses_gone", "deleted-two"),
+              agentNaming("agt_orphan", ""),
+            ],
+          );
+
+          const reopened = await PostgresStore.open(db.databaseUrl);
+          await reopened.close();
+
+          expect(await ledger(client)).toEqual([
+            { slug: "acme", retired: false },
+            { slug: "deleted-one", retired: true },
+            { slug: "deleted-two", retired: true },
+          ]);
+        } finally {
+          await client.end();
+        }
+      });
+
+      it("reads a kind across keyset pages, missing no row past the first page", async () => {
+        const client = await v6Client();
+        try {
+          const rows = HISTORY_PAGE_SIZE + 2;
+          for (let i = 0; i < rows; i++) {
+            const id = `agt_${String(i).padStart(4, "0")}`;
+            // The last row, past the first page, is the only one naming
+            // its organization.
+            const org = i === rows - 1 ? "past-the-page" : "on-the-page";
+            await client.query(
+              `INSERT INTO resources (kind, id, data) VALUES ('agent', $1, $2)`,
+              [id, agentNaming(id, org)],
+            );
+          }
+
+          const reopened = await PostgresStore.open(db.databaseUrl);
+          await reopened.close();
+
+          expect(await ledger(client)).toEqual([
+            { slug: "on-the-page", retired: true },
+            { slug: "past-the-page", retired: true },
+          ]);
+        } finally {
+          await client.end();
+        }
+      });
+
+      it("a scoped row that does not decode fails the step, names the row, and leaves the database at v6 with no ledger", async () => {
+        const client = await v6Client();
+        try {
+          await client.query(
+            `INSERT INTO resources (kind, id, data) VALUES ('agent', 'agt_broken', '\\xffffff')`,
+          );
+
+          await expect(PostgresStore.open(db.databaseUrl)).rejects.toThrow(
+            /migrate to v7: .*agent 'agt_broken' cannot be read for the organization it names/,
+          );
+
+          const version = await client.query(
+            `SELECT COALESCE(MAX(version), 0) AS version FROM schema_version`,
+          );
+          expect(Number((version.rows[0] as { version: string }).version)).toBe(
+            SCHEMA_VERSION_7 - 1,
+          );
+          const table = await client.query(
+            `SELECT to_regclass('organization_slugs') AS name`,
+          );
+          expect((table.rows[0] as { name: string | null }).name).toBeNull();
+        } finally {
+          await client.end();
+        }
+      });
+    });
   },
 );

package/src/store/postgres/__tests__/store-contract.test.ts

@@ -38,6 +38,7 @@ const ALL_TABLES = [
   "workflow_execution_events",
   "schedule_runs",
   "signal_dedupe",
+  "organization_slugs",
   "oauth_grant",
   "pending_oauth_state",
 ] as const;

package/src/store/postgres/migrations.ts

@@ -38,6 +38,12 @@
  */
 import type { PoolClient } from "pg";
 
+import {
+  HISTORY_PAGE_SIZE,
+  ORGANIZATION_SCOPED_KINDS_AT_LEDGER,
+  organizationNamedBy,
+  undecodableRowError,
+} from "../organization-slug-history.js";
 import {
   PUBLIC_ROW_KINDS_AT_RETIREMENT,
   movePublicRowToOrg,
@@ -52,9 +58,11 @@ export const SCHEMA_VERSION_4 = 4;
 export const SCHEMA_VERSION_5 = 5;
 /** v6: the list index's columns, key table and indexes (DDL only). */
 export const SCHEMA_VERSION_6 = 6;
+/** v7: the organization-slug ledger, filled with every slug taken before it. */
+export const SCHEMA_VERSION_7 = 7;
 
 /** Target version for new databases. */
-export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_6;
+export const CURRENT_SCHEMA_VERSION = SCHEMA_VERSION_7;
 
 /**
  * Advisory lock key for the migration chain. Arbitrary but stable 64-bit
@@ -95,6 +103,7 @@ export async function runMigrations(
       [SCHEMA_VERSION_4, migrateToV4],
       [SCHEMA_VERSION_5, migrateToV5],
       [SCHEMA_VERSION_6, migrateToV6],
+      [SCHEMA_VERSION_7, migrateToV7],
     ];
 
     for (const [version, migrate] of chain) {
@@ -424,3 +433,77 @@ async function migrateToV6(client: PoolClient): Promise<void> {
       ON resource_list_keys (kind, key, value, created_at, (id COLLATE "C"));
   `);
 }
+
+/**
+ * v7: the organization-slug ledger (the Store's `organizationSlugs`,
+ * interface.ts says what it guarantees), created and filled in one step so
+ * it is complete before the first request. organization-slug-history.ts
+ * says what the fill records, why it is a migration, why its kind table is
+ * frozen, and why an undecodable row fails the step; this step owns the SQL.
+ *
+ * - `claimed_at` and `retired_at` are ledger time crossing the Store
+ *   interface, so TEXT holding the exact RFC-3339 strings (the header's
+ *   convention); `retired_at` is NULL while the slug is held.
+ * - Every query is by the primary key, so the table needs no other index.
+ * - Live organizations are copied by id (an organization's id is its slug)
+ *   with no decode. Each organization-scoped kind is then read in keyset
+ *   pages on `(kind, id)`, the primary key's order, so no kind is ever held
+ *   in memory whole; the organizations its rows name with no live
+ *   organization are recorded retired. ON CONFLICT DO NOTHING keeps a live
+ *   organization's entry unretired whatever its rows say.
+ * - The rows are only read, so nothing takes a row lock; the chain's
+ *   advisory lock already keeps a second instance's boot out of the step.
+ */
+async function migrateToV7(client: PoolClient): Promise<void> {
+  await client.query(`
+    CREATE TABLE organization_slugs (
+      slug TEXT PRIMARY KEY,
+      claimed_at TEXT NOT NULL,
+      retired_at TEXT
+    )
+  `);
+  const recordedAt = new Date().toISOString();
+  await client.query(
+    `INSERT INTO organization_slugs (slug, claimed_at)
+     SELECT id, $1 FROM resources WHERE kind = 'organization'
+     ON CONFLICT (slug) DO NOTHING`,
+    [recordedAt],
+  );
+
+  const named = new Set<string>();
+  for (const entry of ORGANIZATION_SCOPED_KINDS_AT_LEDGER) {
+    let after = "";
+    for (;;) {
+      const page = await client.query<{ id: string; data: Buffer }>(
+        `SELECT id, data FROM resources
+         WHERE kind = $1 AND id > $2
+         ORDER BY id
+         LIMIT $3`,
+        [entry.kind, after, HISTORY_PAGE_SIZE],
+      );
+      for (const row of page.rows) {
+        let org: string;
+        try {
+          org = organizationNamedBy(entry, new Uint8Array(row.data));
+        } catch (error) {
+          throw undecodableRowError(entry, row.id, error);
+        }
+        if (org !== "") {
+          named.add(org);
+        }
+      }
+      if (page.rows.length < HISTORY_PAGE_SIZE) {
+        break;
+      }
+      after = page.rows[page.rows.length - 1]!.id;
+    }
+  }
+  for (const slug of named) {
+    await client.query(
+      `INSERT INTO organization_slugs (slug, claimed_at, retired_at)
+       VALUES ($1, $2, $2)
+       ON CONFLICT (slug) DO NOTHING`,
+      [slug, recordedAt],
+    );
+  }
+}

package/src/store/postgres/store.ts

@@ -52,6 +52,9 @@ import type {
   ClaimResult,
   OAuthGrant,
   OAuthGrantStore,
+  OrganizationSlugClaim,
+  OrganizationSlugEntry,
+  OrganizationSlugStore,
   PendingOAuthState,
   PendingOAuthStateStore,
   RawResourceDocument,
@@ -146,6 +149,7 @@ const RECONCILE_BATCH = 500;
 export class PostgresStore implements Store {
   readonly bootstrapState: BootstrapStateStore;
   readonly signalDedupe: SignalDedupeStore;
+  readonly organizationSlugs: OrganizationSlugStore;
   readonly oauthGrants: OAuthGrantStore;
   readonly pendingOAuthStates: PendingOAuthStateStore;
 
@@ -166,6 +170,9 @@ export class PostgresStore implements Store {
       () => this.open(),
       logger,
     );
+    this.organizationSlugs = new PostgresOrganizationSlugStore(() =>
+      this.open(),
+    );
     this.oauthGrants = new PostgresOAuthGrantStore(() => this.open());
     this.pendingOAuthStates = new PostgresPendingOAuthStateStore(() =>
       this.open(),
@@ -1497,6 +1504,83 @@ function buildDedupeKey(org: string, idempotencyKey: string): string {
   return `${org}:${idempotencyKey}`;
 }
 
+// =============================================================================
+// Organization slugs
+// =============================================================================
+
+/** A ledger row as the driver reads it (`retired_at` NULL while held). */
+interface OrganizationSlugRow {
+  slug: string;
+  claimed_at: string;
+  retired_at: string | null;
+}
+
+function organizationSlugEntryOf(
+  row: OrganizationSlugRow,
+): OrganizationSlugEntry {
+  return {
+    slug: row.slug,
+    claimedAt: row.claimed_at,
+    retiredAt: row.retired_at ?? "",
+  };
+}
+
+class PostgresOrganizationSlugStore implements OrganizationSlugStore {
+  constructor(private readonly open: () => Pool) {}
+
+  async claim(slug: string): Promise<OrganizationSlugClaim> {
+    const pool = this.open();
+    const claimedAt = new Date().toISOString();
+    // ON CONFLICT DO NOTHING: the primary key picks the one winner of
+    // concurrent claims, with no error-text sniffing.
+    const inserted = await pool.query(
+      `INSERT INTO organization_slugs (slug, claimed_at) VALUES ($1, $2)
+       ON CONFLICT (slug) DO NOTHING`,
+      [slug, claimedAt],
+    );
+    if ((inserted.rowCount ?? 0) === 1) {
+      return { claimed: true, entry: { slug, claimedAt, retiredAt: "" } };
+    }
+    const holder = await this.find(slug);
+    if (holder === undefined) {
+      // A holder only disappears through its own create's release, so a
+      // lost claim with no holder is a claim that raced that release:
+      // surfaced as a fault rather than fabricating an entry.
+      throw new Error(
+        `organization slug '${slug}': its holder disappeared during the claim`,
+      );
+    }
+    return { claimed: false, entry: holder };
+  }
+
+  async retire(slug: string): Promise<void> {
+    const retiredAt = new Date().toISOString();
+    await this.open().query(
+      `INSERT INTO organization_slugs (slug, claimed_at, retired_at) VALUES ($1, $2, $2)
+       ON CONFLICT (slug) DO UPDATE SET retired_at = excluded.retired_at
+       WHERE organization_slugs.retired_at IS NULL`,
+      [slug, retiredAt],
+    );
+  }
+
+  async release(entry: OrganizationSlugEntry): Promise<void> {
+    await this.open().query(
+      `DELETE FROM organization_slugs
+       WHERE slug = $1 AND claimed_at = $2 AND retired_at IS NULL`,
+      [entry.slug, entry.claimedAt],
+    );
+  }
+
+  async find(slug: string): Promise<OrganizationSlugEntry | undefined> {
+    const result = await this.open().query<OrganizationSlugRow>(
+      `SELECT slug, claimed_at, retired_at FROM organization_slugs WHERE slug = $1`,
+      [slug],
+    );
+    const row = result.rows[0];
+    return row === undefined ? undefined : organizationSlugEntryOf(row);
+  }
+}
+
 // =============================================================================
 // MCP OAuth grants
 // =============================================================================