@stigmer/server 3.40.1 → 3.41.0

package/src/extensions/__tests__/composed-services-composed.test.ts

@@ -0,0 +1,236 @@
+/**
+ * Pins the two unit points and the edition's sign-in rule on real boots
+ * (extensions/composed-services.ts, boot/compose.ts):
+ *
+ *   - `onComposed` hands every unit the composition's own instances before
+ *     `composeServer` returns: the very Authorizer a unit registered, its
+ *     list scope, its engine's check and its lifecycle, and the one
+ *     in-process transport; under open source's own authorization, the
+ *     built-in Authorizer, list scope and policy check.
+ *   - `start` runs in unit order, first in `start()`, before the server
+ *     reports SERVING; a failing start fails `start()`, naming its unit.
+ *   - An edition above open source with no sign-in is refused at boot,
+ *     whichever authorizer it runs.
+ */
+import { generateKeyPairSync } from "node:crypto";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+import { HealthCheckResponse_ServingStatus } from "@stigmer/protos/grpc/health/v1/health_pb";
+import { ServerEdition } from "@stigmer/protos/ai/stigmer/platform/v1/server_info_pb";
+
+import { loadConfig } from "../../boot/config.js";
+import { composeServer } from "../../boot/compose.js";
+import type { ComposedServer } from "../../boot/compose.js";
+import { createLogger } from "../../boot/logger.js";
+import { newPermissiveSingleTeamAuthorizer } from "../../pipeline/steps/authorize.js";
+import { platformTokenKeyRingFromPem } from "../../platformtoken/key-ring.js";
+import type { AuthorizationQueryEngine } from "../authorization-queries.js";
+import type { ComposedServices } from "../composed-services.js";
+import type { IdentityVerifier } from "../identity.js";
+import type { ListReadScope } from "../list-read-scope.js";
+import type { ServerExtension } from "../registry.js";
+import type { ResourceAuthorizationLifecycle } from "../resource-authorization.js";
+import type { ResourceRowReader } from "../resource-row-reader.js";
+
+const silentLogger = createLogger({
+  level: "error",
+  pretty: false,
+  write: () => {},
+});
+
+const verifier: IdentityVerifier = {
+  name: "fixture-verifier",
+  verify: () => Promise.resolve(null),
+};
+const ringKeys = generateKeyPairSync("rsa", { modulusLength: 2048 });
+const RING = platformTokenKeyRingFromPem({
+  privateKeyPem: ringKeys.privateKey
+    .export({ format: "pem", type: "pkcs8" })
+    .toString(),
+  publicKeyPems: [
+    ringKeys.publicKey.export({ format: "pem", type: "spki" }).toString(),
+  ],
+});
+const noRows: ResourceRowReader = {
+  findById: () => Promise.resolve(undefined),
+};
+
+/** An Enterprise unit on open source's own authorization: sign-in, a key ring, a reader per kind. */
+function enterpriseUnit(extra: Partial<ServerExtension> = {}): ServerExtension {
+  return {
+    name: "enterprise-fixture",
+    edition: ServerEdition.enterprise,
+    requireAuthentication: true,
+    identityVerifiers: [verifier],
+    drivers: {
+      platformTokenKeys: RING,
+      resourceRowReaders: new Map([
+        [ApiResourceKind.identity_provider, noRows],
+        [ApiResourceKind.invitation, noRows],
+        [ApiResourceKind.team, noRows],
+      ]),
+    },
+    ...extra,
+  };
+}
+
+const dirs: string[] = [];
+const servers: ComposedServer[] = [];
+
+async function compose(
+  extensions: ReadonlyArray<ServerExtension>,
+): Promise<ComposedServer> {
+  const dir = mkdtempSync(path.join(tmpdir(), "composed-services-"));
+  dirs.push(dir);
+  const server = await composeServer({
+    config: loadConfig({
+      STIGMER_MODEL_REGISTRY_REFRESH: "off",
+      DB_PATH: path.join(dir, "stigmer.db"),
+      ARTIFACT_LOCAL_BASE_PATH: path.join(dir, "artifacts"),
+    }),
+    logger: silentLogger,
+    extensions: [...extensions],
+    portOverride: 0,
+    host: "127.0.0.1",
+  });
+  servers.push(server);
+  return server;
+}
+
+afterEach(async () => {
+  for (const server of servers.splice(0)) {
+    await server.shutdown();
+  }
+  for (const dir of dirs.splice(0)) {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+describe("onComposed: a unit receives the composition's own instances", () => {
+  it("hands a unit the very Authorizer, list scope, check and lifecycle a unit registered, and the one in-process transport, before composeServer returns", async () => {
+    const authorizer = newPermissiveSingleTeamAuthorizer();
+    const listReadScope: ListReadScope = {
+      authorizedResourceIds: () => Promise.resolve(new Set()),
+      restrictListEntries: () => Promise.resolve(new Set()),
+    };
+    const queries: AuthorizationQueryEngine = {
+      check: () => Promise.resolve(true),
+      listResourceIds: () => Promise.resolve([]),
+      listPrincipalIds: () => Promise.resolve([]),
+    };
+    const lifecycle: ResourceAuthorizationLifecycle = {
+      onResourceCreated: () => Promise.resolve(),
+      onResourceDeleted: () => Promise.resolve(),
+      onVisibilityChanged: () => Promise.resolve(),
+    };
+    let received: ComposedServices | undefined;
+    const server = await compose([
+      {
+        name: "cloud-shaped",
+        edition: ServerEdition.cloud,
+        requireAuthentication: true,
+        identityVerifiers: [verifier],
+        authorizer,
+        drivers: {
+          platformTokenKeys: RING,
+          listReadScope,
+          authorizationQueries: queries,
+          resourceAuthorizationLifecycle: lifecycle,
+        },
+      },
+      {
+        name: "consumer",
+        onComposed: (composed) => {
+          received = composed;
+        },
+      },
+    ]);
+    expect(received).toBeDefined();
+    expect(received?.authorizer).toBe(authorizer);
+    expect(received?.listReadScope).toBe(listReadScope);
+    expect(received?.authorizationQueries).toBe(queries);
+    expect(received?.resourceAuthorizationLifecycle).toBe(lifecycle);
+    expect(received?.inProcessTransport).toBe(server.inProcessTransport);
+  });
+
+  it("under open source's own authorization hands the built-in Authorizer, list scope and policy check, and no lifecycle", async () => {
+    let received: ComposedServices | undefined;
+    await compose([
+      enterpriseUnit({
+        onComposed: (composed) => {
+          received = composed;
+        },
+      }),
+    ]);
+    expect(received?.authorizer).toBeDefined();
+    expect(received?.listReadScope).toBeDefined();
+    expect(received?.authorizationQueries).toBeDefined();
+    expect(received?.resourceAuthorizationLifecycle).toBeUndefined();
+  });
+});
+
+describe("start: a unit's boot work runs before anything serves", () => {
+  it("runs each unit's start in unit order, before the server reports SERVING", async () => {
+    const seen: string[] = [];
+    let server: ComposedServer | undefined;
+    const observe = (unit: string) => (): Promise<void> => {
+      seen.push(
+        `${unit}:${HealthCheckResponse_ServingStatus[server?.healthState.status("") ?? HealthCheckResponse_ServingStatus.UNKNOWN]}`,
+      );
+      return Promise.resolve();
+    };
+    server = await compose([
+      enterpriseUnit({ start: observe("first") }),
+      { name: "second", start: observe("second") },
+    ]);
+    await server.start();
+    expect(seen).toEqual(["first:NOT_SERVING", "second:NOT_SERVING"]);
+    expect(server.healthState.status("")).toBe(
+      HealthCheckResponse_ServingStatus.SERVING,
+    );
+  });
+
+  it("a failing start fails the server's start, naming its unit, and the server never reports SERVING", async () => {
+    const server = await compose([
+      enterpriseUnit({
+        start: () => Promise.reject(new Error("repair failed")),
+      }),
+    ]);
+    await expect(server.start()).rejects.toThrowError(
+      "extension 'enterprise-fixture' failed to start: repair failed",
+    );
+    expect(server.healthState.status("")).toBe(
+      HealthCheckResponse_ServingStatus.NOT_SERVING,
+    );
+  });
+});
+
+describe("an edition above open source signs its callers in", () => {
+  it.each([
+    ["on open source's own authorizer", {}],
+    [
+      "with its own Authorizer",
+      { authorizer: newPermissiveSingleTeamAuthorizer() },
+    ],
+  ])(
+    "refuses to boot trusted-local %s",
+    async (_label, extra: Partial<ServerExtension>) => {
+      await expect(
+        compose([
+          {
+            name: "signed-out-enterprise",
+            edition: ServerEdition.enterprise,
+            ...extra,
+          },
+        ]),
+      ).rejects.toThrowError(
+        "the composition serves edition 'enterprise' without the require-authentication posture — an edition above open source must sign its callers in: declare requireAuthentication on a unit that registers a verifier, or configure STIGMER_OIDC_ISSUER",
+      );
+    },
+  );
+});

package/src/extensions/__tests__/extension-composition.test.ts

@@ -17,11 +17,12 @@
  * to before the parameter existed — is pinned where it belongs: the
  * platform domain suite (edition oss) and the four conformance rosters.
  */
+import { generateKeyPairSync } from "node:crypto";
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 
-import { create } from "@bufbuild/protobuf";
+import { clone, create } from "@bufbuild/protobuf";
 import type { DescMessage } from "@bufbuild/protobuf";
 import { Code, ConnectError, createClient } from "@connectrpc/connect";
 import type { Transport } from "@connectrpc/connect";
@@ -38,6 +39,12 @@ import {
   ExecutionPhase,
 } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
+import { WorkflowSchema } from "@stigmer/protos/ai/stigmer/agentic/workflow/v1/api_pb";
+import { WorkflowCommandController } from "@stigmer/protos/ai/stigmer/agentic/workflow/v1/command_pb";
+import { WorkflowInstanceSchema } from "@stigmer/protos/ai/stigmer/agentic/workflowinstance/v1/api_pb";
+import { WorkflowInstanceCommandController } from "@stigmer/protos/ai/stigmer/agentic/workflowinstance/v1/command_pb";
+import { WorkflowInstanceQueryController } from "@stigmer/protos/ai/stigmer/agentic/workflowinstance/v1/query_pb";
+import { WorkflowExecutionVisibility } from "@stigmer/protos/ai/stigmer/agentic/workflowinstance/v1/spec_pb";
 import { SessionCommandController } from "@stigmer/protos/ai/stigmer/agentic/session/v1/command_pb";
 import { SessionQueryController } from "@stigmer/protos/ai/stigmer/agentic/session/v1/query_pb";
 import { BillingQueryController } from "@stigmer/protos/ai/stigmer/billing/v1/query_pb";
@@ -66,10 +73,13 @@ import type { ComposedServer } from "../../boot/compose.js";
 import { createLogger } from "../../boot/logger.js";
 import { AUTHENTICATION_TOKEN_MISSING_MESSAGE } from "../../pipeline/interceptors/auth.js";
 import type { PipelineStep } from "../../pipeline/pipeline.js";
+import { newPermissiveSingleTeamAuthorizer } from "../../pipeline/steps/authorize.js";
+import { platformTokenKeyRingFromPem } from "../../platformtoken/key-ring.js";
 import type { AuthzCheck } from "../authorizer.js";
 import type { GateSlotName } from "../gate-slots.js";
 import type { OrganizationDirectory } from "../organization-directory.js";
 import type {
+  ExecutionVisibilityChangedEvent,
   ResourceAuthorizationLifecycle,
   ResourceCreatedEvent,
   ResourceDeletedEvent,
@@ -77,6 +87,9 @@ import type {
 } from "../resource-authorization.js";
 import type { AgentExecutionStatusTransition } from "../status-hooks.js";
 import type { ServerExtension } from "../registry.js";
+import type { IdentityVerifier } from "../identity.js";
+import type { ResourceRowReader } from "../resource-row-reader.js";
+import { seedOrganizations } from "../../domain/organization/__tests__/support.js";
 
 /** The refusal a call answered; a call that succeeds fails the case. */
 async function refusalOf(work: Promise<unknown>): Promise<ConnectError> {
@@ -91,6 +104,50 @@ async function refusalOf(work: Promise<unknown>): Promise<ConnectError> {
 const BILLING_PROCEDURE =
   "/ai.stigmer.billing.v1.BillingQueryController/getBillingAccount";
 
+/**
+ * What a composition serving an edition above open source must bring: its
+ * callers sign in (compose.ts refuses such an edition trusted-local), and
+ * it supplies its own platform-token key ring (platformtoken/key-ring.ts).
+ * One verifier claims the fixture token; the ring is generated per run.
+ */
+const EDITION_CALLER_TOKEN = "edition-fixture-caller";
+const editionVerifier: IdentityVerifier = {
+  name: "edition-fixture-verifier",
+  verify: (token) =>
+    Promise.resolve(
+      token === EDITION_CALLER_TOKEN
+        ? {
+            identityId: "ida_edition_fixture",
+            callerClass: "user",
+            issuer: "edition-fixture",
+            rawToken: token,
+          }
+        : null,
+    ),
+};
+const editionKeys = generateKeyPairSync("rsa", { modulusLength: 2048 });
+const EDITION_RING = platformTokenKeyRingFromPem({
+  privateKeyPem: editionKeys.privateKey
+    .export({ format: "pem", type: "pkcs8" })
+    .toString(),
+  publicKeyPems: [
+    editionKeys.publicKey.export({ format: "pem", type: "spki" }).toString(),
+  ],
+});
+
+/** A client transport on the bound port presenting the fixture token. */
+function editionCallerTransport(port: number): Transport {
+  return createGrpcTransport({
+    baseUrl: `http://127.0.0.1:${port}`,
+    interceptors: [
+      (next) => (request) => {
+        request.header.set("authorization", `Bearer ${EDITION_CALLER_TOKEN}`);
+        return next(request);
+      },
+    ],
+  });
+}
+
 describe("extension composition (composed server)", () => {
   let server: ComposedServer;
   let dir: string;
@@ -98,9 +155,16 @@ describe("extension composition (composed server)", () => {
   // Captured NDJSON log lines — the interceptor-chain proof reads them.
   const logLines: string[] = [];
 
+  // A cloud-shaped unit: the cloud edition, its own sign-in, its own
+  // Authorizer (a permissive one; this suite pins service visibility and
+  // chain traversal, not authorization) and its own key ring.
   const fakeBillingExtension: ServerExtension = {
     name: "fake-billing",
     edition: ServerEdition.cloud,
+    requireAuthentication: true,
+    identityVerifiers: [editionVerifier],
+    authorizer: newPermissiveSingleTeamAuthorizer(),
+    drivers: { platformTokenKeys: EDITION_RING },
     services: [
       (router): void => {
         // Partial implementation is deliberate: the fake pins service
@@ -132,9 +196,7 @@ describe("extension composition (composed server)", () => {
       host: "127.0.0.1",
     });
     const port = await server.start();
-    portTransport = createGrpcTransport({
-      baseUrl: `http://127.0.0.1:${port}`,
-    });
+    portTransport = editionCallerTransport(port);
   });
 
   afterAll(async () => {
@@ -194,21 +256,35 @@ describe("extension composition (composed server)", () => {
 });
 
 /**
- * The third edition on the wire (editions program, sp.edition-contract):
- * a unit declaring ServerEdition.enterprise is answered verbatim by
- * getServerInfo. The registry is generic over concrete editions, so this
- * pins the end-to-end path (declaration → resolved registry → platform
- * controller → wire) for the value no composition emits yet — the day
- * P4's Enterprise composition declares it, the wire is already proven.
+ * The third edition on the wire, on open source's own authorizer: a unit
+ * declaring ServerEdition.enterprise, signing its callers in and
+ * registering a row reader for each kind the edition serves beyond open
+ * source, boots with no Authorizer of its own and is answered verbatim by
+ * getServerInfo. It pins the end-to-end path (declaration → resolved
+ * registry → the built-in posture's row rule → platform controller →
+ * wire) a self-hosted Enterprise composition takes.
  */
 describe("extension composition (enterprise edition on the wire)", () => {
   let server: ComposedServer;
   let dir: string;
   let portTransport: Transport;
 
+  const noRows: ResourceRowReader = {
+    findById: () => Promise.resolve(undefined),
+  };
   const enterpriseUnit: ServerExtension = {
     name: "fake-enterprise",
     edition: ServerEdition.enterprise,
+    requireAuthentication: true,
+    identityVerifiers: [editionVerifier],
+    drivers: {
+      platformTokenKeys: EDITION_RING,
+      resourceRowReaders: new Map([
+        [ApiResourceKind.identity_provider, noRows],
+        [ApiResourceKind.invitation, noRows],
+        [ApiResourceKind.team, noRows],
+      ]),
+    },
   };
 
   beforeAll(async () => {
@@ -225,9 +301,7 @@ describe("extension composition (enterprise edition on the wire)", () => {
       host: "127.0.0.1",
     });
     const port = await server.start();
-    portTransport = createGrpcTransport({
-      baseUrl: `http://127.0.0.1:${port}`,
-    });
+    portTransport = editionCallerTransport(port);
   });
 
   afterAll(async () => {
@@ -596,7 +670,7 @@ describe("extension composition (require-authentication posture)", () => {
     }
   });
 
-  it("a wider edition under the posture with no Authorizer is a boot throw — the built-in authorizer enforces the open-source kinds only", async () => {
+  it("a wider edition on the built-in authorizer with no row readers is a boot throw naming every kind it cannot read", async () => {
     const enterpriseDir = mkdtempSync(
       path.join(tmpdir(), "require-auth-edition-"),
     );
@@ -616,15 +690,16 @@ describe("extension composition (require-authentication posture)", () => {
           extensions: [
             {
               ...requiringExtension,
-              name: "enterprise-without-authorizer",
+              name: "enterprise-without-readers",
               edition: ServerEdition.enterprise,
+              drivers: { platformTokenKeys: EDITION_RING },
             },
           ],
           portOverride: 0,
           host: "127.0.0.1",
         }),
       ).rejects.toThrowError(
-        /serves edition 'enterprise' under the require-authentication posture but registers no Authorizer/,
+        /serves edition 'enterprise' under the built-in authorizer but registers no row reader for 'identity_provider', 'invitation', 'team'/,
       );
     } finally {
       rmSync(enterpriseDir, { recursive: true, force: true });
@@ -846,6 +921,7 @@ describe("extension composition (O4 gate slots + status hooks)", () => {
     portTransport = createGrpcTransport({
       baseUrl: `http://127.0.0.1:${port}`,
     });
+    await seedOrganizations(portTransport, ["acme"]);
   });
 
   afterAll(async () => {
@@ -1010,8 +1086,10 @@ describe("extension composition (C2 tuple lifecycle + organization directory)",
   const createdEvents: ResourceCreatedEvent[] = [];
   const deletedEvents: ResourceDeletedEvent[] = [];
   const visibilityEvents: VisibilityChangedEvent[] = [];
+  const executionVisibilityEvents: ExecutionVisibilityChangedEvent[] = [];
   let failCreates = false;
   let failDeletes = false;
+  let failExecutionVisibility = false;
 
   const fakeLifecycle: ResourceAuthorizationLifecycle = {
     async onResourceCreated(event): Promise<void> {
@@ -1029,6 +1107,12 @@ describe("extension composition (C2 tuple lifecycle + organization directory)",
     async onVisibilityChanged(event): Promise<void> {
       visibilityEvents.push(event);
     },
+    async onExecutionVisibilityChanged(event): Promise<void> {
+      if (failExecutionVisibility) {
+        throw new Error("fga is down");
+      }
+      executionVisibilityEvents.push(event);
+    },
   };
 
   // The directory answers are test-mutable so each case can stage its
@@ -1330,4 +1414,133 @@ describe("extension composition (C2 tuple lifecycle + organization directory)",
       expect(missing.rawMessage).toBe(`Organization not found: ${externalOrgId}`);
     }
   });
+
+  // The run audience of a workflow instance (`spec.execution_visibility`):
+  // open source derives `execution_viewer` from the row when a check asks;
+  // an edition that stores tuples hears it here (stigmer-cloud#720). The
+  // event carries the audience the level now names — the whole target
+  // state, so a retry or a repeat converges — and fires from the two doors
+  // that may set the level: create, and the dedicated RPC. Update and Apply
+  // ignore a request-carried level (the oss#573 rule for
+  // metadata.visibility), so neither can silently narrow run access.
+  describe("the run audience of a workflow instance", () => {
+    const workflows = () => createClient(WorkflowCommandController, portTransport);
+    const instances = () => createClient(WorkflowInstanceCommandController, portTransport);
+    const instanceQuery = () => createClient(WorkflowInstanceQueryController, portTransport);
+    let workflowId = "";
+
+    beforeAll(async () => {
+      const workflow = await workflows().create(
+        create(WorkflowSchema, {
+          apiVersion: "agentic.stigmer.ai/v1",
+          kind: "Workflow",
+          metadata: { name: "c2-run-audience-workflow", org: "c2seededorg" },
+          spec: {
+            document: { dsl: "1.0.0", namespace: "tests", name: "c2-run-audience", version: "0.1.0" },
+            tasks: [{ name: "seed", kind: 1, taskConfig: { variables: { greeting: "hello" } } }],
+          },
+        }),
+      );
+      workflowId = workflow.metadata?.id ?? "";
+    });
+
+    function instanceInput(name: string, level: WorkflowExecutionVisibility) {
+      return create(WorkflowInstanceSchema, {
+        apiVersion: "agentic.stigmer.ai/v1",
+        kind: "WorkflowInstance",
+        metadata: { name, org: "c2seededorg" },
+        spec: { workflowId, executionVisibility: level },
+      });
+    }
+
+    it("create at ORGANIZATION fires the audience once; the workflow's default instance and a private create fire nothing", async () => {
+      // The workflow's in-process default instance was created in beforeAll
+      // at the unset level, which names no audience.
+      expect(executionVisibilityEvents).toEqual([]);
+      const shared = await instances().create(
+        instanceInput("c2-shared-runs", WorkflowExecutionVisibility.organization),
+      );
+      await instances().create(instanceInput("c2-private-runs", WorkflowExecutionVisibility.private));
+      expect(executionVisibilityEvents).toEqual([
+        {
+          instanceKind: ApiResourceKind.workflow_instance,
+          instanceId: shared.metadata?.id,
+          orgId: "c2seededorg",
+          shapes: ["org-viewer"],
+        },
+      ]);
+    });
+
+    it("updateExecutionVisibility fires the level's audience every time, the empty one included", async () => {
+      const instance = await instances().create(
+        instanceInput("c2-toggled-runs", WorkflowExecutionVisibility.private),
+      );
+      const id = instance.metadata?.id ?? "";
+      executionVisibilityEvents.length = 0;
+      await instances().updateExecutionVisibility({
+        resourceId: id,
+        executionVisibility: WorkflowExecutionVisibility.organization,
+      });
+      await instances().updateExecutionVisibility({
+        resourceId: id,
+        executionVisibility: WorkflowExecutionVisibility.private,
+      });
+      expect(executionVisibilityEvents.map((event) => [event.instanceId, event.shapes])).toEqual([
+        [id, ["org-viewer"]],
+        [id, []],
+      ]);
+    });
+
+    it("update and apply keep the stored level whatever the request carries, and fire nothing", async () => {
+      const instance = await instances().create(
+        instanceInput("c2-kept-runs", WorkflowExecutionVisibility.organization),
+      );
+      const id = instance.metadata?.id ?? "";
+      executionVisibilityEvents.length = 0;
+      const lowered = clone(WorkflowInstanceSchema, instance);
+      lowered.spec!.executionVisibility = WorkflowExecutionVisibility.private;
+      lowered.spec!.description = "edited";
+      const updated = await instances().update(lowered);
+      expect(updated.spec?.description, "the update itself landed").toBe("edited");
+      expect(updated.spec?.executionVisibility).toBe(WorkflowExecutionVisibility.organization);
+      const reapplied = clone(WorkflowInstanceSchema, lowered);
+      reapplied.spec!.executionVisibility = WorkflowExecutionVisibility.unspecified;
+      const applied = await instances().apply(reapplied);
+      expect(applied.spec?.executionVisibility).toBe(WorkflowExecutionVisibility.organization);
+      const stored = await instanceQuery().get({ value: id });
+      expect(stored.spec?.executionVisibility).toBe(WorkflowExecutionVisibility.organization);
+      expect(executionVisibilityEvents).toEqual([]);
+    });
+
+    it("a driver failure fails the RPC with the level already persisted; the retry converges", async () => {
+      const instance = await instances().create(
+        instanceInput("c2-retried-runs", WorkflowExecutionVisibility.private),
+      );
+      const id = instance.metadata?.id ?? "";
+      failExecutionVisibility = true;
+      let refused: ConnectError | undefined;
+      try {
+        await instances().updateExecutionVisibility({
+          resourceId: id,
+          executionVisibility: WorkflowExecutionVisibility.organization,
+        });
+      } catch (error) {
+        refused = ConnectError.from(error);
+      } finally {
+        failExecutionVisibility = false;
+      }
+      expect(refused?.code).toBe(Code.Internal);
+      expect(refused?.rawMessage).toBe("failed to update execution visibility tuples");
+      const stored = await instanceQuery().get({ value: id });
+      expect(stored.spec?.executionVisibility, "post-persist: the level landed").toBe(
+        WorkflowExecutionVisibility.organization,
+      );
+      executionVisibilityEvents.length = 0;
+      await instances().updateExecutionVisibility({
+        resourceId: id,
+        executionVisibility: WorkflowExecutionVisibility.organization,
+      });
+      expect(executionVisibilityEvents.map((event) => event.shapes)).toEqual([["org-viewer"]]);
+    });
+  });
 });