@consiliency/agent-board-schema 1.12.0 → 1.23.0

package/tests/isolation/specs/fedsession_provider_absent_owner_lock.spec

@@ -0,0 +1,63 @@
+# Consiliency/message-board#193 — enroll_federated_human, with the provider ABSENT,
+# must fail BEFORE it takes the auth.users owner lock.
+#
+# WHY A LOCK TRACE AND NOT A RETURN VALUE. The defect is an ORDER, and the outcome
+# of the absent-provider call is 42501 either way (pre-fix from the delegate's
+# step (4), post-fix from the pre-lock gate). What differs is what the transaction
+# is HOLDING when it gets there. A guard session holds the owner's auth.users row,
+# so:
+#   * PRE-FIX  the call blocks -- isolationtester prints `<waiting ...>` -- which is
+#     the observable form of "it took the owner lock while the provider ordering was
+#     skipped entirely". That is the state from which the delegate, on its fresh READ
+#     COMMITTED snapshot, could see a concurrently-committed provider and reach its
+#     default board while still holding the owner: owner -> board, inverted against
+#     fleet onboarding's board -> owner.
+#   * POST-FIX it raises immediately, with NO `<waiting ...>` line at all.
+# Re-introducing the skip therefore changes the golden. Mutation-verified both ways.
+#
+# The owner row is materialized directly through _ensure_agent_auth_user for an
+# (issuer, subject) whose provider NEVER exists, which is the shape the existing
+# atomic_fedsession_fleet_lock_order fixture cannot express: it creates the provider
+# and the human before concurrency begins.
+
+setup
+{
+  create table lo193_owner as
+  select agent_board._ensure_agent_auth_user(
+    'fed-human-' || substr(
+      encode(extensions.digest(
+        convert_to('https://lo193-iso.test' || '|' || 'iso-orphan', 'UTF8'), 'sha256'), 'hex'),
+      1, 32)) as uid;
+}
+
+# Holds the human owner row that the absent-provider path must never reach.
+session "ohold"
+step "ohold_begin" { BEGIN; }
+step "ohold_lock" {
+  select 1 as locked from auth.users u
+   where u.id = (select uid from lo193_owner)
+   for update;
+}
+step "ohold_commit" { COMMIT; }
+
+session "portal"
+step "portal_begin" { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "portal_enroll" {
+  select status from agent_board.enroll_federated_human(
+    'https://lo193-iso.test', 'message-board', 'iso-orphan',
+    jsonb_build_object('sub','iso-orphan'), 'Iso Orphan'
+  );
+}
+step "portal_commit" { COMMIT; }
+
+session "check"
+step "check" {
+  select
+    (select count(*) from agent_board.agents a
+      where a.kind = 'human'
+        and a.metadata #>> ARRAY['federation','subject'] = 'iso-orphan') as human_agents,
+    (select count(*) from agent_board.identity_providers ip
+      where ip.issuer = 'https://lo193-iso.test') as providers;
+}
+
+permutation "ohold_begin" "ohold_lock" "portal_begin" "portal_enroll" "ohold_commit" "portal_commit" "check"

package/tests/isolation/specs/onboarding_ceiling_lock_order.spec

@@ -0,0 +1,183 @@
+# Consiliency/message-board#196 — onboard_federated_fleet_agent must take the
+# board-keyed ceiling ADVISORY lock BEFORE it locks the boards row.
+#
+# WHAT THIS PINS AND WHY IT IS NOT COVERED ELSEWHERE. This is the #189 ABBA edge:
+# set_board_authority_ceiling goes advisory -> boards (its upsert reaches the
+# boards row as FOR KEY SHARE through board_authority_ceilings' FK), so onboarding
+# must agree. The first cut of 20260901000110 had them inverted and deadlocked
+# deterministically. atomic_fleet_authority_races.spec CANNOT see this -- its own
+# header says so: every permutation there runs each RPC to completion in ONE step,
+# so neither transaction can be paused between its two acquisitions and no cycle
+# can form. That spec passed while the deadlock was live.
+#
+# Until #196 the only guard was a positional pgTAP assertion over RAW function
+# text, which one `--` comment naming the marker defeats. Measured against all 20
+# committed specs, an inverted onboarding (advisory taken after the boards lock)
+# moved NOTHING.
+#
+# HOW THE ORDER IS OBSERVED. A guard session holds the ceiling advisory lock, so
+# onboarding blocks -- `<waiting ...>`. A third session then probes the BOARDS row
+# with `for update nowait`:
+#   * CORRECT ORDER: onboarding is parked on the advisory lock and has not reached
+#     `select * into v_board ... for update`, so the probe returns the board;
+#   * INVERTED ORDER: onboarding already holds the boards row, so the probe fails
+#     with `ERROR: could not obtain lock on row in relation "boards"`.
+# The probe never joins the wait graph, so this terminates in BOTH orders instead
+# of reproducing the deadlock it is guarding against.
+#
+# SCOPE. This pins the advisory-before-boards edge ONLY. The boards-lock-before-
+# ceiling-read edge (the FOR KEY SHARE mechanism 20260901000110's header calls the
+# load-bearing one) is pinned elsewhere, and not by this spec -- a mutant that moves
+# only that lock leaves this golden byte-identical. What does catch it, measured:
+#   * agent_board_federated_lock_order_class.test.sql EC-3 fails BY NAME, and it
+#     already reads a stripped body, so a comment or literal mask does not hide it;
+#   * atomic_fleet_authority_races.spec pins the OUTCOME only for the state where
+#     BOTH serializers are gone -- its own header records that removing the boards
+#     FOR UPDATE alone still passes, because the advisory lock takes over;
+#   * atomic_fleet_board_archive.spec BLOCKS on that mutant rather than failing by
+#     name, which under the isolation runner (no timeout around isolationtester) is
+#     a stuck job, not a diagnosis.
+#
+# MUTATION-VERIFIED: `perform agent_board._lock_board_authority_ceiling(v_inv.board_id)`
+# moved below the boards lock -> this golden changes. Clean schema -> byte-identical
+# across 10 runs.
+
+setup
+{
+  insert into auth.users (id) values ('19610000-0000-0000-0000-0000000000a1');
+  insert into agent_board.agents(id, agent_type, kind, handle, display_name, status, auth_user_id)
+  values ('19610000-0000-0000-0000-000000000001', 'agent', 'agent',
+          'onboard-ceiling-owner', 'Onboard Ceiling Owner', 'active',
+          '19610000-0000-0000-0000-0000000000a1');
+
+  select count(*) from agent_board.create_board(
+    '19610000-0000-0000-0000-000000000001',
+    'onboard-ceiling-board', 'Onboard Ceiling Board',
+    'Consiliency/message-board#196 onboarding ceiling lock-order probe',
+    jsonb_build_object(), 'private'
+  );
+  insert into agent_board.fleet_trust_domains(
+    id, slug, display_name, domain_kind, owner_agent_id, lifecycle_state, approved_at
+  ) values (
+    '19610000-0000-0000-0000-000000000002', 'onboard-ceiling-domain',
+    'Onboard Ceiling Domain', 'tailnet', '19610000-0000-0000-0000-000000000001',
+    'active', now()
+  );
+  insert into agent_board.fleet_machines(
+    id, trust_domain_id, owner_agent_id, machine_label, machine_fingerprint,
+    lifecycle_state, enrollment_source, enrolled_at
+  ) values (
+    '19610000-0000-0000-0000-000000000003',
+    '19610000-0000-0000-0000-000000000002',
+    '19610000-0000-0000-0000-000000000001',
+    'claw', 'sha256:onboard-ceiling-machine', 'active', 'portal', now()
+  );
+  insert into agent_board.identity_providers(
+    issuer, audience, supported, policy_approved,
+    allowed_scopes, allowed_board_scopes, allowed_agent_kinds
+  ) values (
+    'https://onboard-ceiling.test', 'message-board', true, true,
+    ARRAY['board:send'], ARRAY[]::text[], ARRAY['agent']
+  );
+  select count(*) from agent_board.create_fleet_onboarding_invite(
+    (select id from agent_board.boards where slug='onboard-ceiling-board'),
+    '19610000-0000-0000-0000-000000000003', 'codex', 'phase-loop',
+    'message-board', 'sha256:onboard-ceiling-repo', 'worker', 'phase_worker',
+    'onboard-ceiling-agent', 1, now() + interval '1 hour',
+    '19610000-0000-0000-0000-000000000001', jsonb_build_object()
+  );
+}
+
+teardown
+{
+  delete from agent_board.fleet_machine_agents where machine_id='19610000-0000-0000-0000-000000000003';
+  delete from agent_board.fleet_repo_scopes where harness_registration_id in (
+    select id from agent_board.fleet_harness_registrations where machine_id='19610000-0000-0000-0000-000000000003'
+  );
+  delete from agent_board.fleet_harness_registrations where machine_id='19610000-0000-0000-0000-000000000003';
+  delete from agent_board.board_invite_redemptions where invite_id in (
+    select id from agent_board.board_invite_tokens where created_by_agent_id='19610000-0000-0000-0000-000000000001'
+  );
+  truncate agent_board.fleet_onboarding_invite_bounds;
+  delete from agent_board.board_invite_tokens where created_by_agent_id='19610000-0000-0000-0000-000000000001';
+  delete from agent_board.board_authority_ceilings where board_id in (
+    select id from agent_board.boards where slug='onboard-ceiling-board'
+  );
+  delete from agent_board.board_memberships where board_id in (
+    select id from agent_board.boards where slug='onboard-ceiling-board'
+  );
+  delete from agent_board.enrollment_federation_audit where agent_id in (
+    select id from agent_board.agents where handle='onboard-ceiling-agent'
+  );
+  delete from agent_board.agent_signing_keys where agent_id in (
+    select id from agent_board.agents where handle='onboard-ceiling-agent'
+  );
+  delete from agent_board.agents where handle='onboard-ceiling-agent';
+  delete from agent_board.boards where slug='onboard-ceiling-board';
+  delete from agent_board.fleet_machines where id='19610000-0000-0000-0000-000000000003';
+  delete from agent_board.fleet_trust_domains where id='19610000-0000-0000-0000-000000000002';
+  delete from agent_board.agents where id='19610000-0000-0000-0000-000000000001';
+  delete from auth.users where id='19610000-0000-0000-0000-0000000000a1';
+  delete from agent_board.identity_providers where issuer='https://onboard-ceiling.test';
+}
+
+# Holds the SAME board-keyed advisory lock onboarding must take first. It calls the
+# helper rather than re-deriving the key, so a key-derivation change cannot silently
+# make this guard lock a different object.
+session "holder"
+step "holder_begin" { BEGIN; }
+step "hold_ceiling_lock" {
+  select agent_board._lock_board_authority_ceiling(
+    (select id from agent_board.boards where slug='onboard-ceiling-board')) as locked;
+}
+step "holder_commit" { COMMIT; }
+
+session "onboard"
+step "onboard_begin" { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "onboard_call" {
+  select identity_disposition, membership_disposition, binding_disposition
+  from agent_board.onboard_federated_fleet_agent(
+    'https://onboard-ceiling.test', 'message-board', 'human-onboard-ceiling',
+    jsonb_build_object('iss','https://onboard-ceiling.test','aud','message-board','sub','human-onboard-ceiling'),
+    'onboard-ceiling-agent',
+    'yd_DIFGWGusVg3SbktDJQ1bE5UWvtR33xln4b05JIhk',
+    'raw_ed25519_base64url',
+    'sha256:NeVC2vx1MbTY56jUwJcLLr6YwB4IKCERsXWbUwhf5pA',
+    true, null, 'agent', ARRAY['board:send'], ARRAY[]::text[],
+    (select code from agent_board.board_invite_tokens where created_by_agent_id='19610000-0000-0000-0000-000000000001'),
+    (select id from agent_board.boards where slug='onboard-ceiling-board'),
+    '19610000-0000-0000-0000-000000000003', 'codex', 'phase-loop',
+    'message-board', 'sha256:onboard-ceiling-repo', 'worker', 'phase_worker',
+    'onboard-ceiling-agent'
+  );
+}
+step "onboard_commit" { COMMIT; }
+
+# NOWAIT probe: witnesses whether the blocked onboarding already holds the boards
+# row. Never blocks, so the permutation terminates in both orders.
+session "probe"
+step "probe_begin" { BEGIN; }
+step "probe_boards_row" {
+  select status from agent_board.boards
+   where id = (select id from agent_board.boards where slug='onboard-ceiling-board')
+   for update nowait;
+}
+step "probe_commit" { COMMIT; }
+
+session "check"
+step "check" {
+  select
+    (select count(*) from agent_board.agents where handle='onboard-ceiling-agent') as onboarded_agents,
+    (select count(*) from agent_board.board_memberships bm
+      where bm.board_id=(select id from agent_board.boards where slug='onboard-ceiling-board')
+        and bm.agent_id in (select id from agent_board.agents where handle='onboard-ceiling-agent')) as memberships;
+}
+
+# THE ORDERING PROBE. Onboarding is parked on the advisory lock; the boards row
+# must still be free.
+permutation "holder_begin" "hold_ceiling_lock" "onboard_begin" "onboard_call" "probe_begin" "probe_boards_row" "probe_commit" "holder_commit" "onboard_commit" "check"
+
+# Uncontended control: with nobody holding the advisory lock onboarding completes
+# in one step, so a `<waiting ...>` above is attributable to the advisory lock and
+# not to some unrelated serialization.
+permutation "onboard_begin" "onboard_call" "onboard_commit" "probe_begin" "probe_boards_row" "probe_commit" "check"

package/tests/isolation/specs/outbox_wake_ordering.spec

@@ -0,0 +1,202 @@
+# message-board#187 — the notification outbox has a monotonic write order (`seq`),
+# and the claim RPC uses it instead of a random uuid.
+#
+# WHY THIS IS NOT A pgTAP FILE. Two facts here need SEPARATE COMMITTED SESSIONS and
+# cannot be produced inside one rolled-back transaction:
+#
+#   1. `seq` comes from a NON-TRANSACTIONAL sequence, so a transaction that commits
+#      LATER can hold a LOWER seq than one that already committed. That is only
+#      observable when two sessions overlap.
+#   2. What a bounded claim TAKES, when several rows share `created_at`, is a
+#      cross-session fact about a committed queue — the defect this issue fixes.
+#
+# WHAT THE TRACE PROVES (all four are VALUE diffs, not just trace diffs):
+#
+#   * f_report — a fan-out to 8 subscribers, one statement, one transaction, writes 8
+#     rows with ONE distinct `created_at` and 8 distinct, CONTIGUOUS `seq`. `now()` is
+#     `transaction_timestamp()`, so time cannot order this set; `seq` can.
+#   * c_first — a claim of 4 takes exactly `fan_base .. fan_base+3`, IN THAT ORDER.
+#     Under the old `available_at, created_at, id` ordering the tiebreak was a random
+#     v4 uuid, so both the set (1 in 70) and the order (1 in 24) were chance.
+#   * c_second — after the late session commits, the next claim returns its row FIRST,
+#     because its `seq` was allocated BEFORE the fan-out's even though it committed
+#     after.
+#   * The same c_second line records the CONSEQUENCE for a consumer design: a
+#     late-committing writer lands BELOW the high-water mark a naive `seq`-cursor
+#     consumer would already have persisted. The claim RPC is unaffected — it is
+#     status-driven, not cursor-driven — but the wake contract this issue unblocks
+#     must not assume "seq greater than my cursor" is sufficient on its own.
+#
+# MUTATION SIGNAL: restore `order by coe.available_at asc, coe.created_at asc,
+# coe.id asc` in claim_callback_outbox_events and both c_first and c_second flip from
+# `t` to `f` (verified). Note what this spec does NOT pin: deleting `order by u.seq asc`
+# from the RETURNED batch changes nothing here, because the plan feeds the ordered
+# candidate CTE straight through and the observed order stays ascending anyway. That
+# property is pinned as a source assertion in agent_board_outbox_wake_ordering.test.sql
+# (WO-4 "RETURNED ordered by seq"), which is where the mutation actually lands.
+
+setup
+{
+  create table public.iso187_marks(kind text primary key, val bigint);
+
+  insert into agent_board.agents(id, agent_type, handle, display_name, status)
+  values ('18700000-0000-4000-8000-000000000000', 'agent', 'iso187-sender', 'Iso187 Sender', 'active');
+  insert into agent_board.agents(id, agent_type, handle, display_name, status)
+  select ('18700000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         'agent', 'iso187-sub-' || n, 'Iso187 Subscriber ' || n, 'active'
+    from generate_series(1, 8) n;
+
+  insert into agent_board.boards(id, slug, name, created_by_agent_id, visibility, status)
+  values ('18701000-0000-4000-8000-00000000000a', 'iso187-board-a', 'Iso187 Board A',
+          '18700000-0000-4000-8000-000000000000', 'open', 'active'),
+         ('18701000-0000-4000-8000-00000000000b', 'iso187-board-b', 'Iso187 Board B',
+          '18700000-0000-4000-8000-000000000000', 'open', 'active');
+
+  insert into agent_board.threads(id, thread_type, created_by_agent_id, subject, board_id)
+  values ('18702000-0000-4000-8000-00000000000a', 'coordination',
+          '18700000-0000-4000-8000-000000000000', 'Iso187 thread A',
+          '18701000-0000-4000-8000-00000000000a'),
+         ('18702000-0000-4000-8000-00000000000b', 'coordination',
+          '18700000-0000-4000-8000-000000000000', 'Iso187 thread B',
+          '18701000-0000-4000-8000-00000000000b');
+
+  insert into agent_board.messages(id, thread_id, author_agent_id, message_sequence, message_type, body)
+  values ('18703000-0000-4000-8000-00000000000a', '18702000-0000-4000-8000-00000000000a',
+          '18700000-0000-4000-8000-000000000000', 1, 'comment', 'iso187 board A post'),
+         ('18703000-0000-4000-8000-00000000000b', '18702000-0000-4000-8000-00000000000b',
+          '18700000-0000-4000-8000-000000000000', 1, 'comment', 'iso187 board B post');
+
+  insert into agent_board.board_memberships(id, board_id, agent_id, role, membership_state, created_by_agent_id)
+  select ('18704000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         '18701000-0000-4000-8000-00000000000a',
+         ('18700000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         'member', 'joined', '18700000-0000-4000-8000-000000000000'
+    from generate_series(1, 8) n;
+  insert into agent_board.board_memberships(id, board_id, agent_id, role, membership_state, created_by_agent_id)
+  values ('18704000-0000-4000-8000-0000000000b1', '18701000-0000-4000-8000-00000000000b',
+          '18700000-0000-4000-8000-000000000001', 'member', 'joined',
+          '18700000-0000-4000-8000-000000000000');
+
+  insert into agent_board.callback_endpoints(id, owner_agent_id, endpoint_url, endpoint_status, verification_requested_at, verified_at)
+  select ('18705000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         ('18700000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         'https://example.test/iso187/' || n, 'verified', now(), now()
+    from generate_series(1, 8) n;
+
+  insert into agent_board.callback_subscriptions(
+    id, endpoint_id, agent_id, subscription_scope, event_category,
+    board_id, board_membership_id, subscription_status)
+  select ('18706000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         ('18705000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         ('18700000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         'board', 'board.post.delivered',
+         '18701000-0000-4000-8000-00000000000a',
+         ('18704000-0000-4000-8000-' || lpad(n::text, 12, '0'))::uuid,
+         'active'
+    from generate_series(1, 8) n;
+  insert into agent_board.callback_subscriptions(
+    id, endpoint_id, agent_id, subscription_scope, event_category,
+    board_id, board_membership_id, subscription_status)
+  values ('18706000-0000-4000-8000-0000000000b1', '18705000-0000-4000-8000-000000000001',
+          '18700000-0000-4000-8000-000000000001', 'board', 'board.post.delivered',
+          '18701000-0000-4000-8000-00000000000b',
+          '18704000-0000-4000-8000-0000000000b1', 'active');
+
+  -- claim_callback_outbox_events polls the GLOBAL ready queue. Neutralize anything the
+  -- template left ready so this spec controls what a bounded claim can take.
+  update agent_board.callback_outbox_events
+     set outbox_status = 'delivered', delivered_at = now(), updated_at = now()
+   where outbox_status in ('pending', 'retryable');
+}
+
+teardown
+{
+  delete from agent_board.callback_outbox_events where board_id in ('18701000-0000-4000-8000-00000000000a', '18701000-0000-4000-8000-00000000000b');
+  delete from agent_board.callback_subscriptions  where board_id in ('18701000-0000-4000-8000-00000000000a', '18701000-0000-4000-8000-00000000000b');
+  delete from agent_board.callback_endpoints      where endpoint_url like 'https://example.test/iso187/%';
+  delete from agent_board.board_memberships       where board_id in ('18701000-0000-4000-8000-00000000000a', '18701000-0000-4000-8000-00000000000b');
+  delete from agent_board.messages                where thread_id in ('18702000-0000-4000-8000-00000000000a', '18702000-0000-4000-8000-00000000000b');
+  delete from agent_board.threads                 where id in ('18702000-0000-4000-8000-00000000000a', '18702000-0000-4000-8000-00000000000b');
+  delete from agent_board.boards                  where id in ('18701000-0000-4000-8000-00000000000a', '18701000-0000-4000-8000-00000000000b');
+  delete from agent_board.agents                  where handle like 'iso187-%';
+  drop table public.iso187_marks;
+}
+
+# "late" writes FIRST and commits LAST. Its row therefore holds a seq BELOW every row
+# the fan-out writes, while becoming visible only after the fan-out is already claimed.
+session "late"
+step "l_begin"  { BEGIN; }
+step "l_insert" {
+  select agent_board._queue_board_post_callbacks(
+    '18701000-0000-4000-8000-00000000000b'::uuid,
+    '18703000-0000-4000-8000-00000000000b'::uuid,
+    'Iso187 late', 'Iso187 late preview', 'high', jsonb_build_object()) as late_rows_written;
+}
+step "l_mark"   {
+  insert into public.iso187_marks(kind, val)
+  select 'late_seq', min(seq) from agent_board.callback_outbox_events
+   where board_id = '18701000-0000-4000-8000-00000000000b';
+}
+step "l_commit" { COMMIT; }
+
+# "fan" is the multicast fan-out: ONE statement, ONE transaction, eight recipients.
+session "fan"
+step "f_begin"  { BEGIN; }
+step "f_fanout" {
+  select agent_board._queue_board_post_callbacks(
+    '18701000-0000-4000-8000-00000000000a'::uuid,
+    '18703000-0000-4000-8000-00000000000a'::uuid,
+    'Iso187 fanout', 'Iso187 fanout preview', 'high', jsonb_build_object()) as fanout_rows_written;
+}
+step "f_mark"   {
+  insert into public.iso187_marks(kind, val)
+  select 'fan_base', min(seq) from agent_board.callback_outbox_events
+   where board_id = '18701000-0000-4000-8000-00000000000a';
+}
+# The heart of the issue: identical created_at across the whole fan-out, distinct and
+# contiguous seq. Absolute seq values are never printed — they depend on how many rows
+# the template ever wrote — so the golden stays deterministic.
+step "f_report" {
+  select count(*)                        as rows_written,
+         count(distinct created_at)      as distinct_created_at,
+         count(distinct seq)             as distinct_seq,
+         (max(seq) - min(seq) + 1)       as seq_span
+    from agent_board.callback_outbox_events
+   where board_id = '18701000-0000-4000-8000-00000000000a';
+}
+step "f_commit" { COMMIT; }
+
+# "claimer" is the notification worker. Autocommit: each claim is its own transaction,
+# exactly as a worker loop behaves.
+session "claimer"
+# The late row is still uncommitted here, so the visible queue is the fan-out alone.
+# The claim must take fan_base..fan_base+3 IN ASCENDING ORDER — set AND order.
+step "c_first"  {
+  select (select array_agg(outbox_seq) from agent_board.claim_callback_outbox_events(4))
+         = (select array_agg(g order by g)
+              from generate_series((select val from public.iso187_marks where kind = 'fan_base'),
+                                   (select val from public.iso187_marks where kind = 'fan_base') + 3) g)
+         as claimed_lowest_four_in_seq_order;
+}
+# The late row is committed now. It sorts FIRST despite committing last, and it sits
+# BELOW the high-water mark a naive seq-cursor consumer would have persisted from the
+# first claim — the boundary the wake contract must respect.
+step "c_second" {
+  with claimed as (
+    select array_agg(outbox_seq) as got from agent_board.claim_callback_outbox_events(4)
+  ),
+  marks as (
+    select (select val from public.iso187_marks where kind = 'late_seq') as late_seq,
+           (select val from public.iso187_marks where kind = 'fan_base') as fan_base
+  )
+  select (select got from claimed)
+           = array[(select late_seq from marks)]
+             || (select array_agg(g order by g)
+                   from generate_series((select fan_base from marks) + 4,
+                                        (select fan_base from marks) + 6) g)
+             as late_row_claimed_first_then_next_fan_rows,
+         (select late_seq from marks) < (select fan_base from marks)
+             as late_committer_holds_a_lower_seq_than_the_persisted_cursor;
+}
+
+permutation "l_begin" "l_insert" "l_mark" "f_begin" "f_fanout" "f_mark" "f_report" "f_commit" "c_first" "l_commit" "c_second"

package/tests/isolation/specs/reactivate_revoke_race.spec

@@ -0,0 +1,115 @@
+# #146 MUST-FIX 3 (concurrency) — reactivate_fleet_machine serializes against a concurrent
+# revoke, and re-checks the revoked precondition UNDER a row lock.
+#
+# reactivate_fleet_machine's read-decide-write was unconditional and unlocked: a revoke that
+# committed mid-operation could be silently overwritten (machine ends active with reactivated_at
+# newer than the revoke's event — the revoke, possibly a security response, is lost). The fix
+# takes `select ... for update` on the fleet_machines row before the still-revoked re-check, so
+# the whole operation serializes against revoke_fleet_subject (whose fleet_machines UPDATE also
+# locks that row).
+#
+# Two permutations, both service_role (reactivate is operator-only; revoke is operator here):
+#
+# P1 — reactivate-vs-revoke (the panel's requested pin). reactivate takes the lock and holds
+# it; the concurrent revoke BLOCKS (isolationtester prints `<waiting ...>`), then applies after
+# reactivate commits. FINAL: the machine is revoked and evaluate denies (machine_revoked) — the
+# revoke is NOT lost. Combined with MUST-FIX 1 (revoke advances effective_at=now()), the late
+# revoke both flips lifecycle AND post-dates reactivated_at, so both guards deny.
+#
+# P2 — reactivate-vs-reactivate (the for-update MUTATION signal). The second reactivate blocks
+# on the first's lock, then RE-READS under the lock: the machine is now active, so its
+# still-revoked re-check fails and it RAISES `fleet machine is not revoked; nothing to
+# reactivate`. MUTATION SIGNAL: drop `for update` from reactivate's read → the second reactivate
+# reads the stale (pre-commit) revoked snapshot, its re-check passes, and it succeeds instead of
+# raising — the `<waiting>`/error trace diverges from this golden.
+
+setup
+{
+  insert into agent_board.agents(id, agent_type, handle, display_name, status) values
+    ('a0000000-0000-4000-8000-000000000001', 'agent', 'react-iso-owner', 'React Iso Owner', 'active'),
+    ('a0000000-0000-4000-8000-000000000002', 'agent', 'react-iso-sender', 'React Iso Sender', 'active'),
+    ('a0000000-0000-4000-8000-000000000003', 'agent', 'react-iso-recip', 'React Iso Recipient', 'active');
+
+  insert into agent_board.fleet_trust_domains(id, slug, display_name, domain_kind, owner_agent_id, lifecycle_state, approved_at)
+  values ('d0000000-0000-4000-8000-000000000001', 'react-iso-domain', 'React Iso Domain', 'tailnet',
+          'a0000000-0000-4000-8000-000000000001', 'active', now());
+
+  -- A machine that is already revoked (with a PRIOR machine-revocation event, effective 1h ago).
+  insert into agent_board.fleet_machines(id, trust_domain_id, owner_agent_id, machine_label, machine_fingerprint,
+                                         lifecycle_state, enrollment_source, enrolled_at, revoked_at, reactivated_at, metadata)
+  values ('60000000-0000-4000-8000-000000000001', 'd0000000-0000-4000-8000-000000000001',
+          'a0000000-0000-4000-8000-000000000001', 'react-iso-m', 'sha256:react-iso-m',
+          'revoked', 'portal', now(), now(), null, jsonb_build_object('redacted', true, 'metadata-safe', true));
+
+  insert into agent_board.fleet_machine_agents(machine_id, agent_id, agent_role)
+  values ('60000000-0000-4000-8000-000000000001', 'a0000000-0000-4000-8000-000000000002', 'phase_worker');
+
+  insert into agent_board.fleet_revocation_events(id, subject_kind, subject_id, machine_id, revoked_by_agent_id, revocation_reason, effective_at, metadata)
+  values ('e0000000-0000-4000-8000-000000000001', 'machine', '60000000-0000-4000-8000-000000000001', '60000000-0000-4000-8000-000000000001',
+          'a0000000-0000-4000-8000-000000000001', 'iso initial revoke', now() - interval '1 hour',
+          jsonb_build_object('redacted', true, 'metadata-safe', true));
+
+  -- Allow-by-default receive policy so a non-revoked bound machine would pass; the only lever
+  -- the checker exercises is the machine-revocation state.
+  insert into agent_board.fleet_receive_policies(recipient_agent_id, policy_name, default_action, require_signed_sender, lifecycle_state, metadata)
+  values ('a0000000-0000-4000-8000-000000000003', 'react iso allow', 'allow', false, 'active', jsonb_build_object());
+}
+
+teardown
+{
+  delete from agent_board.fleet_revocation_events where subject_id = '60000000-0000-4000-8000-000000000001';
+  delete from agent_board.fleet_machine_agents where machine_id = '60000000-0000-4000-8000-000000000001';
+  delete from agent_board.fleet_machines where id = '60000000-0000-4000-8000-000000000001';
+  delete from agent_board.fleet_receive_policies where recipient_agent_id = 'a0000000-0000-4000-8000-000000000003';
+  delete from agent_board.fleet_trust_domains where id = 'd0000000-0000-4000-8000-000000000001';
+  delete from agent_board.agents where id in (
+    'a0000000-0000-4000-8000-000000000001',
+    'a0000000-0000-4000-8000-000000000002',
+    'a0000000-0000-4000-8000-000000000003');
+}
+
+# ra: the reactivation. Its `select ... for update` locks the machine row until ra commits.
+session "ra"
+step "ra_begin"  { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "ra_react"  {
+  select lifecycle_state
+    from agent_board.reactivate_fleet_machine(
+      '60000000-0000-4000-8000-000000000001', 'a0000000-0000-4000-8000-000000000001', 'iso reactivate');
+}
+step "ra_commit" { COMMIT; }
+
+# rb: a competing writer — either a revoke (P1) or a second reactivate (P2). Its row write
+# blocks on ra's lock until ra commits.
+session "rb"
+step "rb_begin"   { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "rb_revoke"  {
+  select revocation_event_id
+    from agent_board.revoke_fleet_subject(
+      'machine', '60000000-0000-4000-8000-000000000001', 'a0000000-0000-4000-8000-000000000001',
+      'iso re-revoke', jsonb_build_object('redacted', true));
+}
+step "rb_react"   {
+  select lifecycle_state
+    from agent_board.reactivate_fleet_machine(
+      '60000000-0000-4000-8000-000000000001', 'a0000000-0000-4000-8000-000000000001', 'iso second reactivate');
+}
+step "rb_commit"  { COMMIT; }
+
+# checker: final machine lifecycle + the policy verdict for the bound sender.
+session "chk"
+step "chk" {
+  select fm.lifecycle_state,
+         (select denial_class
+            from agent_board.evaluate_fleet_sender_policy(
+              'a0000000-0000-4000-8000-000000000002', 'a0000000-0000-4000-8000-000000000003',
+              fm.id, null, 'send_direct_message', null, false)) as denial_class
+    from agent_board.fleet_machines fm
+   where fm.id = '60000000-0000-4000-8000-000000000001';
+}
+
+# P1: reactivate holds the lock; the revoke blocks then applies → final revoked + denied.
+permutation "ra_begin" "ra_react" "rb_begin" "rb_revoke" "ra_commit" "rb_commit" "chk"
+
+# P2: reactivate holds the lock; the second reactivate blocks, re-reads under the lock, and
+# raises (nothing to reactivate) → the for-update mutation signal.
+permutation "ra_begin" "ra_react" "rb_begin" "rb_react" "ra_commit" "rb_commit" "chk"

package/tests/isolation/specs/reclaim_revoke_race.spec

@@ -0,0 +1,94 @@
+# #150 MUST-FIX 1 (codex) — reclaim_fleet_machine serializes against a concurrent squatter
+# revoke('machine'), and revoke re-checks machine ownership UNDER a row lock.
+#
+# revoke_fleet_subject's machine-ownership authz check was UNLOCKED: a squatter could call
+# revoke('machine', M); its ownership SELECT reads owner=squatter (passes) with no lock, then its
+# lifecycle UPDATE blocks on reclaim's `select ... for update` and — pre-fix — applies AFTER
+# reclaim commits (owner=new_owner), revoking the machine the operator just handed over (a
+# DoS-flavoured defeat of reclaim's atomicity). The fix takes `select ... for update` on the owning
+# fleet_machines row BEFORE the ownership check, so revoke serializes behind reclaim and the
+# re-read sees committed ownership (owner=new_owner) → the squatter's revoke is refused 42501.
+#
+# P1 — reclaim-vs-squatter-revoke (the pin). reclaim (service_role) takes the lock and holds it;
+# the squatter's authenticated revoke BLOCKS at its own FOR UPDATE (isolationtester prints
+# `<waiting ...>`), then after reclaim commits RE-READS ownership under the lock, sees owner=
+# new_owner, and RAISES `machine is not owned by current_agent_id()`. FINAL: owner=new_owner,
+# lifecycle=active — the reclaimed machine is NOT nuked.
+#
+# MUTATION SIGNAL: drop the FOR UPDATE lock from revoke_fleet_subject → the squatter's ownership
+# SELECT reads the stale (pre-commit) owner=squatter and PASSES without waiting; revoke then blocks
+# at its lifecycle UPDATE, and after reclaim commits it SUCCEEDS (returns an event id) and the
+# checker shows lifecycle=revoked / owner=new_owner — the trace diverges from this golden.
+
+setup
+{
+  insert into auth.users(id) values ('b1000000-0000-4000-8000-000000000002');
+
+  insert into agent_board.agents(id, agent_type, handle, display_name, status, auth_user_id) values
+    ('a1000000-0000-4000-8000-000000000001', 'agent', 'reclaim-iso-operator', 'Reclaim Iso Operator', 'active', null),
+    ('a1000000-0000-4000-8000-000000000002', 'agent', 'reclaim-iso-squatter', 'Reclaim Iso Squatter', 'active', 'b1000000-0000-4000-8000-000000000002'),
+    ('a1000000-0000-4000-8000-000000000003', 'agent', 'reclaim-iso-newowner', 'Reclaim Iso New Owner', 'active', null);
+
+  insert into agent_board.fleet_trust_domains(id, slug, display_name, domain_kind, owner_agent_id, lifecycle_state, approved_at)
+  values ('d1000000-0000-4000-8000-000000000001', 'reclaim-iso-domain', 'Reclaim Iso Domain', 'tailnet',
+          'a1000000-0000-4000-8000-000000000002', 'active', now());
+
+  -- An active machine owned by the squatter (the pre-registered/squatted fingerprint).
+  insert into agent_board.fleet_machines(id, trust_domain_id, owner_agent_id, machine_label, machine_fingerprint,
+                                         lifecycle_state, enrollment_source, enrolled_at, revoked_at, metadata)
+  values ('61000000-0000-4000-8000-000000000001', 'd1000000-0000-4000-8000-000000000001',
+          'a1000000-0000-4000-8000-000000000002', 'reclaim-iso-m', 'sha256:reclaim-iso-m',
+          'active', 'portal', now(), null, jsonb_build_object('redacted', true, 'metadata-safe', true));
+}
+
+teardown
+{
+  delete from agent_board.fleet_revocation_events where subject_id = '61000000-0000-4000-8000-000000000001';
+  delete from agent_board.fleet_machines where id = '61000000-0000-4000-8000-000000000001';
+  delete from agent_board.fleet_trust_domains where id = 'd1000000-0000-4000-8000-000000000001';
+  delete from agent_board.agents where id in (
+    'a1000000-0000-4000-8000-000000000001',
+    'a1000000-0000-4000-8000-000000000002',
+    'a1000000-0000-4000-8000-000000000003');
+  delete from auth.users where id = 'b1000000-0000-4000-8000-000000000002';
+}
+
+# ra: the operator reclaim (service_role). Its `select ... for update` locks the machine row until
+# ra commits, reassigning ownership to the new owner.
+session "ra"
+step "ra_begin"   { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "ra_reclaim" {
+  select owner_agent_id
+    from agent_board.reclaim_fleet_machine(
+      '61000000-0000-4000-8000-000000000001', 'a1000000-0000-4000-8000-000000000003',
+      'a1000000-0000-4000-8000-000000000001', 'iso operator reclaim');
+}
+step "ra_commit"  { COMMIT; }
+
+# rb: the squatter's revoke via the authenticated domain-owner path. With the fix it BLOCKS at its
+# own FOR UPDATE, then re-reads committed ownership and is refused 42501.
+session "rb"
+step "rb_begin"  {
+  BEGIN;
+  SET LOCAL "request.jwt.claim.role" = 'authenticated';
+  SET LOCAL "request.jwt.claim.sub" = 'b1000000-0000-4000-8000-000000000002';
+}
+step "rb_revoke" {
+  select revocation_event_id
+    from agent_board.revoke_fleet_subject(
+      'machine', '61000000-0000-4000-8000-000000000001', 'a1000000-0000-4000-8000-000000000002',
+      'iso squatter revoke', jsonb_build_object('redacted', true));
+}
+step "rb_commit" { COMMIT; }
+
+# checker: final ownership + lifecycle of the contested machine.
+session "chk"
+step "chk" {
+  select owner_agent_id, lifecycle_state
+    from agent_board.fleet_machines
+   where id = '61000000-0000-4000-8000-000000000001';
+}
+
+# P1: reclaim holds the lock; the squatter's revoke blocks then is refused → machine stays active
+# under the new owner (the revoke is NOT applied).
+permutation "ra_begin" "ra_reclaim" "rb_begin" "rb_revoke" "ra_commit" "rb_commit" "chk"