@consiliency/agent-board-schema 1.50.0 → 1.52.0

package/tests/isolation/expected/challenge_supersede_lock_order.out

@@ -0,0 +1,213 @@
+Parsed test spec with 3 sessions
+
+starting permutation: mint_begin mint_lock_challenge issue_begin issue_at_cap mint_lock_key issue_commit mint_commit check
+step mint_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step mint_lock_challenge: 
+  select count(*)::int as locked_challenges from (
+    select 1 from agent_board.agent_signing_key_challenges c
+     where c.id = 'f1c00000-0000-4000-8000-000000000001' for update) s;
+
+locked_challenges
+-----------------
+                1
+(1 row)
+
+step issue_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step issue_at_cap: 
+  select c.key_id, c.key_version
+    from agent_board.issue_signing_key_challenge(
+      'f1a00000-0000-4000-8000-000000000001', 'primary', interval '5 minutes') c;
+
+key_id |key_version
+-------+-----------
+primary|          1
+(1 row)
+
+step mint_lock_key: 
+  select count(*)::int as locked_keys from (
+    select 1 from agent_board.agent_signing_keys k
+     where k.id = 'f1b00000-0000-4000-8000-000000000001' for update) s;
+ <waiting ...>
+step issue_commit: COMMIT;
+step mint_lock_key: <... completed>
+locked_keys
+-----------
+          1
+(1 row)
+
+step mint_commit: COMMIT;
+step check: 
+  select
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000001') as oldest_state,
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000002') as second_oldest_state,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and metadata ? 'superseded_by') as evicted_rows,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and proof_state = 'pending' and consumed_at is null
+        and expires_at > now()) as pending_rows;
+
+oldest_state|second_oldest_state|evicted_rows|pending_rows
+------------+-------------------+------------+------------
+pending     |expired            |           1|          16
+(1 row)
+
+
+starting permutation: issue_begin issue_at_cap mint_begin mint_lock_challenge issue_commit mint_lock_key mint_commit check
+step issue_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step issue_at_cap: 
+  select c.key_id, c.key_version
+    from agent_board.issue_signing_key_challenge(
+      'f1a00000-0000-4000-8000-000000000001', 'primary', interval '5 minutes') c;
+
+key_id |key_version
+-------+-----------
+primary|          1
+(1 row)
+
+step mint_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step mint_lock_challenge: 
+  select count(*)::int as locked_challenges from (
+    select 1 from agent_board.agent_signing_key_challenges c
+     where c.id = 'f1c00000-0000-4000-8000-000000000001' for update) s;
+ <waiting ...>
+step issue_commit: COMMIT;
+step mint_lock_challenge: <... completed>
+locked_challenges
+-----------------
+                1
+(1 row)
+
+step mint_lock_key: 
+  select count(*)::int as locked_keys from (
+    select 1 from agent_board.agent_signing_keys k
+     where k.id = 'f1b00000-0000-4000-8000-000000000001' for update) s;
+
+locked_keys
+-----------
+          1
+(1 row)
+
+step mint_commit: COMMIT;
+step check: 
+  select
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000001') as oldest_state,
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000002') as second_oldest_state,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and metadata ? 'superseded_by') as evicted_rows,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and proof_state = 'pending' and consumed_at is null
+        and expires_at > now()) as pending_rows;
+
+oldest_state|second_oldest_state|evicted_rows|pending_rows
+------------+-------------------+------------+------------
+expired     |pending            |           1|          16
+(1 row)
+
+
+starting permutation: issue_begin issue_at_cap issue_commit mint_begin mint_lock_challenge mint_lock_key mint_commit check
+step issue_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step issue_at_cap: 
+  select c.key_id, c.key_version
+    from agent_board.issue_signing_key_challenge(
+      'f1a00000-0000-4000-8000-000000000001', 'primary', interval '5 minutes') c;
+
+key_id |key_version
+-------+-----------
+primary|          1
+(1 row)
+
+step issue_commit: COMMIT;
+step mint_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step mint_lock_challenge: 
+  select count(*)::int as locked_challenges from (
+    select 1 from agent_board.agent_signing_key_challenges c
+     where c.id = 'f1c00000-0000-4000-8000-000000000001' for update) s;
+
+locked_challenges
+-----------------
+                1
+(1 row)
+
+step mint_lock_key: 
+  select count(*)::int as locked_keys from (
+    select 1 from agent_board.agent_signing_keys k
+     where k.id = 'f1b00000-0000-4000-8000-000000000001' for update) s;
+
+locked_keys
+-----------
+          1
+(1 row)
+
+step mint_commit: COMMIT;
+step check: 
+  select
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000001') as oldest_state,
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000002') as second_oldest_state,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and metadata ? 'superseded_by') as evicted_rows,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and proof_state = 'pending' and consumed_at is null
+        and expires_at > now()) as pending_rows;
+
+oldest_state|second_oldest_state|evicted_rows|pending_rows
+------------+-------------------+------------+------------
+expired     |pending            |           1|          16
+(1 row)
+
+
+starting permutation: mint_begin mint_lock_all issue_begin issue_at_cap issue_commit mint_commit check
+step mint_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step mint_lock_all: 
+  select count(*)::int as locked_challenges from (
+    select 1 from agent_board.agent_signing_key_challenges c
+     where c.signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+       and c.proof_state = 'pending'
+       and c.consumed_at is null
+     order by c.id
+     for update) s;
+
+locked_challenges
+-----------------
+               16
+(1 row)
+
+step issue_begin: BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role';
+step issue_at_cap: 
+  select c.key_id, c.key_version
+    from agent_board.issue_signing_key_challenge(
+      'f1a00000-0000-4000-8000-000000000001', 'primary', interval '5 minutes') c;
+
+ERROR:  challenge_pending_cap: too many pending signing-key challenges for this key (max 16)
+step issue_commit: COMMIT;
+step mint_commit: COMMIT;
+step check: 
+  select
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000001') as oldest_state,
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000002') as second_oldest_state,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and metadata ? 'superseded_by') as evicted_rows,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and proof_state = 'pending' and consumed_at is null
+        and expires_at > now()) as pending_rows;
+
+oldest_state|second_oldest_state|evicted_rows|pending_rows
+------------+-------------------+------------+------------
+pending     |pending            |           0|          16
+(1 row)
+

package/tests/isolation/specs/challenge_supersede_lock_order.spec

@@ -0,0 +1,183 @@
+# F1 (code-review-2026-09-01) — issue_signing_key_challenge must be ACYCLIC against
+# mint_fleet_agent_session once it supersedes at the pending-challenge cap.
+#
+# THE CYCLE THIS CHANGE WOULD HAVE INTRODUCED. Superseding at the cap makes
+# issue_signing_key_challenge WRITE a challenge row while it already holds the key row:
+#     agent_signing_keys(FOR UPDATE, 20260719000100 / this migration)
+#       -> agent_signing_key_challenges(UPDATE, the eviction)
+# mint_fleet_agent_session takes exactly that pair in the OPPOSITE order:
+#     agent_signing_key_challenges(FOR UPDATE, 20260718000600:73-76)
+#       -> agent_signing_keys(FOR UPDATE, 20260718000600:95-100)
+# That is an ABBA between two honest callers, and not a rare one: the traffic this change
+# is FOR — a flood driving evictions while legitimate agents mint — is precisely the
+# traffic that produces it. agent_board_fk_lock_edges.test.sql FKL-10 named the pair the
+# first time this migration was written without the fix; that assertion is a static
+# detector and cannot see the fix, which is why the proof lives here.
+#
+# THE FIX this spec pins: the eviction's row selection carries FOR UPDATE SKIP LOCKED, so
+# the issuing transaction NEVER WAITS on a challenge row. A row a mint already holds is
+# passed over for the next-oldest. A transaction that cannot wait cannot close a cycle.
+#
+# WHAT THE GOLDEN SHOWS, AND WHAT WOULD CHANGE IF THE CLAUSE WERE REMOVED. In permutation
+# 1 `mint_lock_challenge` takes the OLDEST pending row; `issue` then runs at the cap and
+# COMPLETES WITHOUT A `<waiting ...>` LINE, evicting the SECOND-oldest instead. Delete
+# `skip locked` and that step blocks — the trace gains `<waiting ...>`, and once
+# `mint_lock_key` runs the two transactions are in a cycle and the server reports
+# `ERROR: deadlock detected`. Either outcome is a byte diff against this golden.
+#
+# THE "mint" SESSION IS A LOCK-EQUIVALENT STAND-IN, and here is the equivalence.
+#   * It runs mint_fleet_agent_session's two cycle-relevant acquisitions, in its order and
+#     its modes: the challenge row FOR UPDATE (:73-76), then the key row FOR UPDATE
+#     (:95-100).
+#   * It omits that function's other work — _agent_principal_active's reads, the
+#     delegation_grants lookup, the auth_user_id read, the challenge consume UPDATE and the
+#     nonfleet_sessions insert. None of those touches agent_signing_keys, and the consume
+#     UPDATE re-writes a row this session already holds, so none can add an edge to the
+#     pair under test.
+#   * A real mint call was rejected only because it needs a valid PoP challenge, an
+#     auth_user_id-bearing principal and a live lineage inside a setup block, and because
+#     every one of its rejection paths RAISES, which would abort the session mid-permutation
+#     and turn a lock-order question into an error-message question. The claim under test is
+#     a LOCK ORDER, and these are the locks.
+#
+# THE ISSUE SESSION IS THE REAL FUNCTION, unmodified. That side is where the fix lives, so
+# a stand-in there would be testing the spec instead of the schema.
+
+setup
+{
+  insert into agent_board.agents(id, agent_type, kind, handle, display_name, status)
+  values ('f1a00000-0000-4000-8000-000000000001','agent','agent','f1iso-owner','F1 ISO Owner','active');
+
+  insert into agent_board.agent_signing_keys(
+    id, agent_id, key_id, key_version, public_key, public_key_format, fingerprint, status, proof_state)
+  values ('f1b00000-0000-4000-8000-000000000001','f1a00000-0000-4000-8000-000000000001',
+          'primary', 1, public.test_pubkey('f1-iso'), 'raw_ed25519_base64url',
+          public.test_key_fingerprint('f1-iso'), 'active', 'proven');
+
+  -- Exactly sixteen pending challenges: the key is AT the cap, so the next issue must
+  -- evict. created_at is set explicitly so "oldest" and "second-oldest" are unambiguous
+  -- rather than depending on insert clock resolution.
+  insert into agent_board.agent_signing_key_challenges(
+    id, signing_key_id, agent_id, challenge_nonce, challenge_bytes, expires_at, created_at)
+  select
+    ('f1c00000-0000-4000-8000-0000000000' || lpad(i::text, 2, '0'))::uuid,
+    'f1b00000-0000-4000-8000-000000000001',
+    'f1a00000-0000-4000-8000-000000000001',
+    'f1-iso-nonce-' || i,
+    'agent-signing-key-proof.v1:f1-iso:' || i,
+    now() + interval '5 minutes',
+    now() - make_interval(secs => (17 - i))
+  from generate_series(1, 16) i;
+}
+
+teardown
+{
+  delete from agent_board.agent_signing_key_challenges
+   where agent_id = 'f1a00000-0000-4000-8000-000000000001';
+  delete from agent_board.agent_signing_keys
+   where agent_id = 'f1a00000-0000-4000-8000-000000000001';
+  delete from agent_board.agents where handle like 'f1iso-%';
+}
+
+# mint_fleet_agent_session's two cycle-relevant locks, in its order and modes.
+session "mint"
+step "mint_begin" { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+# The OLDEST pending row — the one the eviction would otherwise take.
+step "mint_lock_challenge" {
+  select count(*)::int as locked_challenges from (
+    select 1 from agent_board.agent_signing_key_challenges c
+     where c.id = 'f1c00000-0000-4000-8000-000000000001' for update) s;
+}
+# ALL sixteen pending rows at once. This is the adversary for the no-evictable-row branch:
+# SKIP LOCKED can only skip, so with every candidate held the eviction finds nothing.
+step "mint_lock_all" {
+  select count(*)::int as locked_challenges from (
+    select 1 from agent_board.agent_signing_key_challenges c
+     where c.signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+       and c.proof_state = 'pending'
+       and c.consumed_at is null
+     order by c.id
+     for update) s;
+}
+step "mint_lock_key" {
+  select count(*)::int as locked_keys from (
+    select 1 from agent_board.agent_signing_keys k
+     where k.id = 'f1b00000-0000-4000-8000-000000000001' for update) s;
+}
+step "mint_commit" { COMMIT; }
+
+# The real issuer, unmodified, at the cap. Only the deterministic columns are selected:
+# challenge_id and the nonce inside `challenge` are fresh random values per run and must
+# never reach a golden.
+session "issue"
+step "issue_begin" { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "issue_at_cap" {
+  select c.key_id, c.key_version
+    from agent_board.issue_signing_key_challenge(
+      'f1a00000-0000-4000-8000-000000000001', 'primary', interval '5 minutes') c;
+}
+step "issue_commit" { COMMIT; }
+
+session "check"
+# WHICH row was evicted is the observable that distinguishes skipping from waiting: with
+# SKIP LOCKED the mint-held oldest row is untouched and the SECOND-oldest carries the
+# marker. The pending count proves the bound still holds.
+step "check" {
+  select
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000001') as oldest_state,
+    (select proof_state from agent_board.agent_signing_key_challenges
+      where id = 'f1c00000-0000-4000-8000-000000000002') as second_oldest_state,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and metadata ? 'superseded_by') as evicted_rows,
+    (select count(*)::int from agent_board.agent_signing_key_challenges
+      where signing_key_id = 'f1b00000-0000-4000-8000-000000000001'
+        and proof_state = 'pending' and consumed_at is null
+        and expires_at > now()) as pending_rows;
+}
+
+# 1. THE CYCLE, in the order that would produce the deadlock. mint holds the oldest
+#    challenge row; issue then runs at the cap. It must NOT wait (no `<waiting ...>`), and
+#    mint's key lock must then be the only thing that blocks.
+permutation "mint_begin" "mint_lock_challenge" "issue_begin" "issue_at_cap" "mint_lock_key" "issue_commit" "mint_commit" "check"
+
+# 2. The reverse arrival order: issue arrives FIRST, evicts the true oldest, and mint then
+#    blocks on that row and proceeds once issue commits. This is the direction in which the
+#    eviction DOES cost a mint a wait, and it pins that the cost is a wait-then-proceed and
+#    not a deadlock.
+#
+#    `issue_commit` MUST precede `mint_lock_key`. The first version of this permutation put
+#    the commit last, which does not test the schema at all: mint blocks on the challenge
+#    row, isolationtester will not run a blocked session's next step, and nothing in the
+#    permutation can release the lock — so the tester sat out its own 300-second cancel and
+#    baked `canceling step mint_lock_challenge after 300 seconds` into the golden. That is a
+#    property of the spec's step order, not of the lock order, and it made every iteration
+#    cost five minutes.
+permutation "issue_begin" "issue_at_cap" "mint_begin" "mint_lock_challenge" "issue_commit" "mint_lock_key" "mint_commit" "check"
+
+# 3. Fully serial: issue commits before mint starts. The baseline the two concurrent
+#    permutations are read against.
+permutation "issue_begin" "issue_at_cap" "issue_commit" "mint_begin" "mint_lock_challenge" "mint_lock_key" "mint_commit" "check"
+
+# 4. EVERY candidate locked -- the branch that exists only because SKIP LOCKED can skip.
+#
+#    With all sixteen pending rows held by another session the eviction's
+#    `for update skip locked limit 1` returns NO ROW, so `v_evicted` is null. The function
+#    must then REFUSE (challenge_pending_cap) rather than issue: inserting a seventeenth
+#    would push the pending set past the bound the cap exists to hold, and the count that
+#    said the set was full was correct. This is the ONE path where the pre-change refusal
+#    survives, and the PR describes it as "confined to a window in which sixteen mints for
+#    one key are in flight at once" -- a claim that was visible in the SQL and asserted
+#    nowhere until this permutation.
+#
+#    Three things the golden shows, and all three matter:
+#      * issue_at_cap ERRORS. It does not wait -- SKIP LOCKED means no `<waiting ...>`.
+#      * `evicted_rows = 0`. Nothing was superseded, so no legitimate challenge was
+#        destroyed on the way to refusing.
+#      * `pending_rows = 16`. No seventeenth row was inserted.
+#
+#    MUTATION: delete the `if v_evicted is null then raise` guard and this permutation's
+#    golden changes -- the refusal disappears and pending_rows becomes 17, which is the
+#    bound being exceeded, i.e. exactly the failure the guard is there to prevent.
+permutation "mint_begin" "mint_lock_all" "issue_begin" "issue_at_cap" "issue_commit" "mint_commit" "check"