@consiliency/agent-board-schema 1.44.0 → 1.48.0

package/tests/isolation/specs/fleet_domain_no_downgrade_demote_race.spec

@@ -0,0 +1,292 @@
+# #397 — register_fleet_machine's no-downgrade guard (#395) against a destination DEMOTED while
+# the move waits.
+#
+# THE WINDOW. The guard's rank comparison is a conjunct of the ON CONFLICT DO UPDATE's WHERE, so
+# it is atomic against the conflicting MACHINE row. The DESTINATION domain's lifecycle is not:
+# pinning it needs a row lock on fleet_trust_domains, which was added to this function once
+# (PR #158, ab38805) and REVERTED for deadlocking 40P01 against reclaim_fleet_machine — #159 owns
+# the lock order and is open. So a `service_role` demotion of the destination can commit while the
+# upsert is blocked on the machine row, and the machine lands in a domain that is 'pending' at
+# commit time, still 'active' itself, with its fleet_machine_agents bindings intact (they key on
+# machine_id) — #395 again, under concurrency.
+#
+# WHAT WAS MEASURED, and why the obvious repair is not the one shipped. Moving the destination
+# read INTO the upsert as a correlated subquery does NOT close this: under READ COMMITTED an
+# upsert that waits on the conflicting row re-reads THAT ROW's latest version on resume but
+# evaluates every other relation against the statement's ORIGINAL snapshot, so the domain is still
+# read as of before the wait. Both the cached read and the correlated subquery were run against
+# this interleaving and both allowed the move. What closes it is a re-check of the destination in
+# a SEPARATE statement after the move — a fresh snapshot — which raises 42501 and unwinds it.
+#
+# P1 — the pin. rl locks the machine row and holds it; rm's register BLOCKS (isolationtester
+# prints `<waiting ...>`); rl then demotes the destination to 'pending' and commits; rm resumes,
+# its upsert applies the move against the row it now locks, and the post-move re-check — a fresh
+# statement, fresh snapshot — sees 'pending' and RAISES 42501, aborting rm. FINAL: the machine is
+# still in its ORIGINAL active domain.
+#
+# P2 — the twin. The same move with NO concurrent demotion SUCCEEDS and lands. Without it, a guard
+# that refused every move would satisfy P1 and this spec would be scoring an over-refusal as a
+# fix. P2 leaves the machine in the destination domain, so it runs after P1's fixture reset (each
+# permutation re-runs setup).
+#
+# WHAT EACH PERMUTATION PINS, and the mutation that falsifies it. Every refusal case carries a twin
+# so a guard that refused everything could not pass this spec.
+#
+#   P1 / P2            destination demoted during the wait → refused; uncontended move → succeeds
+#   P3 / P3-twin       a concurrent session RELOCATES the machine, then demotes → refused;
+#                      the relocation alone → the equal-rank move succeeds
+#   P4 / P4-twin       the LOCK HOLDER relocates, then demotes → refused; relocation alone → succeeds
+#   P5 / P5-reverse    register and reclaim serialize on the machine row in both starting orders
+#                      (see the rc session for what these do NOT show)
+#   P6 / P6-promote /  the first-registration race: the loser reaches the conflict path with no row
+#   P6-control         to lock, so it is REFUSED 40001 whether the destination is demoted (P6) or the
+#                      SOURCE is promoted (P6-promote); serialized, the same two registrations
+#                      succeed (P6-control)
+#
+# MUTATION SIGNALS, all four measured:
+#   * delete the post-move comparison            → P1, P3, P4 no longer raise; the checker shows the
+#                                                  machine 'active' in a 'pending' domain
+#   * remove the pre-read's FOR UPDATE           → P3, P4 no longer raise (the stale-pre-read bypass)
+#   * delete the lost-race (40001) refusal       → P6, P6-promote no longer raise
+#   * inject the #158-shaped fleet_trust_domains
+#     FOR UPDATE into register                   → P1, P3, P4 become `ERROR: deadlock detected`.
+#     THIS is the evidence that the machine-only lock avoids ab38805's revert; P5/P5-reverse stay
+#     GREEN under that injection and were mis-cited for it in an earlier round.
+
+setup
+{
+  insert into agent_board.agents(id, agent_type, handle, display_name, status) values
+    ('a3970000-0000-4000-8000-000000000001', 'agent', 'nodowngrade-iso-owner', 'No-downgrade Iso Owner', 'active'),
+    ('a3970000-0000-4000-8000-000000000002', 'agent', 'nodowngrade-iso-operator', 'No-downgrade Iso Operator', 'active'),
+    ('a3970000-0000-4000-8000-000000000003', 'agent', 'nodowngrade-iso-newowner', 'No-downgrade Iso New Owner', 'active');
+
+  -- Two ACTIVE domains owned by that agent: the machine's current one, and the move's destination.
+  insert into agent_board.fleet_trust_domains(id, slug, display_name, domain_kind, owner_agent_id, lifecycle_state, approved_at)
+  values ('d3970000-0000-4000-8000-000000000001', 'nodowngrade-iso-current', 'No-downgrade Iso Current', 'tailnet',
+          'a3970000-0000-4000-8000-000000000001', 'active', now()),
+         ('d3970000-0000-4000-8000-000000000002', 'nodowngrade-iso-dest', 'No-downgrade Iso Destination', 'tailnet',
+          'a3970000-0000-4000-8000-000000000001', 'active', now()),
+         -- X: a third ACTIVE domain the concurrent session RELOCATES the machine into (#397 round 2).
+         ('d3970000-0000-4000-8000-000000000003', 'nodowngrade-iso-x', 'No-downgrade Iso X', 'tailnet',
+          'a3970000-0000-4000-8000-000000000001', 'active', now()),
+         -- The reclaim probe's destination: active, owned by the arbitrated NEW owner.
+         ('d3970000-0000-4000-8000-000000000004', 'nodowngrade-iso-newdom', 'No-downgrade Iso New Owner Domain', 'tailnet',
+          'a3970000-0000-4000-8000-000000000003', 'active', now()),
+         -- P6 family: the first-registration race runs between two PENDING domains, which is the
+         -- shape the source-promotion case needs (#397 round 3, finding 1).
+         ('d3970000-0000-4000-8000-000000000005', 'nodowngrade-iso-race-src', 'No-downgrade Iso Race Source', 'tailnet',
+          'a3970000-0000-4000-8000-000000000001', 'pending', null),
+         ('d3970000-0000-4000-8000-000000000006', 'nodowngrade-iso-race-dest', 'No-downgrade Iso Race Destination', 'tailnet',
+          'a3970000-0000-4000-8000-000000000001', 'pending', null);
+
+  -- The bound-shaped machine the move would relocate. Inserted directly so its id is fixed.
+  insert into agent_board.fleet_machines(id, trust_domain_id, owner_agent_id, machine_label, machine_fingerprint,
+                                         lifecycle_state, enrollment_source, enrolled_at, metadata)
+  values ('c3970000-0000-4000-8000-000000000001', 'd3970000-0000-4000-8000-000000000001',
+          'a3970000-0000-4000-8000-000000000001', 'nodowngrade-iso-machine', 'sha256:nodowngrade-iso',
+          'active', 'portal', now(), jsonb_build_object('redacted', true, 'metadata-safe', true));
+}
+
+teardown
+{
+  delete from agent_board.fleet_machines where id = 'c3970000-0000-4000-8000-000000000001';
+  delete from agent_board.fleet_machines where machine_fingerprint = 'sha256:nodowngrade-iso-fresh';
+  delete from agent_board.fleet_trust_domains where id in (
+    'd3970000-0000-4000-8000-000000000001',
+    'd3970000-0000-4000-8000-000000000002',
+    'd3970000-0000-4000-8000-000000000003',
+    'd3970000-0000-4000-8000-000000000004',
+    'd3970000-0000-4000-8000-000000000005',
+    'd3970000-0000-4000-8000-000000000006');
+  delete from agent_board.agents where id in (
+    'a3970000-0000-4000-8000-000000000001',
+    'a3970000-0000-4000-8000-000000000002',
+    'a3970000-0000-4000-8000-000000000003');
+}
+
+# rl: holds the machine row (any concurrent writer of that row will do — reclaim, revoke and
+# reactivate all take it) and demotes the destination domain before releasing it.
+session "rl"
+step "rl_begin"  { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "rl_lock"   {
+  select id from agent_board.fleet_machines
+   where id = 'c3970000-0000-4000-8000-000000000001' for update;
+}
+step "rl_demote" {
+  update agent_board.fleet_trust_domains
+     set lifecycle_state = 'pending', approved_at = null, updated_at = now()
+   where id = 'd3970000-0000-4000-8000-000000000002';
+}
+step "rl_relocate_dest" {
+  update agent_board.fleet_machines
+     set trust_domain_id = 'd3970000-0000-4000-8000-000000000002', updated_at = now()
+   where id = 'c3970000-0000-4000-8000-000000000001';
+}
+step "rl_relocate_x" {
+  update agent_board.fleet_machines
+     set trust_domain_id = 'd3970000-0000-4000-8000-000000000003', updated_at = now()
+   where id = 'c3970000-0000-4000-8000-000000000001';
+}
+step "rl_promote_race_src" {
+  update agent_board.fleet_trust_domains
+     set lifecycle_state = 'active', approved_at = now(), updated_at = now()
+   where id = 'd3970000-0000-4000-8000-000000000005';
+}
+step "rl_demote_race_dest" {
+  update agent_board.fleet_trust_domains
+     set lifecycle_state = 'quarantined', approved_at = null, updated_at = now()
+   where id = 'd3970000-0000-4000-8000-000000000006';
+}
+step "rl_demote_current" {
+  update agent_board.fleet_trust_domains
+     set lifecycle_state = 'pending', approved_at = null, updated_at = now()
+   where id = 'd3970000-0000-4000-8000-000000000001';
+}
+step "rl_commit" { COMMIT; }
+
+# rm: the owner re-registering their own fingerprint into the destination domain — the #395 move.
+session "rm"
+step "rm_begin"  { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "rm_move"   {
+  select trust_domain_id
+    from agent_board.register_fleet_machine(
+      'a3970000-0000-4000-8000-000000000001', 'd3970000-0000-4000-8000-000000000002',
+      'nodowngrade-iso-machine-moved', 'sha256:nodowngrade-iso', 'portal',
+      jsonb_build_object('redacted', true));
+}
+step "rm_move_current"   {
+  select trust_domain_id
+    from agent_board.register_fleet_machine(
+      'a3970000-0000-4000-8000-000000000001', 'd3970000-0000-4000-8000-000000000001',
+      'nodowngrade-iso-machine-relabelled', 'sha256:nodowngrade-iso', 'portal',
+      jsonb_build_object('redacted', true));
+}
+step "rm_commit" { COMMIT; }
+
+# rc: the operator reclaim. It locks the machine row and then takes an FK KEY SHARE on the
+# destination domain — machine ≺ domain, the SAME acquisition order register takes.
+#
+# WHAT P5/P5-reverse DO AND DO NOT SHOW, corrected after a review seat pointed out that the first
+# version of this comment claimed the stronger thing. They show that register and reclaim SERIALIZE
+# on the machine row in both starting orders and that neither wedges. They do NOT reproduce the
+# historical domain↔machine cycle: they target different domains, and each RPC only starts after
+# the other has RETURNED, so no session is ever mid-call holding one lock and waiting for the other.
+# Measured, not assumed: injecting the #158-shaped fleet_trust_domains FOR UPDATE leaves BOTH of
+# them green. The permutations that actually falsify that lock are P1/P3/P4 — see the header.
+session "rc"
+step "rc_begin"   { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "rc_reclaim" {
+  select owner_agent_id, trust_domain_id
+    from agent_board.reclaim_fleet_machine(
+      'c3970000-0000-4000-8000-000000000001', 'a3970000-0000-4000-8000-000000000003',
+      'a3970000-0000-4000-8000-000000000002', 'iso reclaim vs register',
+      'd3970000-0000-4000-8000-000000000004');
+}
+step "rc_commit"  { COMMIT; }
+
+# rf: the session that WINS a first-registration race on a fingerprint no row exists for yet.
+# The loser's pre-read legitimately finds nothing, so v_prior_domain is null and the skip predicate
+# has no baseline — the one conflict path the locked pre-read cannot cover.
+session "rf"
+step "rf_begin"    { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "rf_register" {
+  select trust_domain_id
+    from agent_board.register_fleet_machine(
+      'a3970000-0000-4000-8000-000000000001', 'd3970000-0000-4000-8000-000000000005',
+      'nodowngrade-iso-fresh-winner', 'sha256:nodowngrade-iso-fresh', 'portal',
+      jsonb_build_object('redacted', true));
+}
+step "rf_commit"   { COMMIT; }
+
+# rm2: the loser of that race. Its insert blocks on the unique index, then takes the CONFLICT path
+# against a row it never read.
+session "rm2"
+step "rm2_begin"    { BEGIN; SET LOCAL "request.jwt.claim.role" = 'service_role'; }
+step "rm2_register" {
+  select trust_domain_id
+    from agent_board.register_fleet_machine(
+      'a3970000-0000-4000-8000-000000000001', 'd3970000-0000-4000-8000-000000000006',
+      'nodowngrade-iso-fresh-loser', 'sha256:nodowngrade-iso-fresh', 'portal',
+      jsonb_build_object('redacted', true));
+}
+step "rm2_commit"   { COMMIT; }
+
+# checker for the raced fingerprint.
+session "chkf"
+step "chkf" {
+  select fm.trust_domain_id, fm.lifecycle_state as machine_state, ftd.lifecycle_state as domain_state
+    from agent_board.fleet_machines fm
+    join agent_board.fleet_trust_domains ftd on ftd.id = fm.trust_domain_id
+   where fm.machine_fingerprint = 'sha256:nodowngrade-iso-fresh';
+}
+
+# checker: where the machine ended up, and the lifecycle of the domain it is in.
+session "chk"
+step "chk" {
+  select fm.trust_domain_id, fm.owner_agent_id, fm.lifecycle_state as machine_state,
+         ftd.lifecycle_state as domain_state
+    from agent_board.fleet_machines fm
+    join agent_board.fleet_trust_domains ftd on ftd.id = fm.trust_domain_id
+   where fm.id = 'c3970000-0000-4000-8000-000000000001';
+}
+
+# P1: the move blocks, the destination is demoted under it, and the post-move re-check refuses.
+permutation "rl_begin" "rl_lock" "rm_begin" "rm_move" "rl_demote" "rl_commit" "rm_commit" "chk"
+
+# P2 (twin): the same move with no concurrent demotion still succeeds and lands.
+permutation "rm_begin" "rm_move" "rm_commit" "chk"
+
+# P3 (#397 round 2, case A — the concurrent session RELOCATES the machine). rl moves the machine
+# D1→D2 uncommitted; rm re-registers into D1, so rm's prior-read sees the COMMITTED D1 and records
+# v_prior_domain = D1; rl then demotes D1 and commits; rm resumes against the machine row as it now
+# is (D2) and moves it D2→D1. The destination it lands in EQUALS the stale v_prior_domain, so a
+# skip predicate built on that pre-read reads "nothing moved" and skips the fresh-snapshot check —
+# leaving an active machine in a pending domain. MUST refuse.
+permutation "rl_begin" "rl_relocate_dest" "rm_begin" "rm_move_current" "rl_demote_current" "rl_commit" "rm_commit" "chk"
+
+# P3-twin: the same relocation with NO demotion. D1 is still active, so the move back into it is a
+# legitimate equal-rank relocation and MUST succeed — an over-refusing guard fails here.
+permutation "rl_begin" "rl_relocate_dest" "rm_begin" "rm_move_current" "rl_commit" "rm_commit" "chk"
+
+# P4 (#397 round 2, case B — the LOCK HOLDER relocates). rm re-registers into the machine's OWN
+# domain D1 (a label update, so both the destination gate and the rank pass and v_prior_domain =
+# D1); rl holds the machine row, relocates it to X, demotes D1, and commits; rm resumes and the
+# upsert genuinely moves the machine X→D1, which is now pending. Same stale-predicate skip.
+# rm blocks at the UPSERT, not at the prior-read — a plain SELECT does not wait, which is why the
+# pre-read is stale rather than merely early. MUST refuse.
+permutation "rl_begin" "rl_lock" "rm_begin" "rm_move_current" "rl_relocate_x" "rl_demote_current" "rl_commit" "rm_commit" "chk"
+
+# P4-twin: the same, without the demotion — X→D1 is equal-rank and MUST succeed.
+permutation "rl_begin" "rl_lock" "rm_begin" "rm_move_current" "rl_relocate_x" "rl_commit" "rm_commit" "chk"
+
+# P5 (lock order). register's machine-row FOR UPDATE is the SAME row reclaim_fleet_machine locks,
+# and both then reach fleet_trust_domains only through the FK — machine ≺ domain in both, so no
+# cycle. Evidence, not assertion: reclaim holds the machine row, register blocks, reclaim commits,
+# register resumes and is refused by #141 (the machine now belongs to the new owner). No 40P01.
+permutation "rc_begin" "rc_reclaim" "rm_begin" "rm_move_current" "rc_commit" "rm_commit" "chk"
+
+# P5-reverse: the other acquisition order. register takes the machine row first and holds it to
+# commit; reclaim blocks and completes afterwards. Still no cycle.
+permutation "rm_begin" "rm_move_current" "rc_begin" "rc_reclaim" "rm_commit" "rc_commit" "chk"
+
+# P6 (the first-registration race — the one conflict path the locked pre-read cannot cover). Both
+# race domains are PENDING. rf first-registers the fingerprint into SRC and holds it uncommitted;
+# rm2 registers the SAME fingerprint into DEST and blocks on the unique index; DEST is demoted to
+# quarantined (rank 0, below pending) and committed while rm2 waits; rf commits; rm2 resumes onto
+# the CONFLICT path against a row it never read and never locked. There is no recoverable baseline
+# — the domain the machine was in belonged to a row we never read, and the upsert has overwritten
+# it — so this is REFUSED as a serialization failure (40001, retryable) rather than approximated.
+permutation "rf_begin" "rf_register" "rm2_begin" "rm2_register" "rl_begin" "rl_demote_race_dest" "rl_commit" "rf_commit" "rm2_commit" "chkf"
+
+# P6-promote: THE ROUND-3 PANEL'S CASE, in its own shape. Nothing is demoted — the SOURCE is
+# PROMOTED to active while both racers are in flight, so rm2 would move the machine OUT of a
+# now-active domain INTO a pending one. Round 2's fallback compared the cached destination rank
+# against itself (1 >= 1) and passed this; the refusal must come from the ABSENT BASELINE, which is
+# why it does not depend on any rank comparison at all.
+permutation "rf_begin" "rf_register" "rm2_begin" "rm2_register" "rl_begin" "rl_promote_race_src" "rl_commit" "rf_commit" "rm2_commit" "chkf"
+
+# P6-control (the twin): the SAME two registrations, SERIALIZED — rf commits before rm2 starts, so
+# rm2 has a row to lock and a real baseline. Both domains are pending, so its move is equal-rank and
+# MUST succeed and land in DEST. Without this, a guard that refused every second registration would
+# satisfy P6 and P6-promote.
+permutation "rf_begin" "rf_register" "rf_commit" "rm2_begin" "rm2_register" "rm2_commit" "chkf"