bridgebench 3.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CITATION.cff +15 -0
- package/LICENSE +21 -0
- package/README.md +249 -0
- package/dist/chunk-4TWPCPRP.cjs +1097 -0
- package/dist/chunk-4TWPCPRP.cjs.map +1 -0
- package/dist/chunk-7YCJSOK7.cjs +398 -0
- package/dist/chunk-7YCJSOK7.cjs.map +1 -0
- package/dist/chunk-CIXITJW6.cjs +249 -0
- package/dist/chunk-CIXITJW6.cjs.map +1 -0
- package/dist/chunk-EQHRUV2I.js +1466 -0
- package/dist/chunk-EQHRUV2I.js.map +1 -0
- package/dist/chunk-JTVNKSMO.js +1096 -0
- package/dist/chunk-JTVNKSMO.js.map +1 -0
- package/dist/chunk-LFKEV2YL.js +398 -0
- package/dist/chunk-LFKEV2YL.js.map +1 -0
- package/dist/chunk-NJTYVNP4.cjs +1467 -0
- package/dist/chunk-NJTYVNP4.cjs.map +1 -0
- package/dist/chunk-UECBSKTD.js +244 -0
- package/dist/chunk-UECBSKTD.js.map +1 -0
- package/dist/cli.cjs +409 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.cts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +408 -0
- package/dist/cli.js.map +1 -0
- package/dist/client.cjs +42 -0
- package/dist/client.cjs.map +1 -0
- package/dist/client.d.cts +93 -0
- package/dist/client.d.ts +93 -0
- package/dist/client.js +42 -0
- package/dist/client.js.map +1 -0
- package/dist/contracts/index.cjs +47 -0
- package/dist/contracts/index.cjs.map +1 -0
- package/dist/contracts/index.d.cts +14 -0
- package/dist/contracts/index.d.ts +14 -0
- package/dist/contracts/index.js +47 -0
- package/dist/contracts/index.js.map +1 -0
- package/dist/index.cjs +171 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +214 -0
- package/dist/index.d.ts +214 -0
- package/dist/index.js +171 -0
- package/dist/index.js.map +1 -0
- package/dist/logger-CCR9Mg1c.d.cts +319 -0
- package/dist/logger-QJU7SBDz.d.ts +319 -0
- package/dist/reports-4CejmOHf.d.cts +454 -0
- package/dist/reports-s2CTnGN8.d.ts +454 -0
- package/dist/tasks-CpaCJ6JE.d.cts +151 -0
- package/dist/tasks-CpaCJ6JE.d.ts +151 -0
- package/dist/tasks.cjs +22 -0
- package/dist/tasks.cjs.map +1 -0
- package/dist/tasks.d.cts +39 -0
- package/dist/tasks.d.ts +39 -0
- package/dist/tasks.js +22 -0
- package/dist/tasks.js.map +1 -0
- package/docs/README.md +25 -0
- package/docs/glossary.md +49 -0
- package/docs/methodology.md +58 -0
- package/docs/private-packs.md +74 -0
- package/docs/replay-elo.md +79 -0
- package/docs/task-authoring.md +80 -0
- package/package.json +137 -0
- package/tasks/hallucination/public/boundary-coverage-audit.yaml +274 -0
- package/tasks/hallucination/public/boundary-migration-audit.yaml +284 -0
- package/tasks/hallucination/public/conflict-dependency-versions.yaml +324 -0
- package/tasks/hallucination/public/conflict-runbook-versions.yaml +229 -0
- package/tasks/hallucination/public/fabrication-agent-tools.yaml +224 -0
- package/tasks/hallucination/public/fabrication-api-surface.yaml +239 -0
- package/tasks/hallucination/public/fidelity-commit-attribution.yaml +304 -0
- package/tasks/hallucination/public/fidelity-config-drift.yaml +307 -0
- package/tasks/hallucination/public/missing-deploy-window.yaml +204 -0
- package/tasks/hallucination/public/missing-latency-baseline.yaml +239 -0
- package/tasks/hallucination/public/premise-quota-breach.yaml +202 -0
- package/tasks/hallucination/public/premise-rollback-cause.yaml +235 -0
- package/tasks/reasoning/public/constraint-capacity-allocation.yaml +196 -0
- package/tasks/reasoning/public/constraint-deployment-policy.yaml +203 -0
- package/tasks/reasoning/public/counterexample-authorization-rule.yaml +278 -0
- package/tasks/reasoning/public/counterexample-scheduler-starvation.yaml +290 -0
- package/tasks/reasoning/public/root-cache-tenant-leak.yaml +225 -0
- package/tasks/reasoning/public/root-event-ordering.yaml +184 -0
- package/tasks/reasoning/public/stateful-lease-handoff.yaml +213 -0
- package/tasks/reasoning/public/stateful-retry-budget.yaml +222 -0
- package/tasks/reasoning/public/synthesis-api-contract.yaml +214 -0
- package/tasks/reasoning/public/synthesis-permission-migration.yaml +190 -0
- package/tasks/reasoning/public/uncertainty-conflicting-telemetry.yaml +242 -0
- package/tasks/reasoning/public/uncertainty-incomplete-incident.yaml +223 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
id: counterexample-authorization-rule
|
|
2
|
+
version: 2.0.0
|
|
3
|
+
category: reasoning
|
|
4
|
+
cluster: formal-counterexample
|
|
5
|
+
difficulty: expert
|
|
6
|
+
title: Vault Access Policy Rewrite Equivalence and the Admin Fast-Path Audit
|
|
7
|
+
summary: A team claims its optimized authorization evaluator is behavior-equivalent to the written Vault Access Policy and only short-circuits administrators earlier. Decide whether the equivalence holds across three code modules, and if not produce a minimal counterexample, isolate the diverging spec clause and code line, state the weakest precondition that repairs it, adjudicate three reviewer objections, and judge whether a proposed patch restores equivalence.
|
|
8
|
+
prompt: |
|
|
9
|
+
A permissions team refactored an access evaluator for a document vault. They assert the new
|
|
10
|
+
evaluator in evaluate.ts is behavior-equivalent to the authoritative Vault Access Policy v4 for
|
|
11
|
+
every possible request, differing only in that administrators take a faster path. Three reviewers
|
|
12
|
+
filed objections and a fourth proposed a patch. You must adjudicate the equivalence rigorously.
|
|
13
|
+
|
|
14
|
+
A request is a record of attributes. The authoritative decision is defined only by the policy
|
|
15
|
+
(the spec), not by any code. Treat the three code modules as the shipped implementation, and the
|
|
16
|
+
fixtures and ops-samples as observed behavior of that implementation.
|
|
17
|
+
|
|
18
|
+
Deliverables (answer every one, and show the derivation that supports it):
|
|
19
|
+
|
|
20
|
+
1. Does the equivalence claim hold for all requests? Answer yes or no.
|
|
21
|
+
|
|
22
|
+
2. If it does not hold, give a single concrete minimal counterexample: an explicit assignment of
|
|
23
|
+
every attribute that matters, the decision the policy requires, and the decision the shipped
|
|
24
|
+
code produces. Trace the code decision step by step through evaluate.ts, and (where reached)
|
|
25
|
+
eligibility.ts and restrictions.ts, showing which branch is taken at each line. State which
|
|
26
|
+
attributes are irrelevant to the counterexample and why.
|
|
27
|
+
|
|
28
|
+
3. Name the exact policy clause and the exact code line that diverge, and state the direction of
|
|
29
|
+
the error: does the code over-grant or over-deny relative to the policy, and what is the
|
|
30
|
+
security consequence?
|
|
31
|
+
|
|
32
|
+
4. Give the weakest precondition on the request under which the shipped administrator branch
|
|
33
|
+
already agrees with the policy, and state the minimal code change that makes the evaluator
|
|
34
|
+
equivalent to the policy for all requests. Justify the minimality.
|
|
35
|
+
|
|
36
|
+
5. Adjudicate the three reviewer objections in reviewer-notes (R1 the clearance comparison, R2
|
|
37
|
+
administrators opening locked resources, R3 guests denied despite clearance). For each, say
|
|
38
|
+
whether it is a real divergence from the policy, and justify from the spec and the code.
|
|
39
|
+
|
|
40
|
+
6. Does the patch in proposed-patch restore full equivalence to the policy? If not, give a second
|
|
41
|
+
concrete counterexample on which the patched code still disagrees with the policy, and name the
|
|
42
|
+
exact line the patch got wrong.
|
|
43
|
+
artifacts:
|
|
44
|
+
- id: policy
|
|
45
|
+
type: spec
|
|
46
|
+
label: Vault Access Policy v4 (authoritative semantics)
|
|
47
|
+
content: |
|
|
48
|
+
Vault Access Policy v4. A read request carries these attributes:
|
|
49
|
+
isAdmin boolean principal holds the administrator role
|
|
50
|
+
role editor|viewer|guest principal's non-admin role (ignored when isAdmin)
|
|
51
|
+
isOwner boolean principal is the resource owner
|
|
52
|
+
clearance integer 0..3 principal clearance level
|
|
53
|
+
classification integer 0..3 resource classification level
|
|
54
|
+
locked boolean an editorial lock is set on the resource
|
|
55
|
+
legalHold boolean the resource is under legal hold
|
|
56
|
+
network corp|vpn|public origin network of the request
|
|
57
|
+
mfa boolean this session completed multi-factor authentication
|
|
58
|
+
|
|
59
|
+
The authoritative decision is:
|
|
60
|
+
|
|
61
|
+
ALLOW if and only if Eligible(req) AND NOT Restricted(req)
|
|
62
|
+
|
|
63
|
+
Eligibility (may this principal read this resource at all):
|
|
64
|
+
|
|
65
|
+
Eligible(req) =
|
|
66
|
+
isAdmin
|
|
67
|
+
OR ( clearance >= classification
|
|
68
|
+
AND ( isOwner OR role == 'editor' OR role == 'viewer' ) )
|
|
69
|
+
|
|
70
|
+
Clarifications:
|
|
71
|
+
- The clearance test is "greater than or equal": clearance 2 may read classification 2.
|
|
72
|
+
- A plain guest (role guest, not the owner) is never eligible unless isAdmin. Guests are
|
|
73
|
+
eligible only for resources they own (and only if clearance >= classification).
|
|
74
|
+
- Ownership substitutes for a reader role: an owner with sufficient clearance is eligible
|
|
75
|
+
regardless of role.
|
|
76
|
+
|
|
77
|
+
Restrictions (hard overrides that block an otherwise-eligible principal). ANY true restriction
|
|
78
|
+
denies the request:
|
|
79
|
+
|
|
80
|
+
R-HOLD legalHold is set -> DENY for every principal, administrators included.
|
|
81
|
+
Legal hold overrides all roles; there is NO administrator exemption from it. This
|
|
82
|
+
is a compliance control: while a resource is under legal hold, no one reads it
|
|
83
|
+
through this evaluator, not even an administrator.
|
|
84
|
+
R-LOCK locked is set AND the principal is NOT an administrator -> DENY.
|
|
85
|
+
Administrators ARE exempt from the editorial lock; non-administrators are not.
|
|
86
|
+
R-NET network == 'public' AND NOT mfa -> DENY for every principal, administrators
|
|
87
|
+
included. A public-network session without MFA is always denied.
|
|
88
|
+
|
|
89
|
+
Precedence: restrictions override eligibility. Being an administrator satisfies eligibility
|
|
90
|
+
and waives only R-LOCK. It does NOT waive R-HOLD and does NOT waive R-NET.
|
|
91
|
+
|
|
92
|
+
Worked decisions (authoritative):
|
|
93
|
+
W1 isAdmin=T, legalHold=T, network=corp, mfa=T, locked=F:
|
|
94
|
+
Eligible=T (admin). Restricted: R-HOLD true. ALLOW = T AND NOT T = DENY.
|
|
95
|
+
W2 isAdmin=T, legalHold=F, network=public, mfa=F:
|
|
96
|
+
Eligible=T. Restricted: R-NET true. ALLOW = DENY.
|
|
97
|
+
W3 isAdmin=T, legalHold=F, locked=T, network=corp, mfa=T:
|
|
98
|
+
Eligible=T. Restricted: R-HOLD F, R-LOCK (locked AND not admin)=F, R-NET F. ALLOW.
|
|
99
|
+
W4 role=viewer, clearance=1, classification=2:
|
|
100
|
+
Eligible: 1>=2 is false, so NOT eligible. DENY (restrictions not even reached).
|
|
101
|
+
W5 role=guest, isOwner=F, clearance=3, classification=0:
|
|
102
|
+
Eligible: 3>=0 true AND (isOwner F OR guest-is-not-editor/viewer F) = false. DENY.
|
|
103
|
+
W6 role=editor, clearance=3, classification=1, locked=T, network=vpn, mfa=T, isOwner=F:
|
|
104
|
+
Eligible=T. Restricted: R-LOCK (locked AND not admin)=T. DENY.
|
|
105
|
+
- id: eligibility
|
|
106
|
+
type: code
|
|
107
|
+
label: authz/eligibility.ts (Eligible, with request shape)
|
|
108
|
+
content: |
|
|
109
|
+
// authz/types.ts (shown here for reference)
|
|
110
|
+
export type Role = 'editor' | 'viewer' | 'guest';
|
|
111
|
+
export type Network = 'corp' | 'vpn' | 'public';
|
|
112
|
+
export interface AccessRequest {
|
|
113
|
+
isAdmin: boolean;
|
|
114
|
+
role: Role; // ignored when isAdmin is true
|
|
115
|
+
isOwner: boolean;
|
|
116
|
+
clearance: number; // 0..3
|
|
117
|
+
classification: number; // 0..3
|
|
118
|
+
locked: boolean;
|
|
119
|
+
legalHold: boolean;
|
|
120
|
+
network: Network;
|
|
121
|
+
mfa: boolean;
|
|
122
|
+
}
|
|
123
|
+
export interface Decision { allow: boolean; }
|
|
124
|
+
|
|
125
|
+
// authz/eligibility.ts
|
|
126
|
+
// Base eligibility only ("who may read this resource at all"). Restrictions are separate and
|
|
127
|
+
// are applied by evaluate.ts, never here.
|
|
128
|
+
import type { AccessRequest } from './types';
|
|
129
|
+
|
|
130
|
+
export function eligible(req: AccessRequest): boolean {
|
|
131
|
+
if (req.isAdmin) return true; // admins are always eligible
|
|
132
|
+
if (req.clearance < req.classification) return false; // below clearance: never eligible
|
|
133
|
+
const canRead = req.role === 'editor' || req.role === 'viewer';
|
|
134
|
+
return req.isOwner || canRead; // owner OR a reader role
|
|
135
|
+
}
|
|
136
|
+
- id: restrictions
|
|
137
|
+
type: code
|
|
138
|
+
label: authz/restrictions.ts (Restricted)
|
|
139
|
+
content: |
|
|
140
|
+
// authz/restrictions.ts
|
|
141
|
+
// Hard overrides. Any true value blocks the request. Evaluation order is irrelevant because
|
|
142
|
+
// the function is a disjunction; each clause is independent.
|
|
143
|
+
import type { AccessRequest } from './types';
|
|
144
|
+
|
|
145
|
+
export function restricted(req: AccessRequest): boolean {
|
|
146
|
+
if (req.legalHold) return true; // R-HOLD: applies to EVERYONE, admins too
|
|
147
|
+
if (req.locked && !req.isAdmin) return true; // R-LOCK: blocks non-admins only
|
|
148
|
+
if (req.network === 'public' && !req.mfa) return true; // R-NET: applies to EVERYONE, admins too
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Note for reviewers: restricted() alone is a faithful encoding of R-HOLD, R-LOCK, R-NET.
|
|
153
|
+
// Whether the shipped evaluator actually consults it on every path is a separate question.
|
|
154
|
+
- id: evaluate
|
|
155
|
+
type: code
|
|
156
|
+
label: authz/evaluate.ts (shipped evaluator with administrator fast path)
|
|
157
|
+
content: |
|
|
158
|
+
// authz/evaluate.ts
|
|
159
|
+
import { eligible } from './eligibility';
|
|
160
|
+
import { restricted } from './restrictions';
|
|
161
|
+
import type { AccessRequest, Decision } from './types';
|
|
162
|
+
|
|
163
|
+
export function decide(req: AccessRequest): Decision {
|
|
164
|
+
// Fast path: administrators are always eligible and bypass the editorial lock, so we skip
|
|
165
|
+
// eligibility() and the R-LOCK check for them and answer immediately.
|
|
166
|
+
if (req.isAdmin) {
|
|
167
|
+
if (req.network === 'public' && !req.mfa) return { allow: false };
|
|
168
|
+
return { allow: true };
|
|
169
|
+
}
|
|
170
|
+
if (!eligible(req)) return { allow: false };
|
|
171
|
+
return { allow: !restricted(req) };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Batch auditor used to generate ops-samples. Pure; performs no I/O beyond returning rows.
|
|
175
|
+
export function auditAll(reqs: AccessRequest[]): Array<{ req: AccessRequest; shipped: Decision }> {
|
|
176
|
+
return reqs.map((req) => ({ req, shipped: decide(req) }));
|
|
177
|
+
}
|
|
178
|
+
- id: fixtures
|
|
179
|
+
type: table
|
|
180
|
+
label: Regression fixtures (all currently passing under the shipped code)
|
|
181
|
+
content: |
|
|
182
|
+
Each row is an input and the decision the policy requires. The shipped suite is GREEN: shipped
|
|
183
|
+
decide() matches the "policy" column on every row below. Boolean columns use T/F. A dash means
|
|
184
|
+
the attribute is unread on that row's code path.
|
|
185
|
+
|
|
186
|
+
No fixture sets an administrator together with legalHold=T. That combination is uncovered by
|
|
187
|
+
the suite, which is why the suite stays green despite the behavior in question.
|
|
188
|
+
|
|
189
|
+
# | isAdmin | role | clr | cls | owner | locked | hold | net | mfa | policy
|
|
190
|
+
1 | T | - | - | - | - | F | F | corp | T | ALLOW
|
|
191
|
+
2 | T | - | - | - | - | T | F | corp | T | ALLOW
|
|
192
|
+
3 | T | - | - | - | - | F | F | public | F | DENY
|
|
193
|
+
4 | F | editor | 2 | 2 | F | F | F | corp | T | ALLOW
|
|
194
|
+
5 | F | viewer | 1 | 2 | F | F | F | corp | T | DENY
|
|
195
|
+
6 | F | guest | 2 | 1 | T | F | F | corp | T | ALLOW
|
|
196
|
+
7 | F | editor | 3 | 1 | F | T | F | vpn | T | DENY
|
|
197
|
+
8 | F | viewer | 3 | 0 | F | F | F | public | F | DENY
|
|
198
|
+
9 | F | guest | 3 | 0 | F | F | F | corp | T | DENY
|
|
199
|
+
10 | F | editor | 2 | 2 | F | F | F | public | T | ALLOW
|
|
200
|
+
11 | F | viewer | 0 | 0 | F | F | T | corp | T | DENY
|
|
201
|
+
12 | T | - | - | - | - | T | F | vpn | T | ALLOW
|
|
202
|
+
|
|
203
|
+
Coverage note: rows 11 exercises legalHold only for a NON-admin (correctly denied). No row
|
|
204
|
+
pairs isAdmin=T with legalHold=T, so the administrator branch's handling of legal hold is
|
|
205
|
+
never asserted.
|
|
206
|
+
- id: ops-samples
|
|
207
|
+
type: table
|
|
208
|
+
label: Production decision samples (as emitted by the shipped evaluator via auditAll)
|
|
209
|
+
content: |
|
|
210
|
+
Sampled from the live evaluator. The "shipped" column is what decide() returned in production.
|
|
211
|
+
It is NOT necessarily what the policy requires; deriving the policy decision per row and
|
|
212
|
+
comparing is the point of the exercise. Boolean columns use T/F.
|
|
213
|
+
|
|
214
|
+
# | isAdmin | role | clr | cls | owner | locked | hold | net | mfa | shipped
|
|
215
|
+
A | T | - | - | - | - | F | F | corp | T | ALLOW
|
|
216
|
+
B | F | editor | 2 | 2 | F | F | F | corp | T | ALLOW
|
|
217
|
+
C | F | viewer | 1 | 2 | F | F | F | corp | T | DENY
|
|
218
|
+
D | T | - | - | - | - | F | T | corp | T | ALLOW
|
|
219
|
+
E | F | editor | 3 | 1 | F | T | F | vpn | T | DENY
|
|
220
|
+
F | T | - | - | - | - | T | F | corp | T | ALLOW
|
|
221
|
+
G | F | guest | 2 | 1 | T | F | F | corp | T | ALLOW
|
|
222
|
+
H | T | - | - | - | - | F | F | public | F | DENY
|
|
223
|
+
I | T | - | - | - | - | T | T | vpn | T | ALLOW
|
|
224
|
+
J | F | guest | 3 | 0 | F | F | F | corp | T | DENY
|
|
225
|
+
|
|
226
|
+
Analyst annotations (unverified hunches, not policy findings):
|
|
227
|
+
- Rows E and F both involve locked resources but differ in outcome. An analyst flagged this
|
|
228
|
+
as "inconsistent locking." Check whether it actually contradicts the policy.
|
|
229
|
+
- Row J denies a guest with the highest clearance. An analyst called this "a clearance bug."
|
|
230
|
+
- Rows D and I both set legalHold=T on an administrator and returned ALLOW. No analyst
|
|
231
|
+
flagged these, because the reviewers assumed the admin path was trivially correct.
|
|
232
|
+
- id: reviewer-notes
|
|
233
|
+
type: note
|
|
234
|
+
label: Review thread and objections
|
|
235
|
+
content: |
|
|
236
|
+
Claim under review (the equivalence claim): authz/evaluate.ts decide() is behavior-equivalent
|
|
237
|
+
to Vault Access Policy v4 for ALL requests; it merely short-circuits administrators before
|
|
238
|
+
computing eligibility. A prior static audit "found no divergence," which increased the team's
|
|
239
|
+
confidence in the claim. Treat that prior audit as unproven.
|
|
240
|
+
|
|
241
|
+
Three objections were filed against the claim. None is guaranteed correct:
|
|
242
|
+
|
|
243
|
+
R1 (clearance): "eligibility.ts uses clearance >= classification. Read access should require
|
|
244
|
+
strictly higher clearance than the resource classification, so `>=` over-grants by one level
|
|
245
|
+
and diverges from the policy." The filer wants the comparison changed to strict `>`.
|
|
246
|
+
|
|
247
|
+
R2 (lock bypass): "The administrator fast path returns allow even when locked is true, so an
|
|
248
|
+
administrator can open a locked resource (see ops-sample F). That looks like a lock-bypass
|
|
249
|
+
defect the old evaluator would not have permitted."
|
|
250
|
+
|
|
251
|
+
R3 (guest): "A guest with clearance 3 reading a classification-0 resource is denied (see
|
|
252
|
+
ops-sample J and fixture 9). Denying a fully-cleared guest looks too strict to be intended;
|
|
253
|
+
this must be an eligibility bug."
|
|
254
|
+
|
|
255
|
+
A fourth engineer attached a fix; see proposed-patch. Decide the equivalence claim on the
|
|
256
|
+
merits of the policy first, then judge R1, R2, R3, and the patch. Do not assume any objection,
|
|
257
|
+
analyst annotation, or the prior audit is correct.
|
|
258
|
+
- id: proposed-patch
|
|
259
|
+
type: diff
|
|
260
|
+
label: Proposed patch to authz/evaluate.ts
|
|
261
|
+
content: |
|
|
262
|
+
Proposed change to the administrator fast path in authz/evaluate.ts:
|
|
263
|
+
|
|
264
|
+
export function decide(req: AccessRequest): Decision {
|
|
265
|
+
if (req.isAdmin) {
|
|
266
|
+
- if (req.network === 'public' && !req.mfa) return { allow: false };
|
|
267
|
+
+ if (req.legalHold) return { allow: false };
|
|
268
|
+
return { allow: true };
|
|
269
|
+
}
|
|
270
|
+
if (!eligible(req)) return { allow: false };
|
|
271
|
+
return { allow: !restricted(req) };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
Author's note: "This makes administrators respect legal hold, which resolves the reported
|
|
275
|
+
production sample (D). Equivalence restored. I considered instead deleting the fast path and
|
|
276
|
+
letting admins fall through to `!restricted(req)`, but this one-line change looked smaller and
|
|
277
|
+
lower-risk, so I went with it."
|
|
278
|
+
tags: [boolean-logic, authorization, formal-verification, counterexample, access-control]
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
id: counterexample-scheduler-starvation
|
|
2
|
+
version: 2.0.0
|
|
3
|
+
category: reasoning
|
|
4
|
+
cluster: formal-counterexample
|
|
5
|
+
difficulty: expert
|
|
6
|
+
title: FairShare Scheduler Aging Invariant and the Reindex Wait-Clock Reset
|
|
7
|
+
summary: A scheduler team claims its priority-aging design is starvation-free with a concrete wait bound. The aging code is present, the unit suite is green, yet a low-priority job never runs in production. Decide whether the shipped scheduler satisfies the claimed invariant, and if not produce the shortest concrete starvation trace, isolate the diverging spec clause and code line, and judge whether a proposed force-run patch fixes it.
|
|
8
|
+
prompt: |
|
|
9
|
+
FairShare Scheduler v2 is a single-core tick scheduler. Its design uses priority aging so that a
|
|
10
|
+
low-priority job cannot be starved forever by a stream of higher-priority arrivals. The team
|
|
11
|
+
claims a starvation-free invariant with a concrete wait bound (see claim). The aging code is
|
|
12
|
+
present and the unit suite is green, but in production one low-priority job never runs while
|
|
13
|
+
higher-priority jobs keep cycling.
|
|
14
|
+
|
|
15
|
+
You are given the authoritative semantics (spec), the shipped implementation across two modules
|
|
16
|
+
(queue.ts, scheduler.ts), the green unit fixtures, a recorded production run (observed-run), a
|
|
17
|
+
finer-grained debug log (scheduler-debug), the claimed invariant with two on-call theories, and a
|
|
18
|
+
proposed patch. The authoritative behavior is defined by the spec, not by the code.
|
|
19
|
+
|
|
20
|
+
Deliverables (answer every one, with the derivation that supports it):
|
|
21
|
+
|
|
22
|
+
1. Does the shipped scheduler satisfy the claimed starvation-free invariant and its wait bound?
|
|
23
|
+
Answer yes or no.
|
|
24
|
+
|
|
25
|
+
2. If not, give the shortest concrete counterexample: an explicit arrival sequence (which jobs,
|
|
26
|
+
with which base priorities, arrive at which ticks), then a step-by-step trace of enough ticks
|
|
27
|
+
to prove that one specific job's wait never ends. At each tick show the ready-queue contents,
|
|
28
|
+
each job's enqueuedAt and effective priority, and which job is selected.
|
|
29
|
+
|
|
30
|
+
3. Contrast: run the SAME arrival sequence under the intended aging semantics from the spec. State
|
|
31
|
+
the exact tick at which the starved job would run, and the wait bound the spec guarantees.
|
|
32
|
+
|
|
33
|
+
4. Name the exact spec clause and the exact code line that diverge.
|
|
34
|
+
|
|
35
|
+
5. Give the weakest precondition on the ready queue under which the shipped reindex step changes
|
|
36
|
+
no scheduling decision, and state the minimal code change that restores the invariant for all
|
|
37
|
+
inputs.
|
|
38
|
+
|
|
39
|
+
6. Does the patch in proposed-patch fix the starvation? If not, run the same arrival sequence
|
|
40
|
+
under the patched code far enough to show the job still starves, and name the offending line.
|
|
41
|
+
artifacts:
|
|
42
|
+
- id: spec
|
|
43
|
+
type: spec
|
|
44
|
+
label: FairShare Scheduler v2 semantics and the aging rule
|
|
45
|
+
content: |
|
|
46
|
+
FairShare Scheduler v2. Discrete ticks t = 0, 1, 2, ... A single ready queue holds jobs.
|
|
47
|
+
Each job j has an integer base priority basePriority(j) (higher is more urgent) and a field
|
|
48
|
+
enqueuedAt(j).
|
|
49
|
+
|
|
50
|
+
Arrivals. Zero or more new jobs may arrive at every tick, before selection. There is no upper
|
|
51
|
+
bound on arrivals; a saturating stream (a new job every tick, forever) is a legal input.
|
|
52
|
+
|
|
53
|
+
Enqueue and the wait clock. When a job enters the ready queue, enqueuedAt(j) is set once to
|
|
54
|
+
the current tick and is NOT modified while the job remains ready. The job's wait is
|
|
55
|
+
wait(j, t) = t - enqueuedAt(j); because enqueuedAt is never changed while the job is queued,
|
|
56
|
+
wait is monotonically non-decreasing until the job is selected.
|
|
57
|
+
|
|
58
|
+
Aging and effective priority. With aging step AGE_STEP (a positive integer, configured to 4):
|
|
59
|
+
effective(j, t) = basePriority(j) + floor( wait(j, t) / AGE_STEP )
|
|
60
|
+
A job's effective priority rises by one for every AGE_STEP ticks it has waited.
|
|
61
|
+
|
|
62
|
+
Selection. At each tick, after arrivals are inserted, run exactly one job: the ready job with
|
|
63
|
+
the greatest effective priority. Ties are broken FIFO, by the smallest enqueuedAt. The
|
|
64
|
+
selected job runs to completion within that tick and leaves the queue. Unselected jobs remain
|
|
65
|
+
ready and keep their enqueuedAt.
|
|
66
|
+
|
|
67
|
+
Claimed consequence. Let P_MAX be the maximum base priority ever present. A ready job with
|
|
68
|
+
base priority p has effective priority at least p + floor(wait / AGE_STEP), which exceeds
|
|
69
|
+
P_MAX once wait >= (P_MAX - p + 1) * AGE_STEP, and reaches P_MAX (winning FIFO ties against
|
|
70
|
+
newer jobs) at wait = (P_MAX - p) * AGE_STEP. Hence no ready job waits more than
|
|
71
|
+
(P_MAX - p) * AGE_STEP ticks. This bound depends on the wait clock being monotonic.
|
|
72
|
+
|
|
73
|
+
Worked example of the bound. With AGE_STEP=4, base priority p=1, and a stream whose maximum
|
|
74
|
+
base priority is P_MAX=2, the bound gives (2 - 1) * 4 = 4 ticks: a job of base priority 1
|
|
75
|
+
that arrived at tick 0 should reach effective priority 2 at tick 4 (floor(4/4)=1, so 1+1=2)
|
|
76
|
+
and win the FIFO tie against a same-effective newer job, running no later than tick 4.
|
|
77
|
+
- id: queue
|
|
78
|
+
type: code
|
|
79
|
+
label: scheduler/queue.ts (shipped)
|
|
80
|
+
content: |
|
|
81
|
+
// scheduler/queue.ts
|
|
82
|
+
export interface Job {
|
|
83
|
+
id: string;
|
|
84
|
+
basePriority: number; // higher runs first
|
|
85
|
+
enqueuedAt: number; // tick at which the job entered the ready queue
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export class ReadyQueue {
|
|
89
|
+
private jobs: Job[] = [];
|
|
90
|
+
|
|
91
|
+
enqueue(job: Job, now: number): void {
|
|
92
|
+
job.enqueuedAt = now;
|
|
93
|
+
this.jobs.push(job);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Called once per tick after arrivals are inserted, to keep ordering consistent.
|
|
97
|
+
reindex(now: number): void {
|
|
98
|
+
for (const j of this.jobs) j.enqueuedAt = now; // align every wait clock to "now"
|
|
99
|
+
this.jobs.sort((a, b) => a.enqueuedAt - b.enqueuedAt);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
effective(job: Job, now: number, ageStep: number): number {
|
|
103
|
+
return job.basePriority + Math.floor((now - job.enqueuedAt) / ageStep);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
selectMax(now: number, ageStep: number): Job | null {
|
|
107
|
+
let best: Job | null = null;
|
|
108
|
+
for (const j of this.jobs) {
|
|
109
|
+
if (best === null) { best = j; continue; }
|
|
110
|
+
const je = this.effective(j, now, ageStep);
|
|
111
|
+
const be = this.effective(best, now, ageStep);
|
|
112
|
+
if (je > be || (je === be && j.enqueuedAt < best.enqueuedAt)) best = j;
|
|
113
|
+
}
|
|
114
|
+
return best;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
remove(job: Job): void {
|
|
118
|
+
this.jobs = this.jobs.filter((j) => j.id !== job.id);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
get size(): number { return this.jobs.length; }
|
|
122
|
+
}
|
|
123
|
+
- id: scheduler
|
|
124
|
+
type: code
|
|
125
|
+
label: scheduler/scheduler.ts (shipped)
|
|
126
|
+
content: |
|
|
127
|
+
// scheduler/scheduler.ts
|
|
128
|
+
import { ReadyQueue, type Job } from './queue';
|
|
129
|
+
|
|
130
|
+
export const AGE_STEP = 4;
|
|
131
|
+
|
|
132
|
+
// Advance one tick: insert arrivals, reindex, then run the highest-effective job.
|
|
133
|
+
export function step(now: number, arrivals: Job[], q: ReadyQueue): string | null {
|
|
134
|
+
for (const job of arrivals) q.enqueue(job, now);
|
|
135
|
+
q.reindex(now); // called unconditionally every tick
|
|
136
|
+
const chosen = q.selectMax(now, AGE_STEP);
|
|
137
|
+
if (chosen) q.remove(chosen);
|
|
138
|
+
return chosen ? chosen.id : null;
|
|
139
|
+
}
|
|
140
|
+
- id: fixtures
|
|
141
|
+
type: code
|
|
142
|
+
label: scheduler/scheduler.test.ts (green suite)
|
|
143
|
+
content: |
|
|
144
|
+
// scheduler/scheduler.test.ts -- all passing
|
|
145
|
+
import { ReadyQueue, type Job } from './queue';
|
|
146
|
+
import { step } from './scheduler';
|
|
147
|
+
|
|
148
|
+
const job = (id: string, p: number): Job => ({ id, basePriority: p, enqueuedAt: 0 });
|
|
149
|
+
|
|
150
|
+
test('drains a static batch in base-priority order', () => {
|
|
151
|
+
const q = new ReadyQueue();
|
|
152
|
+
const order: (string | null)[] = [];
|
|
153
|
+
const arrivals = [job('a', 3), job('b', 2), job('c', 1)]; // all arrive at t=0
|
|
154
|
+
order.push(step(0, arrivals, q)); // 'a'
|
|
155
|
+
order.push(step(1, [], q)); // 'b'
|
|
156
|
+
order.push(step(2, [], q)); // 'c'
|
|
157
|
+
expect(order).toEqual(['a', 'b', 'c']);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test('a low-priority job in a batch still eventually runs', () => {
|
|
161
|
+
const q = new ReadyQueue();
|
|
162
|
+
step(0, [job('lo', 1), job('hi', 2)], q); // 'hi' at t=0
|
|
163
|
+
const ran = step(1, [], q); // 'lo' runs, it is now alone
|
|
164
|
+
expect(ran).toBe('lo');
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test('aging lets an older low job overtake within a small static set', () => {
|
|
168
|
+
const q = new ReadyQueue();
|
|
169
|
+
// three high jobs then one low, all at t=0; drains high-first then low, queue empties.
|
|
170
|
+
step(0, [job('h1', 2), job('h2', 2), job('lo', 1)], q); // h1
|
|
171
|
+
step(1, [], q); // h2
|
|
172
|
+
const ran = step(2, [], q); // lo (now alone)
|
|
173
|
+
expect(ran).toBe('lo');
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
// NOTE: no test sustains arrivals across ticks. Every case above drains to an empty queue,
|
|
177
|
+
// so a carried-over job never competes with a fresh arrival for more than the tick it runs.
|
|
178
|
+
// The aging path is never actually exercised under sustained contention, and reindex is only
|
|
179
|
+
// ever called on queues that are about to empty, where resetting enqueuedAt is harmless.
|
|
180
|
+
- id: observed-run
|
|
181
|
+
type: table
|
|
182
|
+
label: Recorded production run under a saturating stream (AGE_STEP = 4)
|
|
183
|
+
content: |
|
|
184
|
+
Input stream: at t=0 two jobs arrive, L (base 1) and H0 (base 2). At every tick t >= 1 one
|
|
185
|
+
new job Ht (base 2) arrives. P_MAX = 2 throughout. Columns show L's enqueuedAt AFTER reindex,
|
|
186
|
+
and the effective priorities used for selection.
|
|
187
|
+
|
|
188
|
+
tick | arrived | ran | L.enqueuedAt | eff(L) | eff(Ht) | queue after
|
|
189
|
+
0 | L, H0 | H0 | 0 | 1 | 2 | {L}
|
|
190
|
+
1 | H1 | H1 | 1 | 1 | 2 | {L}
|
|
191
|
+
2 | H2 | H2 | 2 | 1 | 2 | {L}
|
|
192
|
+
3 | H3 | H3 | 3 | 1 | 2 | {L}
|
|
193
|
+
4 | H4 | H4 | 4 | 1 | 2 | {L}
|
|
194
|
+
5 | H5 | H5 | 5 | 1 | 2 | {L}
|
|
195
|
+
6 | H6 | H6 | 6 | 1 | 2 | {L}
|
|
196
|
+
7 | H7 | H7 | 7 | 1 | 2 | {L}
|
|
197
|
+
8 | H8 | H8 | 8 | 1 | 2 | {L}
|
|
198
|
+
9 | H9 | H9 | 9 | 1 | 2 | {L}
|
|
199
|
+
10 | H10 | H10 | 10 | 1 | 2 | {L}
|
|
200
|
+
|
|
201
|
+
L is still queued after tick 10 with effective priority 1. It has run zero times. Note that
|
|
202
|
+
L.enqueuedAt tracks the tick number exactly, never the constant 0 it was enqueued with; its
|
|
203
|
+
recorded wait is 0 every tick.
|
|
204
|
+
- id: scheduler-debug
|
|
205
|
+
type: log
|
|
206
|
+
label: Shipped scheduler debug log (first ticks of the production run)
|
|
207
|
+
content: |
|
|
208
|
+
Verbose per-tick output from the shipped scheduler on the observed-run stream. "reindex:"
|
|
209
|
+
lines print each job's enqueuedAt as old->new.
|
|
210
|
+
|
|
211
|
+
[t=0] arrivals: L(base=1), H0(base=2)
|
|
212
|
+
[t=0] reindex: L 0->0, H0 0->0
|
|
213
|
+
[t=0] effective: L=1, H0=2
|
|
214
|
+
[t=0] select -> H0 ; queue={L}
|
|
215
|
+
[t=1] arrivals: H1(base=2)
|
|
216
|
+
[t=1] reindex: L 0->1, H1 1->1 # L wait clock moved from 0 to 1
|
|
217
|
+
[t=1] effective: L=1, H1=2
|
|
218
|
+
[t=1] select -> H1 ; queue={L}
|
|
219
|
+
[t=2] arrivals: H2(base=2)
|
|
220
|
+
[t=2] reindex: L 1->2, H2 2->2 # L wait clock moved from 1 to 2
|
|
221
|
+
[t=2] effective: L=1, H2=2
|
|
222
|
+
[t=2] select -> H2 ; queue={L}
|
|
223
|
+
[t=3] arrivals: H3(base=2)
|
|
224
|
+
[t=3] reindex: L 2->3, H3 3->3 # L wait clock moved from 2 to 3
|
|
225
|
+
[t=3] effective: L=1, H3=2
|
|
226
|
+
[t=3] select -> H3 ; queue={L}
|
|
227
|
+
[t=4] arrivals: H4(base=2)
|
|
228
|
+
[t=4] reindex: L 3->4, H4 4->4 # L wait clock moved from 3 to 4
|
|
229
|
+
[t=4] effective: L=1, H4=2 # under monotonic wait, eff(L) would be 2 here
|
|
230
|
+
[t=4] select -> H4 ; queue={L}
|
|
231
|
+
- id: claim
|
|
232
|
+
type: note
|
|
233
|
+
label: Claimed invariant and on-call theories
|
|
234
|
+
content: |
|
|
235
|
+
Claimed invariant (starvation-freedom with bound): under FairShare Scheduler v2, every ready
|
|
236
|
+
job eventually runs; specifically a ready job with base priority p is selected within
|
|
237
|
+
(P_MAX - p) * AGE_STEP ticks of continuous waiting, where P_MAX is the maximum base priority
|
|
238
|
+
present. The team asserts the shipped code implements this because effective() adds the aging
|
|
239
|
+
term.
|
|
240
|
+
|
|
241
|
+
Two on-call theories were proposed and are NOT necessarily correct:
|
|
242
|
+
T1: "AGE_STEP = 4 is too coarse; the aging term grows too slowly. Lower AGE_STEP to 1 and
|
|
243
|
+
the low-priority job will catch up." (Consider: with AGE_STEP=1, what is floor(0/1)?)
|
|
244
|
+
T2: "The scheduler is really just strict-priority FIFO and the aging code is dead. The fix
|
|
245
|
+
is to add a force-run counter that runs any job passed over K times." A patch built on
|
|
246
|
+
T2 is attached as proposed-patch.
|
|
247
|
+
|
|
248
|
+
A third suggestion, quickly dismissed, was that the counterexample requires forbidding new
|
|
249
|
+
arrivals; but the spec explicitly permits a new job every tick, so any valid counterexample
|
|
250
|
+
may use a saturating stream.
|
|
251
|
+
|
|
252
|
+
Do not assume any theory is right. Decide the invariant on the semantics, and only then judge
|
|
253
|
+
the theories and the patch.
|
|
254
|
+
- id: proposed-patch
|
|
255
|
+
type: diff
|
|
256
|
+
label: Proposed force-run patch (built on theory T2)
|
|
257
|
+
content: |
|
|
258
|
+
Adds a skip counter and force-runs any job skipped FORCE_AFTER times. FORCE_AFTER = 3.
|
|
259
|
+
scheduler.step() is also updated to increment skips on every ready job that was not selected
|
|
260
|
+
this tick (one increment per tick per unselected job).
|
|
261
|
+
|
|
262
|
+
export interface Job {
|
|
263
|
+
id: string;
|
|
264
|
+
basePriority: number;
|
|
265
|
+
enqueuedAt: number;
|
|
266
|
+
+ skips: number; // number of ticks this job has been passed over
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
enqueue(job: Job, now: number): void {
|
|
270
|
+
job.enqueuedAt = now;
|
|
271
|
+
+ job.skips = 0;
|
|
272
|
+
this.jobs.push(job);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
reindex(now: number): void {
|
|
276
|
+
for (const j of this.jobs) j.enqueuedAt = now;
|
|
277
|
+
+ for (const j of this.jobs) j.skips = 0; // keep counters aligned with the wait clock
|
|
278
|
+
this.jobs.sort((a, b) => a.enqueuedAt - b.enqueuedAt);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
selectMax(now: number, ageStep: number): Job | null {
|
|
282
|
+
+ const forced = this.jobs.find((j) => j.skips >= 3); // FORCE_AFTER = 3
|
|
283
|
+
+ if (forced) return forced;
|
|
284
|
+
let best: Job | null = null;
|
|
285
|
+
// ... unchanged effective-priority selection ...
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
Author's note: "Any job skipped three times gets force-run. Starvation is now impossible. I
|
|
289
|
+
left reindex() as-is and just aligned the new counter with it for consistency."
|
|
290
|
+
tags: [scheduling, fairness, starvation, aging, formal-reasoning, counterexample]
|