@riddledc/riddle-proof 0.8.16 → 0.8.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -208,6 +208,27 @@ viewport and returns an `environment_blocked` result without starting partial
|
|
|
208
208
|
jobs when the account balance cannot cover the intended sweep. Use
|
|
209
209
|
`--balance-preflight=false` to bypass this check.
|
|
210
210
|
|
|
211
|
+
## Regression Packs
|
|
212
|
+
|
|
213
|
+
Reusable regression packs live in `examples/regression-packs`. They are
|
|
214
|
+
host-readable manifests for recurring trust-boundary checks that should produce
|
|
215
|
+
the same terminal outcomes across generic runners and wrappers.
|
|
216
|
+
|
|
217
|
+
The first pack is
|
|
218
|
+
`examples/regression-packs/oc-flow-regression.json`. It names the live OpenClaw
|
|
219
|
+
regression cases for route-changing interactions, query/hash preservation,
|
|
220
|
+
negative dropped query/hash evidence, no-diff production audits, selector and
|
|
221
|
+
thrown-error blockers, and stale checkpoint responses after terminal status.
|
|
222
|
+
The same pack also points at the local generic core suite:
|
|
223
|
+
|
|
224
|
+
```sh
|
|
225
|
+
python3 packages/riddle-proof/runtime/tests/trust_boundary_regression.py
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Before counting live wrapper runs, use the pack's runtime gate: verify
|
|
229
|
+
`riddle_proof_status` reports the loaded `@riddledc/openclaw-riddle-proof` and
|
|
230
|
+
`@riddledc/riddle-proof` versions. Disk package versions alone are not enough.
|
|
231
|
+
|
|
211
232
|
Use `--viewport-name <name>` to run only one named viewport from a
|
|
212
233
|
multi-viewport profile while preserving viewport-scoped setup actions and
|
|
213
234
|
checks:
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "riddle-proof.regression-pack.v1",
|
|
3
|
+
"pack_id": "riddle-proof-oc-flow-2026-06",
|
|
4
|
+
"public_name": "Riddle Proof OC Flow Regression Pack",
|
|
5
|
+
"description": "Reusable regression pack for the trust boundary between browser evidence, verifier judgment, retry policy, wrapper checkpoint handling, and terminal status.",
|
|
6
|
+
"minimum_versions": {
|
|
7
|
+
"@riddledc/openclaw-riddle-proof": "0.4.146",
|
|
8
|
+
"@riddledc/riddle-proof": "0.8.18"
|
|
9
|
+
},
|
|
10
|
+
"runtime_gate": {
|
|
11
|
+
"tool": "riddle_proof_status",
|
|
12
|
+
"require_loaded_metadata": true,
|
|
13
|
+
"fresh_runtime_rule": "Count only runs whose riddle_proof_status package_metadata reports the minimum_versions or newer. If disk package versions differ from loaded metadata, restart the gateway and discard stale runs."
|
|
14
|
+
},
|
|
15
|
+
"forbidden_terminal_markers": [
|
|
16
|
+
"codex_invalid_json",
|
|
17
|
+
"codex_no_final_response",
|
|
18
|
+
"codex_timeout",
|
|
19
|
+
"max_iterations_reached",
|
|
20
|
+
"stage_iteration_limit_reached",
|
|
21
|
+
"verify_capture_retry",
|
|
22
|
+
"checkpoint_response_without_packet"
|
|
23
|
+
],
|
|
24
|
+
"local_core_suite": {
|
|
25
|
+
"command": "python3 packages/riddle-proof/runtime/tests/trust_boundary_regression.py",
|
|
26
|
+
"required_cases": [
|
|
27
|
+
"route-change-forward-pass",
|
|
28
|
+
"route-change-retry-state-drift-ignored",
|
|
29
|
+
"route-change-proof-plan-placeholder-ignored",
|
|
30
|
+
"route-change-reverse-pass",
|
|
31
|
+
"route-change-reverse-nested-terminal-url-pass",
|
|
32
|
+
"query-hash-trailing-slash-pass",
|
|
33
|
+
"query-hash-dropped-structured-negative-blocker",
|
|
34
|
+
"same-page-hash-pass",
|
|
35
|
+
"missing-selector-timeout-specific-blocker",
|
|
36
|
+
"thrown-error-preserves-structured-evidence",
|
|
37
|
+
"interaction-thrown-error-specific-blocker",
|
|
38
|
+
"proof-evidence-absent-specific-blocker",
|
|
39
|
+
"no-diff-prod-audit-default-capture-pass"
|
|
40
|
+
]
|
|
41
|
+
},
|
|
42
|
+
"openclaw_live_suite": {
|
|
43
|
+
"target": {
|
|
44
|
+
"repo": "davisdiehl/riddle-site",
|
|
45
|
+
"prod_url": "https://riddledc.com/",
|
|
46
|
+
"ship_mode": "none",
|
|
47
|
+
"implementation_mode": "none",
|
|
48
|
+
"require_diff": false,
|
|
49
|
+
"allow_code_changes": false
|
|
50
|
+
},
|
|
51
|
+
"result_log_fields": [
|
|
52
|
+
"run_id",
|
|
53
|
+
"loaded_metadata",
|
|
54
|
+
"state_path",
|
|
55
|
+
"terminal_status",
|
|
56
|
+
"last_checkpoint",
|
|
57
|
+
"route_expectation_source",
|
|
58
|
+
"proof_evidence_present",
|
|
59
|
+
"proof_json_urls",
|
|
60
|
+
"screenshots",
|
|
61
|
+
"forbidden_terminal_markers_seen"
|
|
62
|
+
],
|
|
63
|
+
"cases": [
|
|
64
|
+
{
|
|
65
|
+
"id": "home-to-proof-route-change-pass",
|
|
66
|
+
"tool": "riddle_proof_change",
|
|
67
|
+
"intent": "Start on the production home page, click the visible Proof navigation link, and prove the terminal route is /proof/.",
|
|
68
|
+
"params": {
|
|
69
|
+
"verification_mode": "interaction",
|
|
70
|
+
"server_path": "/",
|
|
71
|
+
"wait_for_selector": "main#main-content",
|
|
72
|
+
"capture_script_contract": [
|
|
73
|
+
"goto https://riddledc.com/",
|
|
74
|
+
"wait for main#main-content",
|
|
75
|
+
"click a visible Proof link whose href includes /proof",
|
|
76
|
+
"wait for terminal URL https://riddledc.com/proof/",
|
|
77
|
+
"return riddle-proof.interaction.v1 evidence with start, action, terminal, assertions, and routeExpectationSource=capture_script.expectedUrl",
|
|
78
|
+
"save screenshot after-proof"
|
|
79
|
+
]
|
|
80
|
+
},
|
|
81
|
+
"expect": {
|
|
82
|
+
"terminal_status": "ready_to_ship",
|
|
83
|
+
"terminal_path": "/proof/",
|
|
84
|
+
"proof_evidence_present": true,
|
|
85
|
+
"route_expectation_source": "capture_script.expectedUrl",
|
|
86
|
+
"forbidden_terminal_markers": []
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "proof-to-home-route-change-pass",
|
|
91
|
+
"tool": "riddle_proof_change",
|
|
92
|
+
"intent": "Start on /proof/, click the visible Riddle/Home root navigation link, and prove the terminal route is /.",
|
|
93
|
+
"params": {
|
|
94
|
+
"verification_mode": "interaction",
|
|
95
|
+
"prod_url": "https://riddledc.com/proof/",
|
|
96
|
+
"server_path": "/proof/",
|
|
97
|
+
"wait_for_selector": "main#main-content",
|
|
98
|
+
"capture_script_contract": [
|
|
99
|
+
"goto https://riddledc.com/proof/",
|
|
100
|
+
"wait for main#main-content",
|
|
101
|
+
"click the visible Riddle/Home nav link whose href is / or resolves to the site root",
|
|
102
|
+
"wait for terminal URL https://riddledc.com/",
|
|
103
|
+
"return riddle-proof.interaction.v1 evidence with nested terminal.expectedUrl and terminal.observedUrl",
|
|
104
|
+
"save screenshot after-home"
|
|
105
|
+
]
|
|
106
|
+
},
|
|
107
|
+
"expect": {
|
|
108
|
+
"terminal_status": "ready_to_ship",
|
|
109
|
+
"terminal_path": "/",
|
|
110
|
+
"proof_evidence_present": true,
|
|
111
|
+
"route_expectation_source": "capture_script.expectedUrl",
|
|
112
|
+
"forbidden_terminal_markers": []
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"id": "pricing-query-hash-positive-pass",
|
|
117
|
+
"tool": "riddle_proof_change",
|
|
118
|
+
"intent": "Click into Pricing with a terminal query/hash expectation and prove slash, query, and hash are preserved.",
|
|
119
|
+
"params": {
|
|
120
|
+
"verification_mode": "interaction",
|
|
121
|
+
"server_path": "/",
|
|
122
|
+
"wait_for_selector": "main#main-content",
|
|
123
|
+
"expected_terminal_url": "https://riddledc.com/pricing/?rp_probe=1#pricing-probe",
|
|
124
|
+
"capture_script_contract": [
|
|
125
|
+
"start from https://riddledc.com/",
|
|
126
|
+
"navigate or click to the pricing route with ?rp_probe=1#pricing-probe",
|
|
127
|
+
"wait for https://riddledc.com/pricing/?rp_probe=1#pricing-probe",
|
|
128
|
+
"return structured evidence proving pathname /pricing/, search ?rp_probe=1, and hash #pricing-probe",
|
|
129
|
+
"save screenshot after-proof"
|
|
130
|
+
]
|
|
131
|
+
},
|
|
132
|
+
"expect": {
|
|
133
|
+
"terminal_status": "ready_to_ship",
|
|
134
|
+
"terminal_url": "https://riddledc.com/pricing/?rp_probe=1#pricing-probe",
|
|
135
|
+
"proof_evidence_present": true,
|
|
136
|
+
"query_preserved": true,
|
|
137
|
+
"hash_preserved": true,
|
|
138
|
+
"forbidden_terminal_markers": []
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
"id": "pricing-query-hash-dropped-blocker",
|
|
143
|
+
"tool": "riddle_proof_change",
|
|
144
|
+
"intent": "Force a negative control where expected query/hash are dropped at terminal; this must block specifically as invalid browser evidence.",
|
|
145
|
+
"params": {
|
|
146
|
+
"verification_mode": "interaction",
|
|
147
|
+
"server_path": "/",
|
|
148
|
+
"wait_for_selector": "main#main-content",
|
|
149
|
+
"expected_terminal_url": "https://riddledc.com/pricing/?rp_probe=1#pricing-probe",
|
|
150
|
+
"forced_observed_terminal_url": "https://riddledc.com/pricing/",
|
|
151
|
+
"capture_script_contract": [
|
|
152
|
+
"record expected terminal URL with ?rp_probe=1#pricing-probe",
|
|
153
|
+
"intentionally observe or force terminal evidence for https://riddledc.com/pricing/",
|
|
154
|
+
"return riddle-proof.interaction.v1 evidence with expected and observed terminal URLs",
|
|
155
|
+
"save screenshot terminal-pricing-negative-control"
|
|
156
|
+
]
|
|
157
|
+
},
|
|
158
|
+
"expect": {
|
|
159
|
+
"terminal_status": "blocked",
|
|
160
|
+
"last_checkpoint": "verify_capture_blocked",
|
|
161
|
+
"blocker_code": "verify_capture_blocked",
|
|
162
|
+
"capture_decision": "failed_interaction_capture",
|
|
163
|
+
"observed_terminal_url": "https://riddledc.com/pricing/",
|
|
164
|
+
"forbidden_terminal_markers": [
|
|
165
|
+
"ready_to_ship",
|
|
166
|
+
"codex_invalid_json",
|
|
167
|
+
"codex_timeout",
|
|
168
|
+
"max_iterations_reached",
|
|
169
|
+
"checkpoint_response_without_packet"
|
|
170
|
+
]
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
"id": "no-diff-prod-audit-pass",
|
|
175
|
+
"tool": "riddle_proof_change",
|
|
176
|
+
"intent": "Audit the live production home page without implementation or PR handoff, and prove the current target evidence directly.",
|
|
177
|
+
"params": {
|
|
178
|
+
"verification_mode": "interaction",
|
|
179
|
+
"implementation_mode": "none",
|
|
180
|
+
"require_diff": false,
|
|
181
|
+
"allow_code_changes": false,
|
|
182
|
+
"ship_mode": "none",
|
|
183
|
+
"server_path": "/",
|
|
184
|
+
"wait_for_selector": "main#main-content",
|
|
185
|
+
"capture_script_contract": [
|
|
186
|
+
"use the current production target only",
|
|
187
|
+
"do not request implementation or git diff",
|
|
188
|
+
"prove main#main-content and expected home-page content are visible",
|
|
189
|
+
"return structured evidence and save screenshot after-proof"
|
|
190
|
+
]
|
|
191
|
+
},
|
|
192
|
+
"expect": {
|
|
193
|
+
"terminal_status": "ready_to_ship",
|
|
194
|
+
"ship_handoff": "none",
|
|
195
|
+
"implementation_attempted": false,
|
|
196
|
+
"proof_evidence_present": true,
|
|
197
|
+
"forbidden_terminal_markers": []
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
"id": "missing-selector-timeout-blocker",
|
|
202
|
+
"tool": "riddle_proof_change",
|
|
203
|
+
"intent": "Use a missing selector in capture and require a specific Playwright selector timeout in terminal evidence.",
|
|
204
|
+
"params": {
|
|
205
|
+
"verification_mode": "interaction",
|
|
206
|
+
"server_path": "/",
|
|
207
|
+
"wait_for_selector": "main#main-content",
|
|
208
|
+
"capture_script_contract": [
|
|
209
|
+
"start from https://riddledc.com/",
|
|
210
|
+
"try to click or scroll a selector that does not exist",
|
|
211
|
+
"use a short selector timeout",
|
|
212
|
+
"surface the exact Playwright timeout message in structured failure evidence"
|
|
213
|
+
]
|
|
214
|
+
},
|
|
215
|
+
"expect": {
|
|
216
|
+
"terminal_status": "blocked",
|
|
217
|
+
"failure_kind": "specific_blocker",
|
|
218
|
+
"message_contains": "Timeout",
|
|
219
|
+
"forbidden_terminal_markers": [
|
|
220
|
+
"codex_invalid_json",
|
|
221
|
+
"codex_timeout",
|
|
222
|
+
"max_iterations_reached"
|
|
223
|
+
]
|
|
224
|
+
}
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"id": "thrown-error-specific-blocker",
|
|
228
|
+
"tool": "riddle_proof_change",
|
|
229
|
+
"intent": "Throw an intentional marker from capture and require the exact marker to survive in structured failure evidence.",
|
|
230
|
+
"params": {
|
|
231
|
+
"verification_mode": "interaction",
|
|
232
|
+
"server_path": "/",
|
|
233
|
+
"wait_for_selector": "main#main-content",
|
|
234
|
+
"capture_script_contract": [
|
|
235
|
+
"throw intentional-riddle-proof-regression-thrown-error from the capture script",
|
|
236
|
+
"preserve the exact marker in terminal evidence",
|
|
237
|
+
"do not convert the failure to generic Codex JSON/lifecycle failure"
|
|
238
|
+
]
|
|
239
|
+
},
|
|
240
|
+
"expect": {
|
|
241
|
+
"terminal_status": "blocked",
|
|
242
|
+
"failure_kind": "specific_blocker",
|
|
243
|
+
"message_contains": "intentional-riddle-proof-regression-thrown-error",
|
|
244
|
+
"forbidden_terminal_markers": [
|
|
245
|
+
"codex_invalid_json",
|
|
246
|
+
"codex_timeout",
|
|
247
|
+
"max_iterations_reached"
|
|
248
|
+
]
|
|
249
|
+
}
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
"id": "late-stale-checkpoint-ignored",
|
|
253
|
+
"tool": "riddle_proof_change + riddle_proof_review",
|
|
254
|
+
"intent": "Complete a terminal ready_to_ship proof, then submit a stale manual author checkpoint response after terminal status and preserve ready_to_ship.",
|
|
255
|
+
"steps": [
|
|
256
|
+
{
|
|
257
|
+
"tool": "riddle_proof_change",
|
|
258
|
+
"params": {
|
|
259
|
+
"verification_mode": "interaction",
|
|
260
|
+
"server_path": "/",
|
|
261
|
+
"wait_for_selector": "main#main-content",
|
|
262
|
+
"checkpoint_mode": "terminal_only",
|
|
263
|
+
"report_mode": "terminal_only",
|
|
264
|
+
"capture_script_contract": [
|
|
265
|
+
"complete the Home -> Proof route proof",
|
|
266
|
+
"terminalize as ready_to_ship before any manual checkpoint injection",
|
|
267
|
+
"record state_path and run_id from riddle_proof_status"
|
|
268
|
+
]
|
|
269
|
+
},
|
|
270
|
+
"expect": {
|
|
271
|
+
"terminal_status": "ready_to_ship",
|
|
272
|
+
"proof_evidence_present": true
|
|
273
|
+
}
|
|
274
|
+
},
|
|
275
|
+
{
|
|
276
|
+
"tool": "riddle_proof_review",
|
|
277
|
+
"after": "terminal ready_to_ship",
|
|
278
|
+
"params_template": {
|
|
279
|
+
"state_path": "${state_path}",
|
|
280
|
+
"decision": "continue_checkpoint",
|
|
281
|
+
"summary": "Submit stale checkpoint response after terminal ready_to_ship.",
|
|
282
|
+
"checkpoint_response": {
|
|
283
|
+
"version": "riddle-proof.checkpoint_response.v1",
|
|
284
|
+
"run_id": "${run_id}",
|
|
285
|
+
"checkpoint": "author_supervisor_judgment",
|
|
286
|
+
"decision": "author_packet",
|
|
287
|
+
"summary": "Late stale author packet after terminal ready_to_ship.",
|
|
288
|
+
"payload": {
|
|
289
|
+
"proof_plan": "STALE PACKET: do not resume. This packet is intentionally late and should not alter a terminal run.",
|
|
290
|
+
"capture_script": "return { passed: true, staleManualCheckpointProbe: true };"
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
},
|
|
294
|
+
"expect": {
|
|
295
|
+
"terminal_status": "ready_to_ship",
|
|
296
|
+
"ignored_checkpoint_response": true,
|
|
297
|
+
"background_resume_started": false,
|
|
298
|
+
"forbidden_terminal_markers": [
|
|
299
|
+
"checkpoint_response_without_packet",
|
|
300
|
+
"max_iterations_reached",
|
|
301
|
+
"codex_invalid_json"
|
|
302
|
+
]
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
],
|
|
306
|
+
"expect": {
|
|
307
|
+
"terminal_status": "ready_to_ship",
|
|
308
|
+
"ignored_checkpoint_response": true,
|
|
309
|
+
"background_resume_started": false,
|
|
310
|
+
"forbidden_terminal_markers": [
|
|
311
|
+
"checkpoint_response_without_packet",
|
|
312
|
+
"max_iterations_reached",
|
|
313
|
+
"codex_invalid_json"
|
|
314
|
+
]
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
]
|
|
318
|
+
}
|
|
319
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@riddledc/riddle-proof",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.18",
|
|
4
4
|
"description": "Reusable Riddle Proof contracts and helpers for evidence-backed agent changes.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "RiddleDC",
|
|
@@ -227,6 +227,6 @@
|
|
|
227
227
|
"build": "tsup src/index.ts src/types.ts src/result.ts src/state.ts src/checkpoint.ts src/run-card.ts src/runner.ts src/engine-harness.ts src/codex-exec-agent.ts src/local-agent.ts src/cli.ts src/cli/index.ts src/diagnostics.ts src/proof-session.ts src/playability.ts src/basic-gameplay.ts src/profile.ts src/profile/index.ts src/openclaw.ts src/proof-run-core.ts src/proof-run-engine.ts src/riddle-client.ts src/runtime/riddle-client.ts src/spec/index.ts src/spec/types.ts src/spec/result.ts src/spec/state.ts src/spec/checkpoint.ts src/spec/run-card.ts src/runtime/index.ts src/app-contract/index.ts src/advanced/index.ts src/advanced/runner.ts src/advanced/engine-harness.ts src/advanced/proof-run-core.ts src/advanced/proof-run-engine.ts src/adapters/openclaw.ts src/adapters/local-agent.ts src/adapters/codex-exec-agent.ts src/adapters/codex.ts --format cjs,esm --dts --out-dir dist --clean",
|
|
228
228
|
"clean": "rm -rf dist",
|
|
229
229
|
"lint": "echo 'lint: (not configured)'",
|
|
230
|
-
"test": "npm run build && node test.js && node proof-run.test.js && node trust-boundary.test.js && python3 runtime/tests/trust_boundary_regression.py"
|
|
230
|
+
"test": "npm run build && node test.js && node proof-run.test.js && node trust-boundary.test.js && node regression-packs.test.js && python3 runtime/tests/trust_boundary_regression.py"
|
|
231
231
|
}
|
|
232
232
|
}
|
package/runtime/lib/verify.py
CHANGED
|
@@ -2319,11 +2319,22 @@ def text_path_candidate(value):
|
|
|
2319
2319
|
return path_candidate(raw)
|
|
2320
2320
|
|
|
2321
2321
|
|
|
2322
|
+
PROSE_ROUTE_PLACEHOLDER_PATHS = {
|
|
2323
|
+
'/href',
|
|
2324
|
+
'/path',
|
|
2325
|
+
'/pathname',
|
|
2326
|
+
'/route',
|
|
2327
|
+
'/url',
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2330
|
+
|
|
2322
2331
|
def text_route_candidate(value):
|
|
2323
2332
|
candidate = text_path_candidate(value)
|
|
2324
2333
|
if not candidate:
|
|
2325
2334
|
return ''
|
|
2326
2335
|
parsed = urlparse(candidate)
|
|
2336
|
+
if (parsed.path or '').lower() in PROSE_ROUTE_PLACEHOLDER_PATHS and not parsed.query and not parsed.fragment:
|
|
2337
|
+
return ''
|
|
2327
2338
|
first_segment = next((part for part in (parsed.path or '').split('/') if part), '')
|
|
2328
2339
|
if first_segment and first_segment[:1].isupper():
|
|
2329
2340
|
return ''
|
|
@@ -2350,11 +2361,11 @@ def terminal_path_from_text(value):
|
|
|
2350
2361
|
def interaction_assertions_pass(value):
|
|
2351
2362
|
for record in proof_evidence_records(value):
|
|
2352
2363
|
if any(record.get(key) is False for key in (
|
|
2353
|
-
'passed', 'ok', 'proofReady', 'proof_ready', 'routeMatches', 'route_matches',
|
|
2364
|
+
'passed', 'pass', 'ok', 'success', 'proofReady', 'proof_ready', 'routeMatches', 'route_matches',
|
|
2354
2365
|
)):
|
|
2355
2366
|
return False
|
|
2356
2367
|
if any(record.get(key) is True for key in (
|
|
2357
|
-
'passed', 'ok', 'proofReady', 'proof_ready', 'interactionPassed', 'interaction_passed',
|
|
2368
|
+
'passed', 'pass', 'ok', 'success', 'proofReady', 'proof_ready', 'interactionPassed', 'interaction_passed',
|
|
2358
2369
|
)):
|
|
2359
2370
|
return True
|
|
2360
2371
|
for key in ('assertions', 'checks', 'predicates', 'expectations'):
|
|
@@ -2369,7 +2380,7 @@ def interaction_assertions_pass(value):
|
|
|
2369
2380
|
if isinstance(item, bool):
|
|
2370
2381
|
bools.append(item)
|
|
2371
2382
|
elif isinstance(item, dict):
|
|
2372
|
-
for flag_key in ('passed', 'ok', 'valid'):
|
|
2383
|
+
for flag_key in ('passed', 'pass', 'ok', 'valid', 'success'):
|
|
2373
2384
|
if isinstance(item.get(flag_key), bool):
|
|
2374
2385
|
bools.append(item.get(flag_key))
|
|
2375
2386
|
break
|
|
@@ -2381,6 +2392,7 @@ def interaction_assertions_pass(value):
|
|
|
2381
2392
|
INTERACTION_ASSERTION_CONTAINER_KEYS = ('assertions', 'checks', 'predicates', 'expectations')
|
|
2382
2393
|
INTERACTION_FAILURE_FLAG_KEYS = (
|
|
2383
2394
|
'passed',
|
|
2395
|
+
'pass',
|
|
2384
2396
|
'ok',
|
|
2385
2397
|
'valid',
|
|
2386
2398
|
'success',
|
|
@@ -444,6 +444,62 @@ class FakeRiddle:
|
|
|
444
444
|
'totalPixels': 972000,
|
|
445
445
|
},
|
|
446
446
|
}
|
|
447
|
+
if 'clickedProofNavigationOcLiveShape' in script:
|
|
448
|
+
page_state = {
|
|
449
|
+
'bodyTextLength': 4113,
|
|
450
|
+
'visibleTextSample': 'RIDDLE PROOF Turn a URL into evidence an agent can cite.',
|
|
451
|
+
'interactiveElements': 6,
|
|
452
|
+
'visibleInteractiveElements': 6,
|
|
453
|
+
'pathname': '/proof/',
|
|
454
|
+
'href': 'https://riddledc.com/proof/',
|
|
455
|
+
'title': 'Riddle Proof',
|
|
456
|
+
'buttons': ['Proof'],
|
|
457
|
+
'headings': ['Riddle Proof'],
|
|
458
|
+
'links': [],
|
|
459
|
+
'canvasCount': 0,
|
|
460
|
+
'largeVisibleElements': [{'tag': 'h1', 'text': 'Riddle Proof'}],
|
|
461
|
+
}
|
|
462
|
+
proof_evidence = {
|
|
463
|
+
'version': 'riddle-proof.interaction.v1',
|
|
464
|
+
'expectedUrl': 'https://riddledc.com/proof/',
|
|
465
|
+
'routeExpectationSource': 'capture_script.expectedUrl',
|
|
466
|
+
'startRoute': {'url': 'https://riddledc.com/', 'pathname': '/'},
|
|
467
|
+
'terminalRoute': {'url': 'https://riddledc.com/proof/', 'pathname': '/proof/'},
|
|
468
|
+
'action': {'type': 'click', 'target': 'visible Proof navigation link', 'clicked': True},
|
|
469
|
+
'startState': {'url': 'https://riddledc.com/', 'pathname': '/'},
|
|
470
|
+
'terminalState': {'url': 'https://riddledc.com/proof/', 'pathname': '/proof/'},
|
|
471
|
+
'assertions': [
|
|
472
|
+
{
|
|
473
|
+
'name': 'route expectation source is capture_script.expectedUrl',
|
|
474
|
+
'pass': True,
|
|
475
|
+
'routeExpectationSource': 'capture_script.expectedUrl',
|
|
476
|
+
'expectedUrl': 'https://riddledc.com/proof/',
|
|
477
|
+
},
|
|
478
|
+
{'name': 'start URL is production home', 'pass': True, 'startPathname': '/'},
|
|
479
|
+
{
|
|
480
|
+
'name': 'terminal URL matched expected proof route',
|
|
481
|
+
'pass': True,
|
|
482
|
+
'terminalUrl': 'https://riddledc.com/proof/',
|
|
483
|
+
},
|
|
484
|
+
{'name': 'Proof page content visible', 'pass': True},
|
|
485
|
+
],
|
|
486
|
+
'success': True,
|
|
487
|
+
}
|
|
488
|
+
return {
|
|
489
|
+
'ok': True,
|
|
490
|
+
'screenshots': [{'url': 'https://cdn.example.com/proof-after.png'}],
|
|
491
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/proof-after.png'}],
|
|
492
|
+
'result': {'pageState': page_state, 'proofEvidence': proof_evidence},
|
|
493
|
+
'console': [
|
|
494
|
+
'RIDDLE_PROOF_STATE:' + json.dumps(page_state),
|
|
495
|
+
'RIDDLE_PROOF_EVIDENCE:' + json.dumps(proof_evidence),
|
|
496
|
+
],
|
|
497
|
+
'visual_diff': {
|
|
498
|
+
'diffPercentage': 1.2,
|
|
499
|
+
'differentPixels': 12000,
|
|
500
|
+
'totalPixels': 972000,
|
|
501
|
+
},
|
|
502
|
+
}
|
|
447
503
|
if 'clickedProofNavigation' in script:
|
|
448
504
|
page_state = {
|
|
449
505
|
'bodyTextLength': 180,
|
|
@@ -2787,6 +2843,51 @@ def run_verify_interaction_proof_evidence_overrides_stale_expected_path():
|
|
|
2787
2843
|
shutil.rmtree(tempdir, ignore_errors=True)
|
|
2788
2844
|
|
|
2789
2845
|
|
|
2846
|
+
def run_verify_interaction_proof_plan_placeholder_uses_live_evidence():
|
|
2847
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-interaction-placeholder-route-'))
|
|
2848
|
+
state_path = tempdir / 'state.json'
|
|
2849
|
+
try:
|
|
2850
|
+
state = base_state(tempdir, reference='before')
|
|
2851
|
+
state.update({
|
|
2852
|
+
'recon_status': 'ready_for_proof_plan',
|
|
2853
|
+
'author_status': 'ready',
|
|
2854
|
+
'proof_plan_status': 'ready',
|
|
2855
|
+
'implementation_status': 'changes_detected',
|
|
2856
|
+
'verification_mode': 'interaction',
|
|
2857
|
+
'server_path': '/',
|
|
2858
|
+
'before_cdn': 'https://cdn.example.com/before-home.png',
|
|
2859
|
+
'proof_plan': 'Start at /, click Proof, and verify the terminal pathname /pathname.',
|
|
2860
|
+
'capture_script': "clickedProofNavigationOcLiveShape(); await saveScreenshot('after-proof');",
|
|
2861
|
+
'recon_results': {
|
|
2862
|
+
'baselines': {'before': {'path': '/', 'url': 'https://cdn.example.com/before-home.png'}},
|
|
2863
|
+
},
|
|
2864
|
+
})
|
|
2865
|
+
write_state(state_path, state)
|
|
2866
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
2867
|
+
|
|
2868
|
+
fake = FakeRiddle()
|
|
2869
|
+
load_util_with_fake(fake)
|
|
2870
|
+
load_module('verify_interaction_placeholder_route_uses_evidence', VERIFY_PATH)
|
|
2871
|
+
after_verify = json.loads(state_path.read_text())
|
|
2872
|
+
|
|
2873
|
+
assert after_verify['verify_status'] == 'evidence_captured'
|
|
2874
|
+
assert after_verify['route_expectation']['source'] == 'proof_evidence_contract'
|
|
2875
|
+
assert after_verify['route_expectation']['expected_path'] == '/proof'
|
|
2876
|
+
route = after_verify['proof_assessment_request']['semantic_context']['route']
|
|
2877
|
+
assert route['expected_after_path'] == '/proof'
|
|
2878
|
+
assert route['after_observed_path'] == '/proof'
|
|
2879
|
+
assert after_verify.get('expected_terminal_path') == '/proof'
|
|
2880
|
+
assert '/pathname' not in json.dumps(after_verify['route_expectation'])
|
|
2881
|
+
assert 'wrong route' not in after_verify['verify_results']['after']['observation']['reason']
|
|
2882
|
+
return {
|
|
2883
|
+
'ok': True,
|
|
2884
|
+
'expected_path': after_verify['route_expectation']['expected_path'],
|
|
2885
|
+
'source': after_verify['route_expectation']['source'],
|
|
2886
|
+
}
|
|
2887
|
+
finally:
|
|
2888
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
2889
|
+
|
|
2890
|
+
|
|
2790
2891
|
def run_verify_interaction_reverse_terminal_route_from_proof_evidence():
|
|
2791
2892
|
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-interaction-reverse-'))
|
|
2792
2893
|
state_path = tempdir / 'state.json'
|
|
@@ -3665,6 +3766,7 @@ if __name__ == '__main__':
|
|
|
3665
3766
|
'remote_audit_verify_uses_default_capture_script': run_remote_audit_verify_uses_default_capture_script(),
|
|
3666
3767
|
'verify_interaction_terminal_route_from_proof_evidence': run_verify_interaction_terminal_route_from_proof_evidence(),
|
|
3667
3768
|
'verify_interaction_proof_evidence_overrides_stale_expected_path': run_verify_interaction_proof_evidence_overrides_stale_expected_path(),
|
|
3769
|
+
'verify_interaction_proof_plan_placeholder_uses_live_evidence': run_verify_interaction_proof_plan_placeholder_uses_live_evidence(),
|
|
3668
3770
|
'verify_interaction_reverse_terminal_route_from_proof_evidence': run_verify_interaction_reverse_terminal_route_from_proof_evidence(),
|
|
3669
3771
|
'verify_interaction_reverse_terminal_expected_url_from_nested_terminal_evidence': run_verify_interaction_reverse_terminal_expected_url_from_nested_terminal_evidence(),
|
|
3670
3772
|
'verify_interaction_prose_route_noise_uses_proof_evidence': run_verify_interaction_prose_route_noise_uses_proof_evidence(),
|
|
@@ -31,6 +31,12 @@ CASES = [
|
|
|
31
31
|
'function': 'run_verify_interaction_proof_evidence_overrides_stale_expected_path',
|
|
32
32
|
'expected_terminal': 'pass',
|
|
33
33
|
},
|
|
34
|
+
{
|
|
35
|
+
'name': 'route-change-proof-plan-placeholder-ignored',
|
|
36
|
+
'covers': ['route-changing interactions', 'proof-evidence-present'],
|
|
37
|
+
'function': 'run_verify_interaction_proof_plan_placeholder_uses_live_evidence',
|
|
38
|
+
'expected_terminal': 'pass',
|
|
39
|
+
},
|
|
34
40
|
{
|
|
35
41
|
'name': 'route-change-reverse-pass',
|
|
36
42
|
'covers': ['route-changing interactions'],
|