@riddledc/riddle-proof 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +6 -6
  2. package/dist/chunk-3MHFLQKG.js +853 -0
  3. package/dist/{chunk-LVP22WE4.js → chunk-5GZZZ6JA.js} +5 -1
  4. package/dist/engine-harness.cjs +2505 -22
  5. package/dist/engine-harness.js +1 -1
  6. package/dist/index.cjs +2512 -29
  7. package/dist/index.js +2 -2
  8. package/dist/openclaw.cjs +1 -1
  9. package/dist/openclaw.js +1 -1
  10. package/dist/proof-run-core.cjs +909 -0
  11. package/dist/proof-run-core.d.cts +280 -0
  12. package/dist/proof-run-core.d.ts +280 -0
  13. package/dist/proof-run-core.js +48 -0
  14. package/dist/proof-run-engine.cjs +2499 -0
  15. package/dist/proof-run-engine.d.cts +677 -0
  16. package/dist/proof-run-engine.d.ts +677 -0
  17. package/dist/proof-run-engine.js +1649 -0
  18. package/lib/workspace-core.mjs +391 -0
  19. package/package.json +15 -3
  20. package/runtime/lib/author.py +343 -0
  21. package/runtime/lib/implement.py +63 -0
  22. package/runtime/lib/preflight.py +246 -0
  23. package/runtime/lib/recon.py +1048 -0
  24. package/runtime/lib/riddle_core_call.mjs +151 -0
  25. package/runtime/lib/setup.py +387 -0
  26. package/runtime/lib/ship.py +834 -0
  27. package/runtime/lib/util.py +673 -0
  28. package/runtime/lib/verify.py +1223 -0
  29. package/runtime/pipelines/riddle-proof-author.lobster +28 -0
  30. package/runtime/pipelines/riddle-proof-implement.lobster +26 -0
  31. package/runtime/pipelines/riddle-proof-recon.lobster +79 -0
  32. package/runtime/pipelines/riddle-proof-setup.lobster +141 -0
  33. package/runtime/pipelines/riddle-proof-ship.lobster +36 -0
  34. package/runtime/pipelines/riddle-proof-verify.lobster +74 -0
  35. package/runtime/tests/recon_verify_smoke.py +1198 -0
@@ -0,0 +1,1198 @@
1
+ import importlib.util
2
+ import json
3
+ import os
4
+ import shutil
5
+ import subprocess as sp
6
+ import sys
7
+ import tempfile
8
+ from contextlib import contextmanager
9
+ from pathlib import Path
10
+
11
+ ROOT = Path(__file__).resolve().parents[1]
12
+ LIB = ROOT / 'lib'
13
+ UTIL_PATH = LIB / 'util.py'
14
+ RECON_PATH = LIB / 'recon.py'
15
+ VERIFY_PATH = LIB / 'verify.py'
16
+ AUTHOR_PATH = LIB / 'author.py'
17
+ SHIP_PATH = LIB / 'ship.py'
18
+
19
+ BUILD_SCRIPT = "python3 -c \"from pathlib import Path; Path('build').mkdir(exist_ok=True); Path('build/index.html').write_text('<html>ok</html>')\""
20
+
21
+
22
+ def state_console(payload):
23
+ payload = dict(payload)
24
+ payload.setdefault('consoleSummary', {'error_count': 0})
25
+ payload.setdefault('failedChecks', [])
26
+ return {
27
+ 'entries': {
28
+ 'log': [{'message': 'RIDDLE_PROOF_STATE:' + json.dumps(payload)}],
29
+ 'warn': [],
30
+ 'error': [],
31
+ 'info': [],
32
+ },
33
+ }
34
+
35
+
36
+ def load_module(name: str, path: Path):
37
+ sys.modules.pop(name, None)
38
+ spec = importlib.util.spec_from_file_location(name, path)
39
+ module = importlib.util.module_from_spec(spec)
40
+ sys.modules[name] = module
41
+ assert spec.loader is not None
42
+ spec.loader.exec_module(module)
43
+ return module
44
+
45
+
46
+ class FakeRiddle:
47
+ def __init__(self):
48
+ self.calls = []
49
+
50
+ def invoke(self, tool, args, timeout=180):
51
+ self.calls.append({'tool': tool, 'args': args, 'timeout': timeout})
52
+ if tool == 'riddle_preview_delete':
53
+ return {'ok': True}
54
+ raise AssertionError(f'unexpected invoke tool: {tool}')
55
+
56
+ def invoke_retry(self, tool, args, retries=3, timeout=180):
57
+ self.calls.append({'tool': tool, 'args': args, 'timeout': timeout, 'retries': retries})
58
+ if tool == 'riddle_preview':
59
+ label = args.get('label', 'preview')
60
+ return {
61
+ 'ok': True,
62
+ 'id': f'pv-{label}',
63
+ 'preview_url': f'https://preview.example.com/{label}/',
64
+ }
65
+ if tool == 'riddle_script':
66
+ script = args.get('script', '')
67
+ if 'preview.example.com' in script and '/wrong' in script:
68
+ return {
69
+ 'ok': True,
70
+ 'screenshots': [{'url': 'https://cdn.example.com/wrong.png'}],
71
+ 'outputs': [{'name': 'wrong.png', 'url': 'https://cdn.example.com/wrong.png'}],
72
+ 'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":5,"interactiveElements":0,"pathname":"/wrong","title":"Wrong"}'],
73
+ }
74
+ if 'audioNoProof' in script:
75
+ page_state = {
76
+ 'bodyTextLength': 96,
77
+ 'visibleTextSample': 'Neon step sequencer audio workbench',
78
+ 'interactiveElements': 0,
79
+ 'visibleInteractiveElements': 0,
80
+ 'pathname': '/s/pv-after/sequencer',
81
+ 'title': 'Sequencer',
82
+ 'buttons': [],
83
+ 'headings': ['Sequencer'],
84
+ 'links': [],
85
+ 'canvasCount': 1,
86
+ 'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
87
+ }
88
+ return {
89
+ 'ok': True,
90
+ 'outputs': [{'name': 'metrics.json', 'url': 'https://cdn.example.com/metrics.json'}],
91
+ 'result': {'pageState': page_state, 'summary': {'captured': True}},
92
+ 'console': ['RIDDLE_PROOF_STATE:' + json.dumps(page_state)],
93
+ }
94
+ if 'audioFailedProof' in script:
95
+ page_state = {
96
+ 'bodyTextLength': 96,
97
+ 'visibleTextSample': 'Neon step sequencer audio workbench',
98
+ 'interactiveElements': 0,
99
+ 'visibleInteractiveElements': 0,
100
+ 'pathname': '/s/pv-after/sequencer',
101
+ 'title': 'Sequencer',
102
+ 'buttons': [],
103
+ 'headings': ['Sequencer'],
104
+ 'links': [],
105
+ 'canvasCount': 1,
106
+ 'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
107
+ }
108
+ proof_evidence = {
109
+ 'proof_evidence_present': False,
110
+ 'evidence_summary': 'Structured audio/source proof did not satisfy all required Monkberry release-tail checks.',
111
+ 'checks': {
112
+ 'route_ok': True,
113
+ 'ui_context_ok': True,
114
+ 'source_audio_ok': False,
115
+ 'monkberry_scope_ok': False,
116
+ },
117
+ 'import_error': 'Failed to fetch dynamically imported module: /src/Games/songs/index.js',
118
+ }
119
+ return {
120
+ 'ok': True,
121
+ 'outputs': [{'name': 'proof.json', 'url': 'https://cdn.example.com/proof.json'}],
122
+ 'result': {'pageState': page_state},
123
+ 'console': [
124
+ 'RIDDLE_PROOF_STATE:' + json.dumps(page_state),
125
+ 'RIDDLE_PROOF_EVIDENCE ' + json.dumps(proof_evidence),
126
+ ],
127
+ }
128
+ if 'window.__riddleProofEvidence' in script or 'globalThis.__riddleProofEvidence' in script:
129
+ page_state = {
130
+ 'bodyTextLength': 36,
131
+ 'visibleTextSample': 'Neon step sequencer audio workbench',
132
+ 'interactiveElements': 0,
133
+ 'visibleInteractiveElements': 0,
134
+ 'pathname': '/s/pv-after/sequencer',
135
+ 'title': 'Sequencer',
136
+ 'buttons': [],
137
+ 'headings': ['Sequencer'],
138
+ 'links': [],
139
+ 'canvasCount': 1,
140
+ 'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
141
+ }
142
+ proof_evidence = {
143
+ 'modality': 'audio',
144
+ 'attack_ms_before': 42,
145
+ 'attack_ms_after': 12,
146
+ 'transient_energy_delta_db': 4.8,
147
+ 'passed': True,
148
+ }
149
+ return {
150
+ 'ok': True,
151
+ 'outputs': [{'name': 'metrics.json', 'url': 'https://cdn.example.com/metrics.json'}],
152
+ 'result': {'proofEvidence': proof_evidence},
153
+ 'console': [
154
+ 'RIDDLE_PROOF_STATE:' + json.dumps(page_state),
155
+ 'RIDDLE_PROOF_EVIDENCE:' + json.dumps(proof_evidence),
156
+ ],
157
+ }
158
+ if 'after-proof' in script:
159
+ return {
160
+ 'ok': True,
161
+ 'screenshots': [{'url': 'https://cdn.example.com/after.png'}],
162
+ 'outputs': [{'name': 'after.png', 'url': 'https://cdn.example.com/after.png'}],
163
+ 'console': state_console({
164
+ 'bodyTextLength': 180,
165
+ 'visibleTextSample': 'Pricing CTA Buy Now',
166
+ 'interactiveElements': 4,
167
+ 'visibleInteractiveElements': 4,
168
+ 'pathname': '/s/pv-after/pricing',
169
+ 'title': 'After',
170
+ 'buttons': ['Buy Now'],
171
+ 'headings': ['Pricing'],
172
+ 'links': [],
173
+ 'canvasCount': 0,
174
+ 'largeVisibleElements': [{'tag': 'button', 'text': 'Buy Now'}],
175
+ }),
176
+ }
177
+ if 'prod.example.com/pricing' in script:
178
+ return {
179
+ 'ok': True,
180
+ 'screenshots': [{'url': 'https://cdn.example.com/prod.png'}],
181
+ 'outputs': [{'name': 'prod.png', 'url': 'https://cdn.example.com/prod.png'}],
182
+ 'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":180,"interactiveElements":4,"pathname":"/pricing","title":"Prod"}'],
183
+ }
184
+ if 'preview.example.com' in script and '/pricing' in script:
185
+ return {
186
+ 'ok': True,
187
+ 'screenshots': [{'url': 'https://cdn.example.com/before.png'}],
188
+ 'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/before.png'}],
189
+ 'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":180,"interactiveElements":4,"pathname":"/pricing","title":"Before"}'],
190
+ }
191
+ if 'preview.example.com' in script and '/games/tic-tac-toe' in script:
192
+ return {
193
+ 'ok': True,
194
+ 'screenshots': [{'url': 'https://cdn.example.com/tictactoe-before.png'}],
195
+ 'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/tictactoe-before.png'}],
196
+ 'console': state_console({
197
+ 'bodyTextLength': 220,
198
+ 'visibleTextSample': 'LilArcade Tic Tac Toe Player X Reset Game',
199
+ 'interactiveElements': 5,
200
+ 'visibleInteractiveElements': 5,
201
+ 'pathname': '/s/pv-before/games/tic-tac-toe',
202
+ 'title': 'TicTacToe',
203
+ 'buttons': ['Reset Game'],
204
+ 'headings': ['Tic Tac Toe'],
205
+ 'links': [],
206
+ 'canvasCount': 0,
207
+ 'largeVisibleElements': [{'tag': 'button', 'text': 'Reset Game'}],
208
+ }),
209
+ }
210
+ raise AssertionError(f'unexpected riddle_script payload: {script}')
211
+ raise AssertionError(f'unexpected invoke_retry tool: {tool}')
212
+
213
+
214
+ def make_project(root: Path, route_snippet: str):
215
+ root.mkdir(parents=True, exist_ok=True)
216
+ (root / 'src').mkdir(exist_ok=True)
217
+ (root / 'src' / 'routes.tsx').write_text(route_snippet)
218
+ (root / 'package.json').write_text(json.dumps({
219
+ 'name': root.name,
220
+ 'scripts': {'build': BUILD_SCRIPT},
221
+ 'dependencies': {'react': '18.0.0', 'react-router-dom': '6.0.0'},
222
+ }, indent=2))
223
+
224
+
225
+ def write_state(path: Path, payload: dict):
226
+ path.write_text(json.dumps(payload, indent=2))
227
+
228
+
229
+ def run_capture_artifact_enrichment():
230
+ util = load_module('util_artifact_enrichment', UTIL_PATH)
231
+ fixtures = {
232
+ 'https://cdn.example.com/console.json': {
233
+ 'summary': {'total_entries': 1, 'error_count': 1},
234
+ 'entries': {
235
+ 'log': [],
236
+ 'warn': [],
237
+ 'error': [{'message': 'page.waitForSelector: Timeout 30000ms exceeded'}],
238
+ 'info': [],
239
+ },
240
+ },
241
+ 'https://cdn.example.com/proof.json': {
242
+ 'script_error': 'page.waitForSelector: Timeout 30000ms exceeded',
243
+ 'metrics': {'wall_ms': 30000},
244
+ },
245
+ }
246
+
247
+ def fake_fetch_json_artifact(url, max_bytes=0):
248
+ return fixtures[url], ''
249
+
250
+ util.fetch_json_artifact = fake_fetch_json_artifact
251
+ payload = {
252
+ 'ok': True,
253
+ 'outputs': [
254
+ {'name': 'screenshot_1.png', 'url': 'https://cdn.example.com/screenshot_1.png'},
255
+ {'name': 'console.json', 'url': 'https://cdn.example.com/console.json'},
256
+ {'name': 'proof.json', 'url': 'https://cdn.example.com/proof.json'},
257
+ ],
258
+ 'screenshots': [{'name': 'screenshot_1.png', 'url': 'https://cdn.example.com/screenshot_1.png'}],
259
+ }
260
+
261
+ enriched = util.enrich_capture_payload(payload)
262
+ summary = util.summarize_capture_artifacts(payload)
263
+ assert enriched['console']['summary']['error_count'] == 1
264
+ assert enriched['_proof_json']['script_error'].startswith('page.waitForSelector')
265
+ assert summary['artifact_json'] == ['console.json', 'proof.json']
266
+ assert summary['proof_script_error'] is True
267
+ assert summary['console_summary']['error_count'] == 1
268
+
269
+ return {
270
+ 'ok': True,
271
+ 'artifact_json': summary['artifact_json'],
272
+ 'proof_script_error': summary['proof_script_error'],
273
+ }
274
+
275
+
276
+ def run_capture_diagnostics_redact_sensitive_values():
277
+ util = load_module('util_capture_diagnostics', UTIL_PATH)
278
+ state = {}
279
+ args = {
280
+ 'script': 'await page.goto("https://example.com");',
281
+ 'localStorage': {'accessToken': 'secret-token'},
282
+ 'cookies': [{'name': 'session', 'value': 'secret-cookie'}],
283
+ 'headers': {'Authorization': 'Bearer secret-token'},
284
+ 'nested': {'api_key': 'secret-key', 'safe': 'ok'},
285
+ }
286
+ payload = {
287
+ 'ok': False,
288
+ 'error': 'page crashed',
289
+ 'outputs': [{'name': 'console.json', 'url': 'https://cdn.example.com/console.json'}],
290
+ 'artifacts': [{
291
+ 'name': 'metering.json',
292
+ 'kind': 'json',
293
+ 'role': 'diagnostic',
294
+ 'path': '/tmp/riddle-proof/metering.json',
295
+ 'metadata': {'samples': 64},
296
+ }],
297
+ }
298
+ diagnostic = util.append_capture_diagnostic(state, 'after', 'riddle_server_preview', args, payload)
299
+
300
+ assert diagnostic['version'] == 'riddle-proof.capture-diagnostic.v1'
301
+ assert diagnostic['tool'] == 'riddle_server_preview'
302
+ assert diagnostic['args']['script'].startswith('await page.goto')
303
+ assert diagnostic['args']['localStorage'] == '[redacted]'
304
+ assert diagnostic['args']['cookies'] == '[redacted]'
305
+ assert diagnostic['args']['headers'] == '[redacted]'
306
+ assert diagnostic['args']['nested']['api_key'] == '[redacted]'
307
+ assert diagnostic['args']['nested']['safe'] == 'ok'
308
+ assert state['capture_diagnostics'][-1]['artifact_summary']['outputs'][0]['name'] == 'console.json'
309
+ assert state['capture_diagnostics'][-1]['artifact_summary']['artifacts'][0]['metadata_keys'] == ['samples']
310
+ return {'ok': True, 'diagnostics': len(state['capture_diagnostics'])}
311
+
312
+
313
+ def run_apply_auth_context_passes_supported_auth_payloads():
314
+ util = load_module('util_apply_auth_context', UTIL_PATH)
315
+ state = {
316
+ 'auth_localStorage': {'token': 'local'},
317
+ 'auth_cookies': [{'name': 'session', 'value': 'cookie'}],
318
+ 'auth_headers': {'Authorization': 'Bearer header'},
319
+ }
320
+ args = {'script': 'await page.goto("https://example.com");'}
321
+ util.apply_auth_context(state, args)
322
+ assert util.has_auth_context(state) is True
323
+ assert args['localStorage'] == state['auth_localStorage']
324
+ assert args['cookies'] == state['auth_cookies']
325
+ assert args['headers'] == state['auth_headers']
326
+ assert util.has_auth_context({}) is False
327
+ assert util.has_auth_context({'use_auth': 'true'}) is True
328
+ return {'ok': True, 'arg_keys': sorted(args.keys())}
329
+
330
+
331
+ def run_verify_quality_ignores_proof_telemetry_console_text():
332
+ sys.modules.pop('util', None)
333
+ source = VERIFY_PATH.read_text()
334
+ helpers_source = source.split('\ns = load_state()', 1)[0]
335
+ namespace = {'__file__': str(VERIFY_PATH)}
336
+ exec(compile(helpers_source, str(VERIFY_PATH), 'exec'), namespace)
337
+
338
+ telemetry_payload = {
339
+ 'bodyTextLength': 180,
340
+ 'visibleTextSample': 'LilArcade Circle Maze Coin Clicker',
341
+ 'interactiveElements': 4,
342
+ 'visibleInteractiveElements': 4,
343
+ 'pathname': '/',
344
+ 'title': 'LilArcade',
345
+ 'headings': ['LilArcade'],
346
+ 'links': [{'text': 'Circle Maze', 'href': '/games/circle-maze'}],
347
+ 'canvasCount': 0,
348
+ 'largeVisibleElements': [{'tag': 'h1', 'text': 'LilArcade'}],
349
+ 'console_summary': {'error_count': 0},
350
+ 'failed_checks': [],
351
+ }
352
+ quality = namespace['evaluate_capture_quality']({
353
+ 'ok': True,
354
+ 'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
355
+ 'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
356
+ 'console': [
357
+ 'RIDDLE_PROOF_STATE:' + json.dumps(telemetry_payload),
358
+ 'RIDDLE_PROOF_EVIDENCE:' + json.dumps({'passed': True, 'error_count': 0}),
359
+ ],
360
+ }, '/', 'visual')
361
+ assert quality['valid'] is True
362
+ assert quality['details']['has_errors'] is False
363
+ assert quality['details']['capture_error_messages'] == []
364
+
365
+ runtime_error_quality = namespace['evaluate_capture_quality']({
366
+ 'ok': True,
367
+ 'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
368
+ 'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
369
+ 'console': [
370
+ 'RIDDLE_PROOF_STATE:' + json.dumps(telemetry_payload),
371
+ 'Uncaught TypeError: boom',
372
+ ],
373
+ }, '/', 'visual')
374
+ assert runtime_error_quality['valid'] is False
375
+ assert runtime_error_quality['details']['has_errors'] is True
376
+ assert 'console/runtime errors' in runtime_error_quality['reason']
377
+
378
+ strong_delta = namespace['extract_visual_delta']({
379
+ 'ok': True,
380
+ 'result': {
381
+ 'proofEvidence': {
382
+ 'change_pct': '2.31',
383
+ 'changed_pixels': 22395,
384
+ 'width': 1080,
385
+ 'height': 900,
386
+ },
387
+ },
388
+ })
389
+ assert strong_delta['status'] == 'measured'
390
+ assert strong_delta['passed'] is True
391
+ assert strong_delta['change_percent'] == 2.31
392
+ assert strong_delta['changed_pixels'] == 22395
393
+
394
+ weak_delta = namespace['extract_visual_delta']({
395
+ 'ok': True,
396
+ 'result': {
397
+ 'proofEvidence': {
398
+ 'change_pct': '0.06',
399
+ 'changed_pixels': 616,
400
+ 'width': 1080,
401
+ 'height': 900,
402
+ },
403
+ },
404
+ })
405
+ assert weak_delta['status'] == 'measured'
406
+ assert weak_delta['passed'] is False
407
+ assert 'below the legibility threshold' in weak_delta['reason']
408
+
409
+ unmeasured_delta = namespace['extract_visual_delta']({
410
+ 'ok': True,
411
+ 'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
412
+ })
413
+ assert unmeasured_delta['status'] == 'unmeasured'
414
+ assert unmeasured_delta['passed'] is None
415
+
416
+ return {'ok': True, 'telemetry_valid': quality['valid'], 'weak_delta_passed': weak_delta['passed']}
417
+
418
+
419
+ def load_util_with_fake(fake: FakeRiddle):
420
+ util = load_module('util', UTIL_PATH)
421
+ util.invoke = fake.invoke
422
+ util.invoke_retry = fake.invoke_retry
423
+ return util
424
+
425
+
426
+ @contextmanager
427
+ def temporary_env(**updates):
428
+ sentinel = object()
429
+ previous = {}
430
+ for key, value in updates.items():
431
+ previous[key] = os.environ.get(key, sentinel)
432
+ if value is None:
433
+ os.environ.pop(key, None)
434
+ else:
435
+ os.environ[key] = value
436
+ try:
437
+ yield
438
+ finally:
439
+ for key, value in previous.items():
440
+ if value is sentinel:
441
+ os.environ.pop(key, None)
442
+ else:
443
+ os.environ[key] = value
444
+
445
+
446
+ def base_state(tempdir: Path, *, reference='before', prod_url=''):
447
+ before_dir = tempdir / 'before'
448
+ after_dir = tempdir / 'after'
449
+ make_project(before_dir, "export const routes = [{ path: '/pricing', element: <Pricing /> }];\n")
450
+ make_project(after_dir, "export const routes = [{ path: '/pricing', element: <Pricing /> }];\n")
451
+ return {
452
+ 'workspace_ready': True,
453
+ 'repo_dir': str(tempdir),
454
+ 'before_worktree': str(before_dir),
455
+ 'after_worktree': str(after_dir),
456
+ 'mode': 'static',
457
+ 'reference': reference,
458
+ 'requested_reference': reference,
459
+ 'prod_url': prod_url,
460
+ 'change_request': 'Fix the pricing CTA layout',
461
+ 'success_criteria': 'Pricing CTA is visible and aligned on the pricing route.',
462
+ 'verification_mode': 'visual',
463
+ 'build_command': BUILD_SCRIPT,
464
+ 'build_output': 'build',
465
+ 'capture_script': '',
466
+ 'proof_plan_status': 'pending_recon',
467
+ 'author_status': 'pending_recon',
468
+ 'implementation_status': 'pending_recon',
469
+ 'wait_for_selector': '',
470
+ 'server_path': '/pricing',
471
+ 'allow_static_preview_fallback': True,
472
+ 'auth_localStorage': {},
473
+ 'before_cdn': '',
474
+ 'after_cdn': '',
475
+ 'prod_cdn': '',
476
+ 'proof_plan': '',
477
+ 'proof_plan_request': {},
478
+ 'author_request': {},
479
+ 'recon_results': {},
480
+ 'recon_decision_request': {},
481
+ 'verify_results': {},
482
+ 'proof_assessment': {},
483
+ 'proof_assessment_request': {},
484
+ }
485
+
486
+
487
+ def run_recon_then_author_request():
488
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-supervisor-request-'))
489
+ state_path = tempdir / 'state.json'
490
+ try:
491
+ state = base_state(tempdir, reference='both', prod_url='https://prod.example.com/pricing')
492
+ write_state(state_path, state)
493
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
494
+
495
+ fake = FakeRiddle()
496
+ load_util_with_fake(fake)
497
+ load_module('recon_supervisor_request', RECON_PATH)
498
+ after_recon = json.loads(state_path.read_text())
499
+
500
+ assert after_recon['recon_status'] == 'needs_supervisor_judgment'
501
+ assert after_recon['before_cdn'] == ''
502
+ assert after_recon['prod_cdn'] == ''
503
+ assert after_recon['author_status'] == 'needs_recon_judgment'
504
+ assert after_recon['recon_assessment_request']['status'] == 'needs_supervising_agent_assessment'
505
+
506
+ latest_attempt = after_recon['recon_results']['attempt_history'][-1]
507
+ approved_baselines = latest_attempt['captured_baselines']
508
+ after_recon['recon_status'] = 'ready_for_proof_plan'
509
+ after_recon['recon_results']['baselines'] = approved_baselines
510
+ after_recon['recon_results']['selected_attempt'] = latest_attempt
511
+ after_recon['before_cdn'] = approved_baselines['before']['url']
512
+ after_recon['prod_cdn'] = approved_baselines['prod']['url']
513
+ after_recon['author_status'] = 'needs_authoring'
514
+ after_recon['proof_plan_status'] = 'needs_authoring'
515
+ after_recon['recon_assessment_request'] = {}
516
+ after_recon['recon_decision_request'] = {}
517
+ state_path.write_text(json.dumps(after_recon, indent=2))
518
+
519
+ fake = FakeRiddle()
520
+ load_util_with_fake(fake)
521
+ with temporary_env(RIDDLE_PROOF_AUTHOR_RUNTIME_MODEL='openai-codex/gpt-5.4'):
522
+ load_module('author_supervisor_request', AUTHOR_PATH)
523
+ after_author = json.loads(state_path.read_text())
524
+
525
+ assert after_author['author_status'] == 'needs_supervisor_judgment'
526
+ assert after_author['proof_plan_status'] == 'needs_supervisor_judgment'
527
+ assert after_author['author_mode'] == 'supervisor_request'
528
+ assert after_author['author_model'] == 'supervising-agent'
529
+ assert after_author['author_runtime_model_hint'] == 'openai-codex/gpt-5.4'
530
+ assert after_author['author_request']['status'] == 'needs_supervisor_judgment'
531
+ assert after_author['author_request']['fallback_defaults']['server_path'] == '/pricing'
532
+ assert 'supervising agent owns proof authoring' in after_author['author_request']['instructions'][0].lower()
533
+
534
+ return {
535
+ 'ok': True,
536
+ 'recon_status': after_recon['recon_status'],
537
+ 'author_status': after_author['author_status'],
538
+ 'author_mode': after_author['author_mode'],
539
+ }
540
+ finally:
541
+ shutil.rmtree(tempdir, ignore_errors=True)
542
+
543
+
544
+ def run_recon_prefers_route_literals_over_import_paths():
545
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-route-literals-'))
546
+ state_path = tempdir / 'state.json'
547
+ try:
548
+ route_snippet = "const Game = lazy(() => import('./Games/TicTacToe'));\nexport const routes = [{ path: '/games/tic-tac-toe', element: <Game /> }];\n"
549
+ state = base_state(tempdir, reference='before')
550
+ make_project(tempdir / 'before', route_snippet)
551
+ make_project(tempdir / 'after', route_snippet)
552
+ state.update({
553
+ 'server_path': '/',
554
+ 'server_path_source': '',
555
+ 'change_request': 'Change the TicTacToe reset button color',
556
+ 'success_criteria': 'The TicTacToe reset button has the requested color on the game route.',
557
+ })
558
+ write_state(state_path, state)
559
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
560
+
561
+ fake = FakeRiddle()
562
+ load_util_with_fake(fake)
563
+ load_module('recon_route_literal_preference', RECON_PATH)
564
+ after_recon = json.loads(state_path.read_text())
565
+
566
+ current_plan = after_recon['recon_results']['current_plan']
567
+ candidate_paths = [item['path'] for item in current_plan['route_candidates']]
568
+ assert current_plan['target_path'] == '/games/tic-tac-toe', current_plan
569
+ assert '/Games/TicTacToe' not in candidate_paths, candidate_paths
570
+ details = after_recon['recon_results']['attempt_history'][-1]['observations']['before']['details']
571
+ assert details['observed_path'] == '/games/tic-tac-toe'
572
+ assert 'Reset Game' in details['visible_text_sample'], details
573
+ assert details['buttons'] == ['Reset Game'], details
574
+
575
+ return {
576
+ 'ok': True,
577
+ 'target_path': current_plan['target_path'],
578
+ 'candidate_paths': candidate_paths,
579
+ }
580
+ finally:
581
+ shutil.rmtree(tempdir, ignore_errors=True)
582
+
583
+
584
+ def run_author_applies_supervisor_packet():
585
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-supervisor-apply-'))
586
+ state_path = tempdir / 'state.json'
587
+ try:
588
+ state = base_state(tempdir, reference='before')
589
+ state.update({
590
+ 'recon_status': 'ready_for_proof_plan',
591
+ 'before_cdn': 'https://cdn.example.com/before.png',
592
+ 'recon_results': {
593
+ 'baselines': {'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'}},
594
+ 'current_plan': {'target_path': '/pricing'},
595
+ 'attempt_history': [{'attempt': 1, 'result': 'success'}],
596
+ },
597
+ 'author_request': {
598
+ 'current_plan': {'target_path': '/pricing'},
599
+ 'observed_baselines': {'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'}},
600
+ },
601
+ 'supervisor_author_packet': {
602
+ 'proof_plan': 'Use the recon-confirmed /pricing route and capture the CTA state once it stabilizes.',
603
+ 'capture_script': "await page.waitForSelector('[data-testid=pricing-cta]'); await saveScreenshot('after-proof');",
604
+ 'refined_inputs': {
605
+ 'server_path': '/pricing',
606
+ 'wait_for_selector': '[data-testid=pricing-cta]',
607
+ 'reference': 'before',
608
+ },
609
+ 'rationale': ['Recon already confirmed the route, so authoring should stay on /pricing.'],
610
+ 'confidence': 'high',
611
+ 'summary': 'Supervisor supplied the proof packet.',
612
+ },
613
+ })
614
+ write_state(state_path, state)
615
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
616
+
617
+ fake = FakeRiddle()
618
+ load_util_with_fake(fake)
619
+ with temporary_env(RIDDLE_PROOF_AUTHOR_RUNTIME_MODEL='openai-codex/gpt-5.4'):
620
+ load_module('author_supervisor_apply', AUTHOR_PATH)
621
+ after_author = json.loads(state_path.read_text())
622
+
623
+ assert after_author['author_status'] == 'ready'
624
+ assert after_author['proof_plan_status'] == 'ready'
625
+ assert after_author['author_mode'] == 'supervising_agent'
626
+ assert after_author['author_model'] == 'supervising-agent:openai-codex/gpt-5.4'
627
+ assert after_author['wait_for_selector'] == '[data-testid=pricing-cta]'
628
+ assert after_author['proof_plan']
629
+ assert after_author['capture_script']
630
+ assert after_author['author_packet']['mode'] == 'supervising_agent'
631
+ assert after_author['author_request']['status'] == 'ready'
632
+ assert after_author['author_request']['authoring_mode'] == 'supervising_agent'
633
+
634
+ return {
635
+ 'ok': True,
636
+ 'author_status': after_author['author_status'],
637
+ 'author_model': after_author['author_model'],
638
+ 'wait_for_selector': after_author['wait_for_selector'],
639
+ }
640
+ finally:
641
+ shutil.rmtree(tempdir, ignore_errors=True)
642
+
643
+
644
+ def run_verify_requests_supervisor_assessment():
645
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-supervisor-'))
646
+ state_path = tempdir / 'state.json'
647
+ try:
648
+ state = base_state(tempdir, reference='both', prod_url='https://prod.example.com/pricing')
649
+ state.update({
650
+ 'recon_status': 'ready_for_proof_plan',
651
+ 'author_status': 'ready',
652
+ 'proof_plan_status': 'ready',
653
+ 'implementation_status': 'changes_detected',
654
+ 'before_cdn': 'https://cdn.example.com/before.png',
655
+ 'prod_cdn': 'https://cdn.example.com/prod.png',
656
+ 'proof_plan': 'Use the recon-confirmed /pricing route and capture the CTA state once it stabilizes.',
657
+ 'capture_script': "await page.waitForSelector('[data-testid=pricing-cta]'); await saveScreenshot('after-proof');",
658
+ 'wait_for_selector': '[data-testid=pricing-cta]',
659
+ 'recon_results': {
660
+ 'baselines': {
661
+ 'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'},
662
+ 'prod': {'path': '/pricing', 'url': 'https://cdn.example.com/prod.png'},
663
+ },
664
+ },
665
+ })
666
+ write_state(state_path, state)
667
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
668
+
669
+ fake = FakeRiddle()
670
+ load_util_with_fake(fake)
671
+ load_module('verify_supervisor_assessment', VERIFY_PATH)
672
+ after_verify = json.loads(state_path.read_text())
673
+
674
+ assert after_verify['verify_status'] == 'evidence_captured'
675
+ assert after_verify['after_cdn'] == 'https://cdn.example.com/after.png'
676
+ assert after_verify['merge_recommendation'] == 'pending-supervisor-judgment'
677
+ assert after_verify['proof_assessment'] == {}
678
+ assert after_verify['proof_assessment_source'] is None
679
+ assert after_verify['proof_assessment_request']['status'] == 'needs_supervising_agent_assessment'
680
+ visual_delta = after_verify['proof_assessment_request']['visual_delta']
681
+ assert visual_delta['status'] == 'unmeasured'
682
+ assert visual_delta['passed'] is None
683
+ semantic_context = after_verify['proof_assessment_request']['semantic_context']
684
+ assert semantic_context['route']['expected_path'] == '/pricing'
685
+ assert semantic_context['route']['after_observed_path'] == '/pricing'
686
+ assert semantic_context['after']['buttons'] == ['Buy Now'], semantic_context
687
+ assert semantic_context['after']['headings'] == ['Pricing'], semantic_context
688
+ assert 'semantic-context' in after_verify['proof_assessment_request']['evidence_basis']
689
+ assert after_verify['proof_assessment_request']['evidence_bundle']['semantic_context']['after']['buttons'] == ['Buy Now']
690
+ assert 'capture success is not proof' in '\n'.join(after_verify['proof_assessment_request']['instructions'])
691
+ assert after_verify['verify_decision_request']['continue_with_stage'] is None
692
+ assert after_verify['verify_results']['baseline']['before']['source'] == 'recon'
693
+ assert after_verify['verify_results']['baseline']['prod']['source'] == 'recon'
694
+ assert after_verify['verify_results']['after']['observation']['valid'] is True
695
+ after_details = after_verify['verify_results']['after']['observation']['details']
696
+ assert after_details['observed_path'] == '/pricing', after_details
697
+ assert after_details['observed_path_raw'] == '/s/pv-after/pricing', after_details
698
+ assert 'Buy Now' in after_details['visible_text_sample'], after_details
699
+ assert after_details['buttons'] == ['Buy Now'], after_details
700
+
701
+ return {
702
+ 'ok': True,
703
+ 'verify_status': after_verify['verify_status'],
704
+ 'merge_recommendation': after_verify['merge_recommendation'],
705
+ 'assessment_status': after_verify['proof_assessment_request']['status'],
706
+ }
707
+ finally:
708
+ shutil.rmtree(tempdir, ignore_errors=True)
709
+
710
+
711
+ def run_verify_structured_evidence_without_screenshot():
712
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-structured-'))
713
+ state_path = tempdir / 'state.json'
714
+ try:
715
+ state = base_state(tempdir, reference='before')
716
+ state.update({
717
+ 'recon_status': 'ready_for_proof_plan',
718
+ 'author_status': 'ready',
719
+ 'proof_plan_status': 'ready',
720
+ 'implementation_status': 'changes_detected',
721
+ 'before_cdn': 'https://cdn.example.com/before.png',
722
+ 'verification_mode': 'audio',
723
+ 'server_path': '/sequencer',
724
+ 'proof_plan': 'Measure the rendered synth transient envelope and compare attack/energy metrics.',
725
+ 'capture_script': (
726
+ "await page.evaluate(() => { "
727
+ "window.__riddleProofEvidence = { "
728
+ "modality: 'audio', attack_ms_before: 42, attack_ms_after: 12, "
729
+ "transient_energy_delta_db: 4.8, passed: true }; });"
730
+ ),
731
+ 'recon_results': {
732
+ 'baselines': {'before': {'path': '/sequencer', 'url': 'https://cdn.example.com/before.png'}},
733
+ },
734
+ })
735
+ write_state(state_path, state)
736
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
737
+
738
+ fake = FakeRiddle()
739
+ load_util_with_fake(fake)
740
+ load_module('verify_structured_evidence', VERIFY_PATH)
741
+ after_verify = json.loads(state_path.read_text())
742
+
743
+ assert after_verify['verify_status'] == 'evidence_captured'
744
+ assert after_verify['after_cdn'] == ''
745
+ assert after_verify['verify_results']['after']['observation']['valid'] is True
746
+ supporting = after_verify['verify_results']['after']['supporting_artifacts']
747
+ assert supporting['has_structured_payload'] is True
748
+ assert supporting['proof_evidence_present'] is True
749
+ script_calls = [
750
+ call['args']['script']
751
+ for call in fake.calls
752
+ if call['tool'] == 'riddle_script'
753
+ ]
754
+ assert script_calls, 'verify should run a proof capture script'
755
+ capture_script = script_calls[-1]
756
+ assert 'globalThis.__riddleProofEvidence ??' not in capture_script
757
+ assert 'typeof globalThis !== "undefined"' in capture_script
758
+ assert '__riddleProofEvidenceRoot.__riddleProofEvidence' in capture_script
759
+ assert 'attack_ms_after' in supporting['proof_evidence_sample']
760
+ assert after_verify['evidence_bundle']['proof_evidence']['attack_ms_after'] == 12
761
+ assert after_verify['evidence_bundle']['after']['proof_evidence']['attack_ms_after'] == 12
762
+ assert after_verify['proof_assessment_request']['evidence_bundle']['after']['supporting_artifacts']['proof_evidence_present'] is True
763
+ assert 'structured-artifacts' in after_verify['proof_assessment_request']['evidence_basis']
764
+ assert 'semantic-context' in after_verify['proof_assessment_request']['evidence_basis']
765
+ assert after_verify['proof_assessment_request']['semantic_context']['route']['after_observed_path'] == '/sequencer'
766
+
767
+ return {
768
+ 'ok': True,
769
+ 'verify_status': after_verify['verify_status'],
770
+ 'after_cdn': after_verify['after_cdn'],
771
+ 'structured': supporting['has_structured_payload'],
772
+ }
773
+ finally:
774
+ shutil.rmtree(tempdir, ignore_errors=True)
775
+
776
+
777
+ def run_verify_audio_requires_proof_evidence():
778
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-audio-gate-'))
779
+ state_path = tempdir / 'state.json'
780
+ try:
781
+ state = base_state(tempdir, reference='before')
782
+ state.update({
783
+ 'recon_status': 'ready_for_proof_plan',
784
+ 'author_status': 'ready',
785
+ 'proof_plan_status': 'ready',
786
+ 'implementation_status': 'changes_detected',
787
+ 'before_cdn': 'https://cdn.example.com/before.png',
788
+ 'verification_mode': 'audio',
789
+ 'server_path': '/sequencer',
790
+ 'proof_plan': 'Measure the rendered synth transient envelope and compare attack/energy metrics.',
791
+ 'capture_script': "await page.evaluate(() => { document.body.dataset.audioNoProof = '1'; });",
792
+ 'recon_results': {
793
+ 'baselines': {'before': {'path': '/sequencer', 'url': 'https://cdn.example.com/before.png'}},
794
+ },
795
+ })
796
+ write_state(state_path, state)
797
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
798
+
799
+ fake = FakeRiddle()
800
+ load_util_with_fake(fake)
801
+ load_module('verify_audio_requires_proof_evidence', VERIFY_PATH)
802
+ after_verify = json.loads(state_path.read_text())
803
+
804
+ assert after_verify['verify_status'] == 'capture_incomplete'
805
+ assert after_verify['proof_assessment_request'] == {}
806
+ observation = after_verify['verify_results']['after']['observation']
807
+ assert observation['valid'] is True
808
+ supporting = after_verify['verify_results']['after']['supporting_artifacts']
809
+ assert supporting['has_structured_payload'] is True
810
+ assert supporting['proof_evidence_present'] is False
811
+ capture_quality = after_verify['verify_decision_request']['capture_quality']
812
+ assert capture_quality['decision'] == 'missing_proof_evidence'
813
+ assert capture_quality['recommended_stage'] == 'author'
814
+ assert 'Audio verification requires proof_evidence_present=true' in capture_quality['summary']
815
+ assert 'Structured proof evidence gate' in after_verify['proof_summary']
816
+
817
+ return {
818
+ 'ok': True,
819
+ 'verify_status': after_verify['verify_status'],
820
+ 'decision': capture_quality['decision'],
821
+ }
822
+ finally:
823
+ shutil.rmtree(tempdir, ignore_errors=True)
824
+
825
+
826
+ def run_verify_audio_rejects_failed_nested_proof_evidence():
827
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-audio-failed-gate-'))
828
+ state_path = tempdir / 'state.json'
829
+ try:
830
+ state = base_state(tempdir, reference='before')
831
+ state.update({
832
+ 'recon_status': 'ready_for_proof_plan',
833
+ 'author_status': 'ready',
834
+ 'proof_plan_status': 'ready',
835
+ 'implementation_status': 'changes_detected',
836
+ 'before_cdn': 'https://cdn.example.com/before.png',
837
+ 'verification_mode': 'audio',
838
+ 'server_path': '/sequencer',
839
+ 'proof_plan': 'Measure the rendered synth transient envelope and compare attack/energy metrics.',
840
+ 'capture_script': "await page.evaluate(() => { document.body.dataset.audioFailedProof = '1'; });",
841
+ 'recon_results': {
842
+ 'baselines': {'before': {'path': '/sequencer', 'url': 'https://cdn.example.com/before.png'}},
843
+ },
844
+ })
845
+ write_state(state_path, state)
846
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
847
+
848
+ fake = FakeRiddle()
849
+ load_util_with_fake(fake)
850
+ load_module('verify_audio_rejects_failed_nested_proof_evidence', VERIFY_PATH)
851
+ after_verify = json.loads(state_path.read_text())
852
+
853
+ assert after_verify['verify_status'] == 'capture_incomplete'
854
+ assert after_verify['proof_assessment_request'] == {}
855
+ observation = after_verify['verify_results']['after']['observation']
856
+ assert observation['valid'] is True
857
+ assert observation['details']['has_errors'] is False
858
+ supporting = after_verify['verify_results']['after']['supporting_artifacts']
859
+ assert supporting['has_structured_payload'] is True
860
+ assert supporting['proof_evidence_present'] is True
861
+ assert 'Failed to fetch dynamically imported module' in supporting['proof_evidence_sample']
862
+ capture_quality = after_verify['verify_decision_request']['capture_quality']
863
+ assert capture_quality['decision'] == 'failed_proof_evidence'
864
+ assert capture_quality['recommended_stage'] == 'author'
865
+ assert 'proof_evidence_present=false' in capture_quality['summary']
866
+ assert 'source_audio_ok' in capture_quality['summary']
867
+ assert 'Structured proof evidence gate' in after_verify['proof_summary']
868
+
869
+ return {
870
+ 'ok': True,
871
+ 'verify_status': after_verify['verify_status'],
872
+ 'decision': capture_quality['decision'],
873
+ }
874
+ finally:
875
+ shutil.rmtree(tempdir, ignore_errors=True)
876
+
877
+
878
+ def run_verify_capture_retry():
879
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-capture-retry-'))
880
+ state_path = tempdir / 'state.json'
881
+ try:
882
+ state = base_state(tempdir, reference='before')
883
+ state.update({
884
+ 'recon_status': 'ready_for_proof_plan',
885
+ 'author_status': 'ready',
886
+ 'proof_plan_status': 'ready',
887
+ 'implementation_status': 'changes_detected',
888
+ 'before_cdn': 'https://cdn.example.com/before.png',
889
+ 'proof_plan': 'Capture the CTA interaction on the pricing route.',
890
+ 'capture_script': "await page.goto('https://preview.example.com/wrong/'); await saveScreenshot('after-proof-bad');",
891
+ 'recon_results': {
892
+ 'baselines': {'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'}},
893
+ },
894
+ })
895
+ write_state(state_path, state)
896
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
897
+
898
+ fake = FakeRiddle()
899
+ load_util_with_fake(fake)
900
+ load_module('verify_capture_retry', VERIFY_PATH)
901
+ after_verify = json.loads(state_path.read_text())
902
+
903
+ assert after_verify['verify_status'] == 'capture_incomplete'
904
+ assert after_verify['merge_recommendation'] == 'do-not-merge'
905
+ assert after_verify['proof_assessment'] == {}
906
+ assert after_verify['proof_assessment_request'] == {}
907
+ assert after_verify['verify_decision_request']['recommended_stage'] in ('author', 'recon')
908
+ assert after_verify['verify_decision_request']['continue_with_stage'] in ('author', 'recon')
909
+ assert after_verify['verify_results']['after']['observation']['valid'] is False
910
+
911
+ return {
912
+ 'ok': True,
913
+ 'verify_status': after_verify['verify_status'],
914
+ 'continue_with_stage': after_verify['verify_decision_request']['continue_with_stage'],
915
+ }
916
+ finally:
917
+ shutil.rmtree(tempdir, ignore_errors=True)
918
+
919
+
920
+ def run_verify_missing_baseline():
921
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-missing-baseline-'))
922
+ state_path = tempdir / 'state.json'
923
+ try:
924
+ state = base_state(tempdir, reference='before')
925
+ state.update({
926
+ 'recon_status': 'ready_for_proof_plan',
927
+ 'author_status': 'ready',
928
+ 'proof_plan_status': 'ready',
929
+ 'implementation_status': 'changes_detected',
930
+ 'proof_plan': 'Capture the CTA interaction on the pricing route.',
931
+ 'capture_script': "await saveScreenshot('after-proof');",
932
+ 'before_cdn': '',
933
+ 'recon_results': {},
934
+ })
935
+ write_state(state_path, state)
936
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
937
+
938
+ fake = FakeRiddle()
939
+ load_util_with_fake(fake)
940
+ try:
941
+ load_module('verify_missing_baseline', VERIFY_PATH)
942
+ except SystemExit as exc:
943
+ message = str(exc)
944
+ assert 'Recon baseline missing' in message, message
945
+ return {'ok': True, 'error': message}
946
+ raise AssertionError('verify should have failed when recon baseline was missing')
947
+ finally:
948
+ shutil.rmtree(tempdir, ignore_errors=True)
949
+
950
+
951
+ def run_ship_missing_supervisor_gate():
952
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-gate-'))
953
+ state_path = tempdir / 'state.json'
954
+ try:
955
+ state = base_state(tempdir, reference='before')
956
+ state.update({
957
+ 'recon_status': 'ready_for_proof_plan',
958
+ 'author_status': 'ready',
959
+ 'proof_plan_status': 'ready',
960
+ 'implementation_status': 'changes_detected',
961
+ 'verify_status': 'evidence_captured',
962
+ 'before_cdn': 'https://cdn.example.com/before.png',
963
+ 'after_cdn': 'https://cdn.example.com/after.png',
964
+ 'proof_assessment': {},
965
+ 'proof_assessment_source': None,
966
+ })
967
+ write_state(state_path, state)
968
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
969
+
970
+ try:
971
+ load_module('ship_missing_supervisor_gate', SHIP_PATH)
972
+ except SystemExit as exc:
973
+ message = str(exc)
974
+ assert 'proof_assessment.decision=ready_to_ship' in message, message
975
+ return {'ok': True, 'error': message}
976
+ raise AssertionError('ship should have failed without supervising-agent ready_to_ship assessment')
977
+ finally:
978
+ shutil.rmtree(tempdir, ignore_errors=True)
979
+
980
+
981
+ def run_ship_accepts_structured_after_evidence():
982
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-structured-'))
983
+ state_path = tempdir / 'state.json'
984
+ try:
985
+ state = base_state(tempdir, reference='before')
986
+ state.update({
987
+ 'repo_dir': str(tempdir),
988
+ 'after_worktree': str(tempdir / 'missing-after-worktree'),
989
+ 'recon_status': 'ready_for_proof_plan',
990
+ 'author_status': 'ready',
991
+ 'proof_plan_status': 'ready',
992
+ 'implementation_status': 'changes_detected',
993
+ 'verify_status': 'evidence_captured',
994
+ 'before_cdn': 'https://cdn.example.com/before.png',
995
+ 'after_cdn': '',
996
+ 'proof_assessment': {
997
+ 'decision': 'ready_to_ship',
998
+ 'summary': 'Structured audio metrics prove the attack increased.',
999
+ 'reasons': ['proofEvidence contains the requested transient metric change'],
1000
+ 'source': 'supervising_agent',
1001
+ },
1002
+ 'proof_assessment_source': 'supervising_agent',
1003
+ 'evidence_bundle': {
1004
+ 'verification_mode': 'audio',
1005
+ 'expected_path': '/sequencer',
1006
+ 'after': {
1007
+ 'observation': {'valid': True, 'telemetry_ready': True, 'reason': 'ok'},
1008
+ 'supporting_artifacts': {
1009
+ 'has_structured_payload': True,
1010
+ 'proof_evidence_present': True,
1011
+ 'proof_evidence_sample': '{"attack_ms_after":12,"passed":true}',
1012
+ },
1013
+ },
1014
+ },
1015
+ 'finalized': True,
1016
+ 'pr_url': 'https://github.com/example/repo/pull/1',
1017
+ 'pr_number': '1',
1018
+ 'marked_ready': True,
1019
+ 'proof_assessment_comment_posted': True,
1020
+ 'discord_notification': {'ok': True, 'pr_url': 'https://github.com/example/repo/pull/1'},
1021
+ })
1022
+ write_state(state_path, state)
1023
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
1024
+ sys.modules.pop('util', None)
1025
+
1026
+ try:
1027
+ load_module('ship_structured_after_evidence', SHIP_PATH)
1028
+ except SystemExit as exc:
1029
+ assert exc.code == 0 or str(exc) == '0', exc
1030
+ after_ship = json.loads(state_path.read_text())
1031
+ assert after_ship['stage'] == 'ship'
1032
+ assert after_ship['merge_recommendation'].startswith('ready_to_ship')
1033
+ assert after_ship['ship_report']['pr_url'] == 'https://github.com/example/repo/pull/1'
1034
+ assert after_ship['ship_report']['before_artifact_url'] == 'https://cdn.example.com/before.png'
1035
+ return {'ok': True, 'stage': after_ship['stage'], 'after_cdn': after_ship['after_cdn']}
1036
+ raise AssertionError('ship should have exited after finalized structured-evidence sync')
1037
+ finally:
1038
+ shutil.rmtree(tempdir, ignore_errors=True)
1039
+
1040
+
1041
+ def run_ship_discord_thread_target():
1042
+ sys.modules.pop('util', None)
1043
+ source = SHIP_PATH.read_text()
1044
+ helpers_source = source.split('\ns = load_state()', 1)[0]
1045
+ namespace = {'__file__': str(SHIP_PATH)}
1046
+ exec(compile(helpers_source, str(SHIP_PATH), 'exec'), namespace)
1047
+
1048
+ thread_target = namespace['discord_message_target']({
1049
+ 'discord_channel': 'parent-channel-123',
1050
+ 'discord_thread_id': 'thread-456',
1051
+ 'discord_message_id': 'message-789',
1052
+ 'discord_source_url': 'https://discord.com/channels/guild/thread-456/message-789',
1053
+ })
1054
+ assert thread_target['ok'] is True
1055
+ assert thread_target['target_channel_id'] == 'thread-456'
1056
+ assert thread_target['parent_channel_id'] == 'parent-channel-123'
1057
+ assert 'message_reference' not in thread_target
1058
+
1059
+ reply_target = namespace['discord_message_target']({
1060
+ 'discord_channel': 'parent-channel-123',
1061
+ 'discord_message_id': 'message-789',
1062
+ })
1063
+ assert reply_target['ok'] is True
1064
+ assert reply_target['target_channel_id'] == 'parent-channel-123'
1065
+ assert reply_target['message_reference']['message_id'] == 'message-789'
1066
+ assert reply_target['message_reference']['channel_id'] == 'parent-channel-123'
1067
+
1068
+ missing_target = namespace['discord_message_target']({'discord_message_id': 'message-789'})
1069
+ assert missing_target['ok'] is False
1070
+ assert 'discord_channel or discord_thread_id' in missing_target['reason']
1071
+
1072
+ return {'ok': True, 'thread_target': thread_target['target_channel_id'], 'reply_target': reply_target['target_channel_id']}
1073
+
1074
+
1075
+ def run_ship_filters_tool_noise_when_staging():
1076
+ sys.modules.pop('util', None)
1077
+ source = SHIP_PATH.read_text()
1078
+ helpers_source = source.split('\ns = load_state()', 1)[0]
1079
+ namespace = {'__file__': str(SHIP_PATH)}
1080
+ exec(compile(helpers_source, str(SHIP_PATH), 'exec'), namespace)
1081
+
1082
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-stage-'))
1083
+ try:
1084
+ sp.run(['git', 'init', '-b', 'main'], cwd=tempdir, check=True, capture_output=True, text=True)
1085
+ sp.run(['git', 'config', 'user.email', 'test@example.com'], cwd=tempdir, check=True)
1086
+ sp.run(['git', 'config', 'user.name', 'Test User'], cwd=tempdir, check=True)
1087
+ (tempdir / 'tracked.txt').write_text('before\n')
1088
+ sp.run(['git', 'add', 'tracked.txt'], cwd=tempdir, check=True)
1089
+ sp.run(['git', 'commit', '-m', 'initial'], cwd=tempdir, check=True, capture_output=True, text=True)
1090
+
1091
+ (tempdir / 'tracked.txt').write_text('after\n')
1092
+ (tempdir / 'new-code.txt').write_text('new\n')
1093
+ (tempdir / '.codex').write_text('agent scratch\n')
1094
+ (tempdir / '.oc-smoke').mkdir()
1095
+ (tempdir / '.oc-smoke' / 'err').write_text('smoke noise\n')
1096
+
1097
+ status = sp.run(['git', 'status', '--porcelain'], cwd=tempdir, check=True, capture_output=True, text=True).stdout
1098
+ lines = namespace['committable_status_lines'](status)
1099
+ assert any('tracked.txt' in line for line in lines), lines
1100
+ assert any('new-code.txt' in line for line in lines), lines
1101
+ assert not any('.codex' in line for line in lines), lines
1102
+ assert not any('.oc-smoke' in line for line in lines), lines
1103
+
1104
+ staged = namespace['stage_committable_changes'](str(tempdir))
1105
+ assert 'tracked.txt' in staged, staged
1106
+ assert 'new-code.txt' in staged, staged
1107
+ assert '.codex' not in staged, staged
1108
+ assert '.oc-smoke/err' not in staged, staged
1109
+
1110
+ cached = sp.run(['git', 'diff', '--cached', '--name-only'], cwd=tempdir, check=True, capture_output=True, text=True).stdout.splitlines()
1111
+ assert 'tracked.txt' in cached, cached
1112
+ assert 'new-code.txt' in cached, cached
1113
+ assert '.codex' not in cached, cached
1114
+ assert '.oc-smoke/err' not in cached, cached
1115
+
1116
+ return {'ok': True, 'staged': staged}
1117
+ finally:
1118
+ shutil.rmtree(tempdir, ignore_errors=True)
1119
+
1120
+
1121
+ def run_ship_resolves_real_pr_branch():
1122
+ sys.modules.pop('util', None)
1123
+ source = SHIP_PATH.read_text()
1124
+ helpers_source = source.split('\ns = load_state()', 1)[0]
1125
+
1126
+ tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-branch-'))
1127
+ try:
1128
+ state_path = tempdir / 'state.json'
1129
+ os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
1130
+ namespace = {'__file__': str(SHIP_PATH)}
1131
+ exec(compile(helpers_source, str(SHIP_PATH), 'exec'), namespace)
1132
+ state = {
1133
+ 'branch': 'riddle-proof/rp_test-after',
1134
+ 'target_branch': 'riddle-proof/rp_test-after',
1135
+ 'after_worktree_branch': 'riddle-proof/rp_test-after',
1136
+ 'pr_number': '257',
1137
+ 'pr_url': 'https://github.com/example/repo/pull/257',
1138
+ }
1139
+ write_state(state_path, state)
1140
+ namespace['pr_head_branch'] = lambda repo_dir, pr_ref: 'ttt-status-polish-proof'
1141
+ branch = namespace['resolve_ship_branch'](state, str(tempdir))
1142
+ assert branch == 'ttt-status-polish-proof', branch
1143
+ after_state = json.loads(state_path.read_text())
1144
+ assert after_state['branch'] == 'ttt-status-polish-proof'
1145
+ assert after_state['target_branch'] == 'ttt-status-polish-proof'
1146
+
1147
+ temp_state = {
1148
+ 'branch': 'riddle-proof/rp_test-after',
1149
+ 'target_branch': 'riddle-proof/rp_test-after',
1150
+ 'after_worktree_branch': 'riddle-proof/rp_test-after',
1151
+ 'change_request': 'Ship a clean proof branch',
1152
+ 'run_id': 'rp_test_audio_abcdef',
1153
+ }
1154
+ namespace['pr_head_branch'] = lambda repo_dir, pr_ref: ''
1155
+ recovered_branch = namespace['resolve_ship_branch'](temp_state, str(tempdir))
1156
+ assert recovered_branch.startswith('agent/openclaw/ship-a-clean-proof-branch-'), recovered_branch
1157
+ assert temp_state['ship_branch_recovered_from'] == 'riddle-proof/rp_test-after'
1158
+
1159
+ ambiguous_pr_state = {
1160
+ 'branch': 'riddle-proof/rp_test-after',
1161
+ 'target_branch': 'riddle-proof/rp_test-after',
1162
+ 'after_worktree_branch': 'riddle-proof/rp_test-after',
1163
+ 'pr_number': '999',
1164
+ 'pr_url': 'https://github.com/example/repo/pull/999',
1165
+ }
1166
+ try:
1167
+ namespace['resolve_ship_branch'](ambiguous_pr_state, str(tempdir))
1168
+ except SystemExit as exc:
1169
+ assert 'temporary proof branch' in str(exc), exc
1170
+ else:
1171
+ raise AssertionError('temporary proof branch should still be rejected for unresolved existing PRs')
1172
+ return {'ok': True, 'branch': branch, 'recovered_branch': recovered_branch}
1173
+ finally:
1174
+ shutil.rmtree(tempdir, ignore_errors=True)
1175
+
1176
+
1177
+ if __name__ == '__main__':
1178
+ payload = {
1179
+ 'capture_artifact_enrichment': run_capture_artifact_enrichment(),
1180
+ 'capture_diagnostics_redaction': run_capture_diagnostics_redact_sensitive_values(),
1181
+ 'apply_auth_context': run_apply_auth_context_passes_supported_auth_payloads(),
1182
+ 'verify_quality_ignores_proof_telemetry_console_text': run_verify_quality_ignores_proof_telemetry_console_text(),
1183
+ 'recon_then_author_request': run_recon_then_author_request(),
1184
+ 'recon_route_literal_preference': run_recon_prefers_route_literals_over_import_paths(),
1185
+ 'author_applies_supervisor_packet': run_author_applies_supervisor_packet(),
1186
+ 'verify_requests_supervisor_assessment': run_verify_requests_supervisor_assessment(),
1187
+ 'verify_structured_evidence_without_screenshot': run_verify_structured_evidence_without_screenshot(),
1188
+ 'verify_audio_requires_proof_evidence': run_verify_audio_requires_proof_evidence(),
1189
+ 'verify_audio_rejects_failed_nested_proof_evidence': run_verify_audio_rejects_failed_nested_proof_evidence(),
1190
+ 'verify_capture_retry': run_verify_capture_retry(),
1191
+ 'missing_baseline_guard': run_verify_missing_baseline(),
1192
+ 'ship_supervisor_gate': run_ship_missing_supervisor_gate(),
1193
+ 'ship_structured_after_evidence': run_ship_accepts_structured_after_evidence(),
1194
+ 'ship_discord_thread_target': run_ship_discord_thread_target(),
1195
+ 'ship_filters_tool_noise_when_staging': run_ship_filters_tool_noise_when_staging(),
1196
+ 'ship_resolves_real_pr_branch': run_ship_resolves_real_pr_branch(),
1197
+ }
1198
+ print(json.dumps(payload, indent=2))