@riddledc/riddle-proof 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/chunk-3MHFLQKG.js +853 -0
- package/dist/{chunk-LVP22WE4.js → chunk-5GZZZ6JA.js} +5 -1
- package/dist/engine-harness.cjs +2505 -22
- package/dist/engine-harness.js +1 -1
- package/dist/index.cjs +2512 -29
- package/dist/index.js +2 -2
- package/dist/openclaw.cjs +1 -1
- package/dist/openclaw.js +1 -1
- package/dist/proof-run-core.cjs +909 -0
- package/dist/proof-run-core.d.cts +280 -0
- package/dist/proof-run-core.d.ts +280 -0
- package/dist/proof-run-core.js +48 -0
- package/dist/proof-run-engine.cjs +2499 -0
- package/dist/proof-run-engine.d.cts +677 -0
- package/dist/proof-run-engine.d.ts +677 -0
- package/dist/proof-run-engine.js +1649 -0
- package/lib/workspace-core.mjs +391 -0
- package/package.json +15 -3
- package/runtime/lib/author.py +343 -0
- package/runtime/lib/implement.py +63 -0
- package/runtime/lib/preflight.py +246 -0
- package/runtime/lib/recon.py +1048 -0
- package/runtime/lib/riddle_core_call.mjs +151 -0
- package/runtime/lib/setup.py +387 -0
- package/runtime/lib/ship.py +834 -0
- package/runtime/lib/util.py +673 -0
- package/runtime/lib/verify.py +1223 -0
- package/runtime/pipelines/riddle-proof-author.lobster +28 -0
- package/runtime/pipelines/riddle-proof-implement.lobster +26 -0
- package/runtime/pipelines/riddle-proof-recon.lobster +79 -0
- package/runtime/pipelines/riddle-proof-setup.lobster +141 -0
- package/runtime/pipelines/riddle-proof-ship.lobster +36 -0
- package/runtime/pipelines/riddle-proof-verify.lobster +74 -0
- package/runtime/tests/recon_verify_smoke.py +1198 -0
|
@@ -0,0 +1,1198 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import subprocess as sp
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
12
|
+
LIB = ROOT / 'lib'
|
|
13
|
+
UTIL_PATH = LIB / 'util.py'
|
|
14
|
+
RECON_PATH = LIB / 'recon.py'
|
|
15
|
+
VERIFY_PATH = LIB / 'verify.py'
|
|
16
|
+
AUTHOR_PATH = LIB / 'author.py'
|
|
17
|
+
SHIP_PATH = LIB / 'ship.py'
|
|
18
|
+
|
|
19
|
+
BUILD_SCRIPT = "python3 -c \"from pathlib import Path; Path('build').mkdir(exist_ok=True); Path('build/index.html').write_text('<html>ok</html>')\""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def state_console(payload):
|
|
23
|
+
payload = dict(payload)
|
|
24
|
+
payload.setdefault('consoleSummary', {'error_count': 0})
|
|
25
|
+
payload.setdefault('failedChecks', [])
|
|
26
|
+
return {
|
|
27
|
+
'entries': {
|
|
28
|
+
'log': [{'message': 'RIDDLE_PROOF_STATE:' + json.dumps(payload)}],
|
|
29
|
+
'warn': [],
|
|
30
|
+
'error': [],
|
|
31
|
+
'info': [],
|
|
32
|
+
},
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load_module(name: str, path: Path):
|
|
37
|
+
sys.modules.pop(name, None)
|
|
38
|
+
spec = importlib.util.spec_from_file_location(name, path)
|
|
39
|
+
module = importlib.util.module_from_spec(spec)
|
|
40
|
+
sys.modules[name] = module
|
|
41
|
+
assert spec.loader is not None
|
|
42
|
+
spec.loader.exec_module(module)
|
|
43
|
+
return module
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class FakeRiddle:
|
|
47
|
+
def __init__(self):
|
|
48
|
+
self.calls = []
|
|
49
|
+
|
|
50
|
+
def invoke(self, tool, args, timeout=180):
|
|
51
|
+
self.calls.append({'tool': tool, 'args': args, 'timeout': timeout})
|
|
52
|
+
if tool == 'riddle_preview_delete':
|
|
53
|
+
return {'ok': True}
|
|
54
|
+
raise AssertionError(f'unexpected invoke tool: {tool}')
|
|
55
|
+
|
|
56
|
+
def invoke_retry(self, tool, args, retries=3, timeout=180):
|
|
57
|
+
self.calls.append({'tool': tool, 'args': args, 'timeout': timeout, 'retries': retries})
|
|
58
|
+
if tool == 'riddle_preview':
|
|
59
|
+
label = args.get('label', 'preview')
|
|
60
|
+
return {
|
|
61
|
+
'ok': True,
|
|
62
|
+
'id': f'pv-{label}',
|
|
63
|
+
'preview_url': f'https://preview.example.com/{label}/',
|
|
64
|
+
}
|
|
65
|
+
if tool == 'riddle_script':
|
|
66
|
+
script = args.get('script', '')
|
|
67
|
+
if 'preview.example.com' in script and '/wrong' in script:
|
|
68
|
+
return {
|
|
69
|
+
'ok': True,
|
|
70
|
+
'screenshots': [{'url': 'https://cdn.example.com/wrong.png'}],
|
|
71
|
+
'outputs': [{'name': 'wrong.png', 'url': 'https://cdn.example.com/wrong.png'}],
|
|
72
|
+
'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":5,"interactiveElements":0,"pathname":"/wrong","title":"Wrong"}'],
|
|
73
|
+
}
|
|
74
|
+
if 'audioNoProof' in script:
|
|
75
|
+
page_state = {
|
|
76
|
+
'bodyTextLength': 96,
|
|
77
|
+
'visibleTextSample': 'Neon step sequencer audio workbench',
|
|
78
|
+
'interactiveElements': 0,
|
|
79
|
+
'visibleInteractiveElements': 0,
|
|
80
|
+
'pathname': '/s/pv-after/sequencer',
|
|
81
|
+
'title': 'Sequencer',
|
|
82
|
+
'buttons': [],
|
|
83
|
+
'headings': ['Sequencer'],
|
|
84
|
+
'links': [],
|
|
85
|
+
'canvasCount': 1,
|
|
86
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
'ok': True,
|
|
90
|
+
'outputs': [{'name': 'metrics.json', 'url': 'https://cdn.example.com/metrics.json'}],
|
|
91
|
+
'result': {'pageState': page_state, 'summary': {'captured': True}},
|
|
92
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps(page_state)],
|
|
93
|
+
}
|
|
94
|
+
if 'audioFailedProof' in script:
|
|
95
|
+
page_state = {
|
|
96
|
+
'bodyTextLength': 96,
|
|
97
|
+
'visibleTextSample': 'Neon step sequencer audio workbench',
|
|
98
|
+
'interactiveElements': 0,
|
|
99
|
+
'visibleInteractiveElements': 0,
|
|
100
|
+
'pathname': '/s/pv-after/sequencer',
|
|
101
|
+
'title': 'Sequencer',
|
|
102
|
+
'buttons': [],
|
|
103
|
+
'headings': ['Sequencer'],
|
|
104
|
+
'links': [],
|
|
105
|
+
'canvasCount': 1,
|
|
106
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
|
|
107
|
+
}
|
|
108
|
+
proof_evidence = {
|
|
109
|
+
'proof_evidence_present': False,
|
|
110
|
+
'evidence_summary': 'Structured audio/source proof did not satisfy all required Monkberry release-tail checks.',
|
|
111
|
+
'checks': {
|
|
112
|
+
'route_ok': True,
|
|
113
|
+
'ui_context_ok': True,
|
|
114
|
+
'source_audio_ok': False,
|
|
115
|
+
'monkberry_scope_ok': False,
|
|
116
|
+
},
|
|
117
|
+
'import_error': 'Failed to fetch dynamically imported module: /src/Games/songs/index.js',
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
'ok': True,
|
|
121
|
+
'outputs': [{'name': 'proof.json', 'url': 'https://cdn.example.com/proof.json'}],
|
|
122
|
+
'result': {'pageState': page_state},
|
|
123
|
+
'console': [
|
|
124
|
+
'RIDDLE_PROOF_STATE:' + json.dumps(page_state),
|
|
125
|
+
'RIDDLE_PROOF_EVIDENCE ' + json.dumps(proof_evidence),
|
|
126
|
+
],
|
|
127
|
+
}
|
|
128
|
+
if 'window.__riddleProofEvidence' in script or 'globalThis.__riddleProofEvidence' in script:
|
|
129
|
+
page_state = {
|
|
130
|
+
'bodyTextLength': 36,
|
|
131
|
+
'visibleTextSample': 'Neon step sequencer audio workbench',
|
|
132
|
+
'interactiveElements': 0,
|
|
133
|
+
'visibleInteractiveElements': 0,
|
|
134
|
+
'pathname': '/s/pv-after/sequencer',
|
|
135
|
+
'title': 'Sequencer',
|
|
136
|
+
'buttons': [],
|
|
137
|
+
'headings': ['Sequencer'],
|
|
138
|
+
'links': [],
|
|
139
|
+
'canvasCount': 1,
|
|
140
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
|
|
141
|
+
}
|
|
142
|
+
proof_evidence = {
|
|
143
|
+
'modality': 'audio',
|
|
144
|
+
'attack_ms_before': 42,
|
|
145
|
+
'attack_ms_after': 12,
|
|
146
|
+
'transient_energy_delta_db': 4.8,
|
|
147
|
+
'passed': True,
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
'ok': True,
|
|
151
|
+
'outputs': [{'name': 'metrics.json', 'url': 'https://cdn.example.com/metrics.json'}],
|
|
152
|
+
'result': {'proofEvidence': proof_evidence},
|
|
153
|
+
'console': [
|
|
154
|
+
'RIDDLE_PROOF_STATE:' + json.dumps(page_state),
|
|
155
|
+
'RIDDLE_PROOF_EVIDENCE:' + json.dumps(proof_evidence),
|
|
156
|
+
],
|
|
157
|
+
}
|
|
158
|
+
if 'after-proof' in script:
|
|
159
|
+
return {
|
|
160
|
+
'ok': True,
|
|
161
|
+
'screenshots': [{'url': 'https://cdn.example.com/after.png'}],
|
|
162
|
+
'outputs': [{'name': 'after.png', 'url': 'https://cdn.example.com/after.png'}],
|
|
163
|
+
'console': state_console({
|
|
164
|
+
'bodyTextLength': 180,
|
|
165
|
+
'visibleTextSample': 'Pricing CTA Buy Now',
|
|
166
|
+
'interactiveElements': 4,
|
|
167
|
+
'visibleInteractiveElements': 4,
|
|
168
|
+
'pathname': '/s/pv-after/pricing',
|
|
169
|
+
'title': 'After',
|
|
170
|
+
'buttons': ['Buy Now'],
|
|
171
|
+
'headings': ['Pricing'],
|
|
172
|
+
'links': [],
|
|
173
|
+
'canvasCount': 0,
|
|
174
|
+
'largeVisibleElements': [{'tag': 'button', 'text': 'Buy Now'}],
|
|
175
|
+
}),
|
|
176
|
+
}
|
|
177
|
+
if 'prod.example.com/pricing' in script:
|
|
178
|
+
return {
|
|
179
|
+
'ok': True,
|
|
180
|
+
'screenshots': [{'url': 'https://cdn.example.com/prod.png'}],
|
|
181
|
+
'outputs': [{'name': 'prod.png', 'url': 'https://cdn.example.com/prod.png'}],
|
|
182
|
+
'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":180,"interactiveElements":4,"pathname":"/pricing","title":"Prod"}'],
|
|
183
|
+
}
|
|
184
|
+
if 'preview.example.com' in script and '/pricing' in script:
|
|
185
|
+
return {
|
|
186
|
+
'ok': True,
|
|
187
|
+
'screenshots': [{'url': 'https://cdn.example.com/before.png'}],
|
|
188
|
+
'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/before.png'}],
|
|
189
|
+
'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":180,"interactiveElements":4,"pathname":"/pricing","title":"Before"}'],
|
|
190
|
+
}
|
|
191
|
+
if 'preview.example.com' in script and '/games/tic-tac-toe' in script:
|
|
192
|
+
return {
|
|
193
|
+
'ok': True,
|
|
194
|
+
'screenshots': [{'url': 'https://cdn.example.com/tictactoe-before.png'}],
|
|
195
|
+
'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/tictactoe-before.png'}],
|
|
196
|
+
'console': state_console({
|
|
197
|
+
'bodyTextLength': 220,
|
|
198
|
+
'visibleTextSample': 'LilArcade Tic Tac Toe Player X Reset Game',
|
|
199
|
+
'interactiveElements': 5,
|
|
200
|
+
'visibleInteractiveElements': 5,
|
|
201
|
+
'pathname': '/s/pv-before/games/tic-tac-toe',
|
|
202
|
+
'title': 'TicTacToe',
|
|
203
|
+
'buttons': ['Reset Game'],
|
|
204
|
+
'headings': ['Tic Tac Toe'],
|
|
205
|
+
'links': [],
|
|
206
|
+
'canvasCount': 0,
|
|
207
|
+
'largeVisibleElements': [{'tag': 'button', 'text': 'Reset Game'}],
|
|
208
|
+
}),
|
|
209
|
+
}
|
|
210
|
+
raise AssertionError(f'unexpected riddle_script payload: {script}')
|
|
211
|
+
raise AssertionError(f'unexpected invoke_retry tool: {tool}')
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def make_project(root: Path, route_snippet: str):
|
|
215
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
216
|
+
(root / 'src').mkdir(exist_ok=True)
|
|
217
|
+
(root / 'src' / 'routes.tsx').write_text(route_snippet)
|
|
218
|
+
(root / 'package.json').write_text(json.dumps({
|
|
219
|
+
'name': root.name,
|
|
220
|
+
'scripts': {'build': BUILD_SCRIPT},
|
|
221
|
+
'dependencies': {'react': '18.0.0', 'react-router-dom': '6.0.0'},
|
|
222
|
+
}, indent=2))
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def write_state(path: Path, payload: dict):
|
|
226
|
+
path.write_text(json.dumps(payload, indent=2))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def run_capture_artifact_enrichment():
|
|
230
|
+
util = load_module('util_artifact_enrichment', UTIL_PATH)
|
|
231
|
+
fixtures = {
|
|
232
|
+
'https://cdn.example.com/console.json': {
|
|
233
|
+
'summary': {'total_entries': 1, 'error_count': 1},
|
|
234
|
+
'entries': {
|
|
235
|
+
'log': [],
|
|
236
|
+
'warn': [],
|
|
237
|
+
'error': [{'message': 'page.waitForSelector: Timeout 30000ms exceeded'}],
|
|
238
|
+
'info': [],
|
|
239
|
+
},
|
|
240
|
+
},
|
|
241
|
+
'https://cdn.example.com/proof.json': {
|
|
242
|
+
'script_error': 'page.waitForSelector: Timeout 30000ms exceeded',
|
|
243
|
+
'metrics': {'wall_ms': 30000},
|
|
244
|
+
},
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
def fake_fetch_json_artifact(url, max_bytes=0):
|
|
248
|
+
return fixtures[url], ''
|
|
249
|
+
|
|
250
|
+
util.fetch_json_artifact = fake_fetch_json_artifact
|
|
251
|
+
payload = {
|
|
252
|
+
'ok': True,
|
|
253
|
+
'outputs': [
|
|
254
|
+
{'name': 'screenshot_1.png', 'url': 'https://cdn.example.com/screenshot_1.png'},
|
|
255
|
+
{'name': 'console.json', 'url': 'https://cdn.example.com/console.json'},
|
|
256
|
+
{'name': 'proof.json', 'url': 'https://cdn.example.com/proof.json'},
|
|
257
|
+
],
|
|
258
|
+
'screenshots': [{'name': 'screenshot_1.png', 'url': 'https://cdn.example.com/screenshot_1.png'}],
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
enriched = util.enrich_capture_payload(payload)
|
|
262
|
+
summary = util.summarize_capture_artifacts(payload)
|
|
263
|
+
assert enriched['console']['summary']['error_count'] == 1
|
|
264
|
+
assert enriched['_proof_json']['script_error'].startswith('page.waitForSelector')
|
|
265
|
+
assert summary['artifact_json'] == ['console.json', 'proof.json']
|
|
266
|
+
assert summary['proof_script_error'] is True
|
|
267
|
+
assert summary['console_summary']['error_count'] == 1
|
|
268
|
+
|
|
269
|
+
return {
|
|
270
|
+
'ok': True,
|
|
271
|
+
'artifact_json': summary['artifact_json'],
|
|
272
|
+
'proof_script_error': summary['proof_script_error'],
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def run_capture_diagnostics_redact_sensitive_values():
|
|
277
|
+
util = load_module('util_capture_diagnostics', UTIL_PATH)
|
|
278
|
+
state = {}
|
|
279
|
+
args = {
|
|
280
|
+
'script': 'await page.goto("https://example.com");',
|
|
281
|
+
'localStorage': {'accessToken': 'secret-token'},
|
|
282
|
+
'cookies': [{'name': 'session', 'value': 'secret-cookie'}],
|
|
283
|
+
'headers': {'Authorization': 'Bearer secret-token'},
|
|
284
|
+
'nested': {'api_key': 'secret-key', 'safe': 'ok'},
|
|
285
|
+
}
|
|
286
|
+
payload = {
|
|
287
|
+
'ok': False,
|
|
288
|
+
'error': 'page crashed',
|
|
289
|
+
'outputs': [{'name': 'console.json', 'url': 'https://cdn.example.com/console.json'}],
|
|
290
|
+
'artifacts': [{
|
|
291
|
+
'name': 'metering.json',
|
|
292
|
+
'kind': 'json',
|
|
293
|
+
'role': 'diagnostic',
|
|
294
|
+
'path': '/tmp/riddle-proof/metering.json',
|
|
295
|
+
'metadata': {'samples': 64},
|
|
296
|
+
}],
|
|
297
|
+
}
|
|
298
|
+
diagnostic = util.append_capture_diagnostic(state, 'after', 'riddle_server_preview', args, payload)
|
|
299
|
+
|
|
300
|
+
assert diagnostic['version'] == 'riddle-proof.capture-diagnostic.v1'
|
|
301
|
+
assert diagnostic['tool'] == 'riddle_server_preview'
|
|
302
|
+
assert diagnostic['args']['script'].startswith('await page.goto')
|
|
303
|
+
assert diagnostic['args']['localStorage'] == '[redacted]'
|
|
304
|
+
assert diagnostic['args']['cookies'] == '[redacted]'
|
|
305
|
+
assert diagnostic['args']['headers'] == '[redacted]'
|
|
306
|
+
assert diagnostic['args']['nested']['api_key'] == '[redacted]'
|
|
307
|
+
assert diagnostic['args']['nested']['safe'] == 'ok'
|
|
308
|
+
assert state['capture_diagnostics'][-1]['artifact_summary']['outputs'][0]['name'] == 'console.json'
|
|
309
|
+
assert state['capture_diagnostics'][-1]['artifact_summary']['artifacts'][0]['metadata_keys'] == ['samples']
|
|
310
|
+
return {'ok': True, 'diagnostics': len(state['capture_diagnostics'])}
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def run_apply_auth_context_passes_supported_auth_payloads():
|
|
314
|
+
util = load_module('util_apply_auth_context', UTIL_PATH)
|
|
315
|
+
state = {
|
|
316
|
+
'auth_localStorage': {'token': 'local'},
|
|
317
|
+
'auth_cookies': [{'name': 'session', 'value': 'cookie'}],
|
|
318
|
+
'auth_headers': {'Authorization': 'Bearer header'},
|
|
319
|
+
}
|
|
320
|
+
args = {'script': 'await page.goto("https://example.com");'}
|
|
321
|
+
util.apply_auth_context(state, args)
|
|
322
|
+
assert util.has_auth_context(state) is True
|
|
323
|
+
assert args['localStorage'] == state['auth_localStorage']
|
|
324
|
+
assert args['cookies'] == state['auth_cookies']
|
|
325
|
+
assert args['headers'] == state['auth_headers']
|
|
326
|
+
assert util.has_auth_context({}) is False
|
|
327
|
+
assert util.has_auth_context({'use_auth': 'true'}) is True
|
|
328
|
+
return {'ok': True, 'arg_keys': sorted(args.keys())}
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def run_verify_quality_ignores_proof_telemetry_console_text():
|
|
332
|
+
sys.modules.pop('util', None)
|
|
333
|
+
source = VERIFY_PATH.read_text()
|
|
334
|
+
helpers_source = source.split('\ns = load_state()', 1)[0]
|
|
335
|
+
namespace = {'__file__': str(VERIFY_PATH)}
|
|
336
|
+
exec(compile(helpers_source, str(VERIFY_PATH), 'exec'), namespace)
|
|
337
|
+
|
|
338
|
+
telemetry_payload = {
|
|
339
|
+
'bodyTextLength': 180,
|
|
340
|
+
'visibleTextSample': 'LilArcade Circle Maze Coin Clicker',
|
|
341
|
+
'interactiveElements': 4,
|
|
342
|
+
'visibleInteractiveElements': 4,
|
|
343
|
+
'pathname': '/',
|
|
344
|
+
'title': 'LilArcade',
|
|
345
|
+
'headings': ['LilArcade'],
|
|
346
|
+
'links': [{'text': 'Circle Maze', 'href': '/games/circle-maze'}],
|
|
347
|
+
'canvasCount': 0,
|
|
348
|
+
'largeVisibleElements': [{'tag': 'h1', 'text': 'LilArcade'}],
|
|
349
|
+
'console_summary': {'error_count': 0},
|
|
350
|
+
'failed_checks': [],
|
|
351
|
+
}
|
|
352
|
+
quality = namespace['evaluate_capture_quality']({
|
|
353
|
+
'ok': True,
|
|
354
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
355
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
356
|
+
'console': [
|
|
357
|
+
'RIDDLE_PROOF_STATE:' + json.dumps(telemetry_payload),
|
|
358
|
+
'RIDDLE_PROOF_EVIDENCE:' + json.dumps({'passed': True, 'error_count': 0}),
|
|
359
|
+
],
|
|
360
|
+
}, '/', 'visual')
|
|
361
|
+
assert quality['valid'] is True
|
|
362
|
+
assert quality['details']['has_errors'] is False
|
|
363
|
+
assert quality['details']['capture_error_messages'] == []
|
|
364
|
+
|
|
365
|
+
runtime_error_quality = namespace['evaluate_capture_quality']({
|
|
366
|
+
'ok': True,
|
|
367
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
368
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
369
|
+
'console': [
|
|
370
|
+
'RIDDLE_PROOF_STATE:' + json.dumps(telemetry_payload),
|
|
371
|
+
'Uncaught TypeError: boom',
|
|
372
|
+
],
|
|
373
|
+
}, '/', 'visual')
|
|
374
|
+
assert runtime_error_quality['valid'] is False
|
|
375
|
+
assert runtime_error_quality['details']['has_errors'] is True
|
|
376
|
+
assert 'console/runtime errors' in runtime_error_quality['reason']
|
|
377
|
+
|
|
378
|
+
strong_delta = namespace['extract_visual_delta']({
|
|
379
|
+
'ok': True,
|
|
380
|
+
'result': {
|
|
381
|
+
'proofEvidence': {
|
|
382
|
+
'change_pct': '2.31',
|
|
383
|
+
'changed_pixels': 22395,
|
|
384
|
+
'width': 1080,
|
|
385
|
+
'height': 900,
|
|
386
|
+
},
|
|
387
|
+
},
|
|
388
|
+
})
|
|
389
|
+
assert strong_delta['status'] == 'measured'
|
|
390
|
+
assert strong_delta['passed'] is True
|
|
391
|
+
assert strong_delta['change_percent'] == 2.31
|
|
392
|
+
assert strong_delta['changed_pixels'] == 22395
|
|
393
|
+
|
|
394
|
+
weak_delta = namespace['extract_visual_delta']({
|
|
395
|
+
'ok': True,
|
|
396
|
+
'result': {
|
|
397
|
+
'proofEvidence': {
|
|
398
|
+
'change_pct': '0.06',
|
|
399
|
+
'changed_pixels': 616,
|
|
400
|
+
'width': 1080,
|
|
401
|
+
'height': 900,
|
|
402
|
+
},
|
|
403
|
+
},
|
|
404
|
+
})
|
|
405
|
+
assert weak_delta['status'] == 'measured'
|
|
406
|
+
assert weak_delta['passed'] is False
|
|
407
|
+
assert 'below the legibility threshold' in weak_delta['reason']
|
|
408
|
+
|
|
409
|
+
unmeasured_delta = namespace['extract_visual_delta']({
|
|
410
|
+
'ok': True,
|
|
411
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
412
|
+
})
|
|
413
|
+
assert unmeasured_delta['status'] == 'unmeasured'
|
|
414
|
+
assert unmeasured_delta['passed'] is None
|
|
415
|
+
|
|
416
|
+
return {'ok': True, 'telemetry_valid': quality['valid'], 'weak_delta_passed': weak_delta['passed']}
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def load_util_with_fake(fake: FakeRiddle):
|
|
420
|
+
util = load_module('util', UTIL_PATH)
|
|
421
|
+
util.invoke = fake.invoke
|
|
422
|
+
util.invoke_retry = fake.invoke_retry
|
|
423
|
+
return util
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
@contextmanager
|
|
427
|
+
def temporary_env(**updates):
|
|
428
|
+
sentinel = object()
|
|
429
|
+
previous = {}
|
|
430
|
+
for key, value in updates.items():
|
|
431
|
+
previous[key] = os.environ.get(key, sentinel)
|
|
432
|
+
if value is None:
|
|
433
|
+
os.environ.pop(key, None)
|
|
434
|
+
else:
|
|
435
|
+
os.environ[key] = value
|
|
436
|
+
try:
|
|
437
|
+
yield
|
|
438
|
+
finally:
|
|
439
|
+
for key, value in previous.items():
|
|
440
|
+
if value is sentinel:
|
|
441
|
+
os.environ.pop(key, None)
|
|
442
|
+
else:
|
|
443
|
+
os.environ[key] = value
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def base_state(tempdir: Path, *, reference='before', prod_url=''):
|
|
447
|
+
before_dir = tempdir / 'before'
|
|
448
|
+
after_dir = tempdir / 'after'
|
|
449
|
+
make_project(before_dir, "export const routes = [{ path: '/pricing', element: <Pricing /> }];\n")
|
|
450
|
+
make_project(after_dir, "export const routes = [{ path: '/pricing', element: <Pricing /> }];\n")
|
|
451
|
+
return {
|
|
452
|
+
'workspace_ready': True,
|
|
453
|
+
'repo_dir': str(tempdir),
|
|
454
|
+
'before_worktree': str(before_dir),
|
|
455
|
+
'after_worktree': str(after_dir),
|
|
456
|
+
'mode': 'static',
|
|
457
|
+
'reference': reference,
|
|
458
|
+
'requested_reference': reference,
|
|
459
|
+
'prod_url': prod_url,
|
|
460
|
+
'change_request': 'Fix the pricing CTA layout',
|
|
461
|
+
'success_criteria': 'Pricing CTA is visible and aligned on the pricing route.',
|
|
462
|
+
'verification_mode': 'visual',
|
|
463
|
+
'build_command': BUILD_SCRIPT,
|
|
464
|
+
'build_output': 'build',
|
|
465
|
+
'capture_script': '',
|
|
466
|
+
'proof_plan_status': 'pending_recon',
|
|
467
|
+
'author_status': 'pending_recon',
|
|
468
|
+
'implementation_status': 'pending_recon',
|
|
469
|
+
'wait_for_selector': '',
|
|
470
|
+
'server_path': '/pricing',
|
|
471
|
+
'allow_static_preview_fallback': True,
|
|
472
|
+
'auth_localStorage': {},
|
|
473
|
+
'before_cdn': '',
|
|
474
|
+
'after_cdn': '',
|
|
475
|
+
'prod_cdn': '',
|
|
476
|
+
'proof_plan': '',
|
|
477
|
+
'proof_plan_request': {},
|
|
478
|
+
'author_request': {},
|
|
479
|
+
'recon_results': {},
|
|
480
|
+
'recon_decision_request': {},
|
|
481
|
+
'verify_results': {},
|
|
482
|
+
'proof_assessment': {},
|
|
483
|
+
'proof_assessment_request': {},
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def run_recon_then_author_request():
|
|
488
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-supervisor-request-'))
|
|
489
|
+
state_path = tempdir / 'state.json'
|
|
490
|
+
try:
|
|
491
|
+
state = base_state(tempdir, reference='both', prod_url='https://prod.example.com/pricing')
|
|
492
|
+
write_state(state_path, state)
|
|
493
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
494
|
+
|
|
495
|
+
fake = FakeRiddle()
|
|
496
|
+
load_util_with_fake(fake)
|
|
497
|
+
load_module('recon_supervisor_request', RECON_PATH)
|
|
498
|
+
after_recon = json.loads(state_path.read_text())
|
|
499
|
+
|
|
500
|
+
assert after_recon['recon_status'] == 'needs_supervisor_judgment'
|
|
501
|
+
assert after_recon['before_cdn'] == ''
|
|
502
|
+
assert after_recon['prod_cdn'] == ''
|
|
503
|
+
assert after_recon['author_status'] == 'needs_recon_judgment'
|
|
504
|
+
assert after_recon['recon_assessment_request']['status'] == 'needs_supervising_agent_assessment'
|
|
505
|
+
|
|
506
|
+
latest_attempt = after_recon['recon_results']['attempt_history'][-1]
|
|
507
|
+
approved_baselines = latest_attempt['captured_baselines']
|
|
508
|
+
after_recon['recon_status'] = 'ready_for_proof_plan'
|
|
509
|
+
after_recon['recon_results']['baselines'] = approved_baselines
|
|
510
|
+
after_recon['recon_results']['selected_attempt'] = latest_attempt
|
|
511
|
+
after_recon['before_cdn'] = approved_baselines['before']['url']
|
|
512
|
+
after_recon['prod_cdn'] = approved_baselines['prod']['url']
|
|
513
|
+
after_recon['author_status'] = 'needs_authoring'
|
|
514
|
+
after_recon['proof_plan_status'] = 'needs_authoring'
|
|
515
|
+
after_recon['recon_assessment_request'] = {}
|
|
516
|
+
after_recon['recon_decision_request'] = {}
|
|
517
|
+
state_path.write_text(json.dumps(after_recon, indent=2))
|
|
518
|
+
|
|
519
|
+
fake = FakeRiddle()
|
|
520
|
+
load_util_with_fake(fake)
|
|
521
|
+
with temporary_env(RIDDLE_PROOF_AUTHOR_RUNTIME_MODEL='openai-codex/gpt-5.4'):
|
|
522
|
+
load_module('author_supervisor_request', AUTHOR_PATH)
|
|
523
|
+
after_author = json.loads(state_path.read_text())
|
|
524
|
+
|
|
525
|
+
assert after_author['author_status'] == 'needs_supervisor_judgment'
|
|
526
|
+
assert after_author['proof_plan_status'] == 'needs_supervisor_judgment'
|
|
527
|
+
assert after_author['author_mode'] == 'supervisor_request'
|
|
528
|
+
assert after_author['author_model'] == 'supervising-agent'
|
|
529
|
+
assert after_author['author_runtime_model_hint'] == 'openai-codex/gpt-5.4'
|
|
530
|
+
assert after_author['author_request']['status'] == 'needs_supervisor_judgment'
|
|
531
|
+
assert after_author['author_request']['fallback_defaults']['server_path'] == '/pricing'
|
|
532
|
+
assert 'supervising agent owns proof authoring' in after_author['author_request']['instructions'][0].lower()
|
|
533
|
+
|
|
534
|
+
return {
|
|
535
|
+
'ok': True,
|
|
536
|
+
'recon_status': after_recon['recon_status'],
|
|
537
|
+
'author_status': after_author['author_status'],
|
|
538
|
+
'author_mode': after_author['author_mode'],
|
|
539
|
+
}
|
|
540
|
+
finally:
|
|
541
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def run_recon_prefers_route_literals_over_import_paths():
|
|
545
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-route-literals-'))
|
|
546
|
+
state_path = tempdir / 'state.json'
|
|
547
|
+
try:
|
|
548
|
+
route_snippet = "const Game = lazy(() => import('./Games/TicTacToe'));\nexport const routes = [{ path: '/games/tic-tac-toe', element: <Game /> }];\n"
|
|
549
|
+
state = base_state(tempdir, reference='before')
|
|
550
|
+
make_project(tempdir / 'before', route_snippet)
|
|
551
|
+
make_project(tempdir / 'after', route_snippet)
|
|
552
|
+
state.update({
|
|
553
|
+
'server_path': '/',
|
|
554
|
+
'server_path_source': '',
|
|
555
|
+
'change_request': 'Change the TicTacToe reset button color',
|
|
556
|
+
'success_criteria': 'The TicTacToe reset button has the requested color on the game route.',
|
|
557
|
+
})
|
|
558
|
+
write_state(state_path, state)
|
|
559
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
560
|
+
|
|
561
|
+
fake = FakeRiddle()
|
|
562
|
+
load_util_with_fake(fake)
|
|
563
|
+
load_module('recon_route_literal_preference', RECON_PATH)
|
|
564
|
+
after_recon = json.loads(state_path.read_text())
|
|
565
|
+
|
|
566
|
+
current_plan = after_recon['recon_results']['current_plan']
|
|
567
|
+
candidate_paths = [item['path'] for item in current_plan['route_candidates']]
|
|
568
|
+
assert current_plan['target_path'] == '/games/tic-tac-toe', current_plan
|
|
569
|
+
assert '/Games/TicTacToe' not in candidate_paths, candidate_paths
|
|
570
|
+
details = after_recon['recon_results']['attempt_history'][-1]['observations']['before']['details']
|
|
571
|
+
assert details['observed_path'] == '/games/tic-tac-toe'
|
|
572
|
+
assert 'Reset Game' in details['visible_text_sample'], details
|
|
573
|
+
assert details['buttons'] == ['Reset Game'], details
|
|
574
|
+
|
|
575
|
+
return {
|
|
576
|
+
'ok': True,
|
|
577
|
+
'target_path': current_plan['target_path'],
|
|
578
|
+
'candidate_paths': candidate_paths,
|
|
579
|
+
}
|
|
580
|
+
finally:
|
|
581
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def run_author_applies_supervisor_packet():
|
|
585
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-supervisor-apply-'))
|
|
586
|
+
state_path = tempdir / 'state.json'
|
|
587
|
+
try:
|
|
588
|
+
state = base_state(tempdir, reference='before')
|
|
589
|
+
state.update({
|
|
590
|
+
'recon_status': 'ready_for_proof_plan',
|
|
591
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
592
|
+
'recon_results': {
|
|
593
|
+
'baselines': {'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'}},
|
|
594
|
+
'current_plan': {'target_path': '/pricing'},
|
|
595
|
+
'attempt_history': [{'attempt': 1, 'result': 'success'}],
|
|
596
|
+
},
|
|
597
|
+
'author_request': {
|
|
598
|
+
'current_plan': {'target_path': '/pricing'},
|
|
599
|
+
'observed_baselines': {'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'}},
|
|
600
|
+
},
|
|
601
|
+
'supervisor_author_packet': {
|
|
602
|
+
'proof_plan': 'Use the recon-confirmed /pricing route and capture the CTA state once it stabilizes.',
|
|
603
|
+
'capture_script': "await page.waitForSelector('[data-testid=pricing-cta]'); await saveScreenshot('after-proof');",
|
|
604
|
+
'refined_inputs': {
|
|
605
|
+
'server_path': '/pricing',
|
|
606
|
+
'wait_for_selector': '[data-testid=pricing-cta]',
|
|
607
|
+
'reference': 'before',
|
|
608
|
+
},
|
|
609
|
+
'rationale': ['Recon already confirmed the route, so authoring should stay on /pricing.'],
|
|
610
|
+
'confidence': 'high',
|
|
611
|
+
'summary': 'Supervisor supplied the proof packet.',
|
|
612
|
+
},
|
|
613
|
+
})
|
|
614
|
+
write_state(state_path, state)
|
|
615
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
616
|
+
|
|
617
|
+
fake = FakeRiddle()
|
|
618
|
+
load_util_with_fake(fake)
|
|
619
|
+
with temporary_env(RIDDLE_PROOF_AUTHOR_RUNTIME_MODEL='openai-codex/gpt-5.4'):
|
|
620
|
+
load_module('author_supervisor_apply', AUTHOR_PATH)
|
|
621
|
+
after_author = json.loads(state_path.read_text())
|
|
622
|
+
|
|
623
|
+
assert after_author['author_status'] == 'ready'
|
|
624
|
+
assert after_author['proof_plan_status'] == 'ready'
|
|
625
|
+
assert after_author['author_mode'] == 'supervising_agent'
|
|
626
|
+
assert after_author['author_model'] == 'supervising-agent:openai-codex/gpt-5.4'
|
|
627
|
+
assert after_author['wait_for_selector'] == '[data-testid=pricing-cta]'
|
|
628
|
+
assert after_author['proof_plan']
|
|
629
|
+
assert after_author['capture_script']
|
|
630
|
+
assert after_author['author_packet']['mode'] == 'supervising_agent'
|
|
631
|
+
assert after_author['author_request']['status'] == 'ready'
|
|
632
|
+
assert after_author['author_request']['authoring_mode'] == 'supervising_agent'
|
|
633
|
+
|
|
634
|
+
return {
|
|
635
|
+
'ok': True,
|
|
636
|
+
'author_status': after_author['author_status'],
|
|
637
|
+
'author_model': after_author['author_model'],
|
|
638
|
+
'wait_for_selector': after_author['wait_for_selector'],
|
|
639
|
+
}
|
|
640
|
+
finally:
|
|
641
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def run_verify_requests_supervisor_assessment():
|
|
645
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-supervisor-'))
|
|
646
|
+
state_path = tempdir / 'state.json'
|
|
647
|
+
try:
|
|
648
|
+
state = base_state(tempdir, reference='both', prod_url='https://prod.example.com/pricing')
|
|
649
|
+
state.update({
|
|
650
|
+
'recon_status': 'ready_for_proof_plan',
|
|
651
|
+
'author_status': 'ready',
|
|
652
|
+
'proof_plan_status': 'ready',
|
|
653
|
+
'implementation_status': 'changes_detected',
|
|
654
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
655
|
+
'prod_cdn': 'https://cdn.example.com/prod.png',
|
|
656
|
+
'proof_plan': 'Use the recon-confirmed /pricing route and capture the CTA state once it stabilizes.',
|
|
657
|
+
'capture_script': "await page.waitForSelector('[data-testid=pricing-cta]'); await saveScreenshot('after-proof');",
|
|
658
|
+
'wait_for_selector': '[data-testid=pricing-cta]',
|
|
659
|
+
'recon_results': {
|
|
660
|
+
'baselines': {
|
|
661
|
+
'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'},
|
|
662
|
+
'prod': {'path': '/pricing', 'url': 'https://cdn.example.com/prod.png'},
|
|
663
|
+
},
|
|
664
|
+
},
|
|
665
|
+
})
|
|
666
|
+
write_state(state_path, state)
|
|
667
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
668
|
+
|
|
669
|
+
fake = FakeRiddle()
|
|
670
|
+
load_util_with_fake(fake)
|
|
671
|
+
load_module('verify_supervisor_assessment', VERIFY_PATH)
|
|
672
|
+
after_verify = json.loads(state_path.read_text())
|
|
673
|
+
|
|
674
|
+
assert after_verify['verify_status'] == 'evidence_captured'
|
|
675
|
+
assert after_verify['after_cdn'] == 'https://cdn.example.com/after.png'
|
|
676
|
+
assert after_verify['merge_recommendation'] == 'pending-supervisor-judgment'
|
|
677
|
+
assert after_verify['proof_assessment'] == {}
|
|
678
|
+
assert after_verify['proof_assessment_source'] is None
|
|
679
|
+
assert after_verify['proof_assessment_request']['status'] == 'needs_supervising_agent_assessment'
|
|
680
|
+
visual_delta = after_verify['proof_assessment_request']['visual_delta']
|
|
681
|
+
assert visual_delta['status'] == 'unmeasured'
|
|
682
|
+
assert visual_delta['passed'] is None
|
|
683
|
+
semantic_context = after_verify['proof_assessment_request']['semantic_context']
|
|
684
|
+
assert semantic_context['route']['expected_path'] == '/pricing'
|
|
685
|
+
assert semantic_context['route']['after_observed_path'] == '/pricing'
|
|
686
|
+
assert semantic_context['after']['buttons'] == ['Buy Now'], semantic_context
|
|
687
|
+
assert semantic_context['after']['headings'] == ['Pricing'], semantic_context
|
|
688
|
+
assert 'semantic-context' in after_verify['proof_assessment_request']['evidence_basis']
|
|
689
|
+
assert after_verify['proof_assessment_request']['evidence_bundle']['semantic_context']['after']['buttons'] == ['Buy Now']
|
|
690
|
+
assert 'capture success is not proof' in '\n'.join(after_verify['proof_assessment_request']['instructions'])
|
|
691
|
+
assert after_verify['verify_decision_request']['continue_with_stage'] is None
|
|
692
|
+
assert after_verify['verify_results']['baseline']['before']['source'] == 'recon'
|
|
693
|
+
assert after_verify['verify_results']['baseline']['prod']['source'] == 'recon'
|
|
694
|
+
assert after_verify['verify_results']['after']['observation']['valid'] is True
|
|
695
|
+
after_details = after_verify['verify_results']['after']['observation']['details']
|
|
696
|
+
assert after_details['observed_path'] == '/pricing', after_details
|
|
697
|
+
assert after_details['observed_path_raw'] == '/s/pv-after/pricing', after_details
|
|
698
|
+
assert 'Buy Now' in after_details['visible_text_sample'], after_details
|
|
699
|
+
assert after_details['buttons'] == ['Buy Now'], after_details
|
|
700
|
+
|
|
701
|
+
return {
|
|
702
|
+
'ok': True,
|
|
703
|
+
'verify_status': after_verify['verify_status'],
|
|
704
|
+
'merge_recommendation': after_verify['merge_recommendation'],
|
|
705
|
+
'assessment_status': after_verify['proof_assessment_request']['status'],
|
|
706
|
+
}
|
|
707
|
+
finally:
|
|
708
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def run_verify_structured_evidence_without_screenshot():
|
|
712
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-structured-'))
|
|
713
|
+
state_path = tempdir / 'state.json'
|
|
714
|
+
try:
|
|
715
|
+
state = base_state(tempdir, reference='before')
|
|
716
|
+
state.update({
|
|
717
|
+
'recon_status': 'ready_for_proof_plan',
|
|
718
|
+
'author_status': 'ready',
|
|
719
|
+
'proof_plan_status': 'ready',
|
|
720
|
+
'implementation_status': 'changes_detected',
|
|
721
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
722
|
+
'verification_mode': 'audio',
|
|
723
|
+
'server_path': '/sequencer',
|
|
724
|
+
'proof_plan': 'Measure the rendered synth transient envelope and compare attack/energy metrics.',
|
|
725
|
+
'capture_script': (
|
|
726
|
+
"await page.evaluate(() => { "
|
|
727
|
+
"window.__riddleProofEvidence = { "
|
|
728
|
+
"modality: 'audio', attack_ms_before: 42, attack_ms_after: 12, "
|
|
729
|
+
"transient_energy_delta_db: 4.8, passed: true }; });"
|
|
730
|
+
),
|
|
731
|
+
'recon_results': {
|
|
732
|
+
'baselines': {'before': {'path': '/sequencer', 'url': 'https://cdn.example.com/before.png'}},
|
|
733
|
+
},
|
|
734
|
+
})
|
|
735
|
+
write_state(state_path, state)
|
|
736
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
737
|
+
|
|
738
|
+
fake = FakeRiddle()
|
|
739
|
+
load_util_with_fake(fake)
|
|
740
|
+
load_module('verify_structured_evidence', VERIFY_PATH)
|
|
741
|
+
after_verify = json.loads(state_path.read_text())
|
|
742
|
+
|
|
743
|
+
assert after_verify['verify_status'] == 'evidence_captured'
|
|
744
|
+
assert after_verify['after_cdn'] == ''
|
|
745
|
+
assert after_verify['verify_results']['after']['observation']['valid'] is True
|
|
746
|
+
supporting = after_verify['verify_results']['after']['supporting_artifacts']
|
|
747
|
+
assert supporting['has_structured_payload'] is True
|
|
748
|
+
assert supporting['proof_evidence_present'] is True
|
|
749
|
+
script_calls = [
|
|
750
|
+
call['args']['script']
|
|
751
|
+
for call in fake.calls
|
|
752
|
+
if call['tool'] == 'riddle_script'
|
|
753
|
+
]
|
|
754
|
+
assert script_calls, 'verify should run a proof capture script'
|
|
755
|
+
capture_script = script_calls[-1]
|
|
756
|
+
assert 'globalThis.__riddleProofEvidence ??' not in capture_script
|
|
757
|
+
assert 'typeof globalThis !== "undefined"' in capture_script
|
|
758
|
+
assert '__riddleProofEvidenceRoot.__riddleProofEvidence' in capture_script
|
|
759
|
+
assert 'attack_ms_after' in supporting['proof_evidence_sample']
|
|
760
|
+
assert after_verify['evidence_bundle']['proof_evidence']['attack_ms_after'] == 12
|
|
761
|
+
assert after_verify['evidence_bundle']['after']['proof_evidence']['attack_ms_after'] == 12
|
|
762
|
+
assert after_verify['proof_assessment_request']['evidence_bundle']['after']['supporting_artifacts']['proof_evidence_present'] is True
|
|
763
|
+
assert 'structured-artifacts' in after_verify['proof_assessment_request']['evidence_basis']
|
|
764
|
+
assert 'semantic-context' in after_verify['proof_assessment_request']['evidence_basis']
|
|
765
|
+
assert after_verify['proof_assessment_request']['semantic_context']['route']['after_observed_path'] == '/sequencer'
|
|
766
|
+
|
|
767
|
+
return {
|
|
768
|
+
'ok': True,
|
|
769
|
+
'verify_status': after_verify['verify_status'],
|
|
770
|
+
'after_cdn': after_verify['after_cdn'],
|
|
771
|
+
'structured': supporting['has_structured_payload'],
|
|
772
|
+
}
|
|
773
|
+
finally:
|
|
774
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def run_verify_audio_requires_proof_evidence():
|
|
778
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-audio-gate-'))
|
|
779
|
+
state_path = tempdir / 'state.json'
|
|
780
|
+
try:
|
|
781
|
+
state = base_state(tempdir, reference='before')
|
|
782
|
+
state.update({
|
|
783
|
+
'recon_status': 'ready_for_proof_plan',
|
|
784
|
+
'author_status': 'ready',
|
|
785
|
+
'proof_plan_status': 'ready',
|
|
786
|
+
'implementation_status': 'changes_detected',
|
|
787
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
788
|
+
'verification_mode': 'audio',
|
|
789
|
+
'server_path': '/sequencer',
|
|
790
|
+
'proof_plan': 'Measure the rendered synth transient envelope and compare attack/energy metrics.',
|
|
791
|
+
'capture_script': "await page.evaluate(() => { document.body.dataset.audioNoProof = '1'; });",
|
|
792
|
+
'recon_results': {
|
|
793
|
+
'baselines': {'before': {'path': '/sequencer', 'url': 'https://cdn.example.com/before.png'}},
|
|
794
|
+
},
|
|
795
|
+
})
|
|
796
|
+
write_state(state_path, state)
|
|
797
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
798
|
+
|
|
799
|
+
fake = FakeRiddle()
|
|
800
|
+
load_util_with_fake(fake)
|
|
801
|
+
load_module('verify_audio_requires_proof_evidence', VERIFY_PATH)
|
|
802
|
+
after_verify = json.loads(state_path.read_text())
|
|
803
|
+
|
|
804
|
+
assert after_verify['verify_status'] == 'capture_incomplete'
|
|
805
|
+
assert after_verify['proof_assessment_request'] == {}
|
|
806
|
+
observation = after_verify['verify_results']['after']['observation']
|
|
807
|
+
assert observation['valid'] is True
|
|
808
|
+
supporting = after_verify['verify_results']['after']['supporting_artifacts']
|
|
809
|
+
assert supporting['has_structured_payload'] is True
|
|
810
|
+
assert supporting['proof_evidence_present'] is False
|
|
811
|
+
capture_quality = after_verify['verify_decision_request']['capture_quality']
|
|
812
|
+
assert capture_quality['decision'] == 'missing_proof_evidence'
|
|
813
|
+
assert capture_quality['recommended_stage'] == 'author'
|
|
814
|
+
assert 'Audio verification requires proof_evidence_present=true' in capture_quality['summary']
|
|
815
|
+
assert 'Structured proof evidence gate' in after_verify['proof_summary']
|
|
816
|
+
|
|
817
|
+
return {
|
|
818
|
+
'ok': True,
|
|
819
|
+
'verify_status': after_verify['verify_status'],
|
|
820
|
+
'decision': capture_quality['decision'],
|
|
821
|
+
}
|
|
822
|
+
finally:
|
|
823
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def run_verify_audio_rejects_failed_nested_proof_evidence():
|
|
827
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-audio-failed-gate-'))
|
|
828
|
+
state_path = tempdir / 'state.json'
|
|
829
|
+
try:
|
|
830
|
+
state = base_state(tempdir, reference='before')
|
|
831
|
+
state.update({
|
|
832
|
+
'recon_status': 'ready_for_proof_plan',
|
|
833
|
+
'author_status': 'ready',
|
|
834
|
+
'proof_plan_status': 'ready',
|
|
835
|
+
'implementation_status': 'changes_detected',
|
|
836
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
837
|
+
'verification_mode': 'audio',
|
|
838
|
+
'server_path': '/sequencer',
|
|
839
|
+
'proof_plan': 'Measure the rendered synth transient envelope and compare attack/energy metrics.',
|
|
840
|
+
'capture_script': "await page.evaluate(() => { document.body.dataset.audioFailedProof = '1'; });",
|
|
841
|
+
'recon_results': {
|
|
842
|
+
'baselines': {'before': {'path': '/sequencer', 'url': 'https://cdn.example.com/before.png'}},
|
|
843
|
+
},
|
|
844
|
+
})
|
|
845
|
+
write_state(state_path, state)
|
|
846
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
847
|
+
|
|
848
|
+
fake = FakeRiddle()
|
|
849
|
+
load_util_with_fake(fake)
|
|
850
|
+
load_module('verify_audio_rejects_failed_nested_proof_evidence', VERIFY_PATH)
|
|
851
|
+
after_verify = json.loads(state_path.read_text())
|
|
852
|
+
|
|
853
|
+
assert after_verify['verify_status'] == 'capture_incomplete'
|
|
854
|
+
assert after_verify['proof_assessment_request'] == {}
|
|
855
|
+
observation = after_verify['verify_results']['after']['observation']
|
|
856
|
+
assert observation['valid'] is True
|
|
857
|
+
assert observation['details']['has_errors'] is False
|
|
858
|
+
supporting = after_verify['verify_results']['after']['supporting_artifacts']
|
|
859
|
+
assert supporting['has_structured_payload'] is True
|
|
860
|
+
assert supporting['proof_evidence_present'] is True
|
|
861
|
+
assert 'Failed to fetch dynamically imported module' in supporting['proof_evidence_sample']
|
|
862
|
+
capture_quality = after_verify['verify_decision_request']['capture_quality']
|
|
863
|
+
assert capture_quality['decision'] == 'failed_proof_evidence'
|
|
864
|
+
assert capture_quality['recommended_stage'] == 'author'
|
|
865
|
+
assert 'proof_evidence_present=false' in capture_quality['summary']
|
|
866
|
+
assert 'source_audio_ok' in capture_quality['summary']
|
|
867
|
+
assert 'Structured proof evidence gate' in after_verify['proof_summary']
|
|
868
|
+
|
|
869
|
+
return {
|
|
870
|
+
'ok': True,
|
|
871
|
+
'verify_status': after_verify['verify_status'],
|
|
872
|
+
'decision': capture_quality['decision'],
|
|
873
|
+
}
|
|
874
|
+
finally:
|
|
875
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def run_verify_capture_retry():
|
|
879
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-capture-retry-'))
|
|
880
|
+
state_path = tempdir / 'state.json'
|
|
881
|
+
try:
|
|
882
|
+
state = base_state(tempdir, reference='before')
|
|
883
|
+
state.update({
|
|
884
|
+
'recon_status': 'ready_for_proof_plan',
|
|
885
|
+
'author_status': 'ready',
|
|
886
|
+
'proof_plan_status': 'ready',
|
|
887
|
+
'implementation_status': 'changes_detected',
|
|
888
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
889
|
+
'proof_plan': 'Capture the CTA interaction on the pricing route.',
|
|
890
|
+
'capture_script': "await page.goto('https://preview.example.com/wrong/'); await saveScreenshot('after-proof-bad');",
|
|
891
|
+
'recon_results': {
|
|
892
|
+
'baselines': {'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before.png'}},
|
|
893
|
+
},
|
|
894
|
+
})
|
|
895
|
+
write_state(state_path, state)
|
|
896
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
897
|
+
|
|
898
|
+
fake = FakeRiddle()
|
|
899
|
+
load_util_with_fake(fake)
|
|
900
|
+
load_module('verify_capture_retry', VERIFY_PATH)
|
|
901
|
+
after_verify = json.loads(state_path.read_text())
|
|
902
|
+
|
|
903
|
+
assert after_verify['verify_status'] == 'capture_incomplete'
|
|
904
|
+
assert after_verify['merge_recommendation'] == 'do-not-merge'
|
|
905
|
+
assert after_verify['proof_assessment'] == {}
|
|
906
|
+
assert after_verify['proof_assessment_request'] == {}
|
|
907
|
+
assert after_verify['verify_decision_request']['recommended_stage'] in ('author', 'recon')
|
|
908
|
+
assert after_verify['verify_decision_request']['continue_with_stage'] in ('author', 'recon')
|
|
909
|
+
assert after_verify['verify_results']['after']['observation']['valid'] is False
|
|
910
|
+
|
|
911
|
+
return {
|
|
912
|
+
'ok': True,
|
|
913
|
+
'verify_status': after_verify['verify_status'],
|
|
914
|
+
'continue_with_stage': after_verify['verify_decision_request']['continue_with_stage'],
|
|
915
|
+
}
|
|
916
|
+
finally:
|
|
917
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def run_verify_missing_baseline():
|
|
921
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-missing-baseline-'))
|
|
922
|
+
state_path = tempdir / 'state.json'
|
|
923
|
+
try:
|
|
924
|
+
state = base_state(tempdir, reference='before')
|
|
925
|
+
state.update({
|
|
926
|
+
'recon_status': 'ready_for_proof_plan',
|
|
927
|
+
'author_status': 'ready',
|
|
928
|
+
'proof_plan_status': 'ready',
|
|
929
|
+
'implementation_status': 'changes_detected',
|
|
930
|
+
'proof_plan': 'Capture the CTA interaction on the pricing route.',
|
|
931
|
+
'capture_script': "await saveScreenshot('after-proof');",
|
|
932
|
+
'before_cdn': '',
|
|
933
|
+
'recon_results': {},
|
|
934
|
+
})
|
|
935
|
+
write_state(state_path, state)
|
|
936
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
937
|
+
|
|
938
|
+
fake = FakeRiddle()
|
|
939
|
+
load_util_with_fake(fake)
|
|
940
|
+
try:
|
|
941
|
+
load_module('verify_missing_baseline', VERIFY_PATH)
|
|
942
|
+
except SystemExit as exc:
|
|
943
|
+
message = str(exc)
|
|
944
|
+
assert 'Recon baseline missing' in message, message
|
|
945
|
+
return {'ok': True, 'error': message}
|
|
946
|
+
raise AssertionError('verify should have failed when recon baseline was missing')
|
|
947
|
+
finally:
|
|
948
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def run_ship_missing_supervisor_gate():
|
|
952
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-gate-'))
|
|
953
|
+
state_path = tempdir / 'state.json'
|
|
954
|
+
try:
|
|
955
|
+
state = base_state(tempdir, reference='before')
|
|
956
|
+
state.update({
|
|
957
|
+
'recon_status': 'ready_for_proof_plan',
|
|
958
|
+
'author_status': 'ready',
|
|
959
|
+
'proof_plan_status': 'ready',
|
|
960
|
+
'implementation_status': 'changes_detected',
|
|
961
|
+
'verify_status': 'evidence_captured',
|
|
962
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
963
|
+
'after_cdn': 'https://cdn.example.com/after.png',
|
|
964
|
+
'proof_assessment': {},
|
|
965
|
+
'proof_assessment_source': None,
|
|
966
|
+
})
|
|
967
|
+
write_state(state_path, state)
|
|
968
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
969
|
+
|
|
970
|
+
try:
|
|
971
|
+
load_module('ship_missing_supervisor_gate', SHIP_PATH)
|
|
972
|
+
except SystemExit as exc:
|
|
973
|
+
message = str(exc)
|
|
974
|
+
assert 'proof_assessment.decision=ready_to_ship' in message, message
|
|
975
|
+
return {'ok': True, 'error': message}
|
|
976
|
+
raise AssertionError('ship should have failed without supervising-agent ready_to_ship assessment')
|
|
977
|
+
finally:
|
|
978
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def run_ship_accepts_structured_after_evidence():
|
|
982
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-structured-'))
|
|
983
|
+
state_path = tempdir / 'state.json'
|
|
984
|
+
try:
|
|
985
|
+
state = base_state(tempdir, reference='before')
|
|
986
|
+
state.update({
|
|
987
|
+
'repo_dir': str(tempdir),
|
|
988
|
+
'after_worktree': str(tempdir / 'missing-after-worktree'),
|
|
989
|
+
'recon_status': 'ready_for_proof_plan',
|
|
990
|
+
'author_status': 'ready',
|
|
991
|
+
'proof_plan_status': 'ready',
|
|
992
|
+
'implementation_status': 'changes_detected',
|
|
993
|
+
'verify_status': 'evidence_captured',
|
|
994
|
+
'before_cdn': 'https://cdn.example.com/before.png',
|
|
995
|
+
'after_cdn': '',
|
|
996
|
+
'proof_assessment': {
|
|
997
|
+
'decision': 'ready_to_ship',
|
|
998
|
+
'summary': 'Structured audio metrics prove the attack increased.',
|
|
999
|
+
'reasons': ['proofEvidence contains the requested transient metric change'],
|
|
1000
|
+
'source': 'supervising_agent',
|
|
1001
|
+
},
|
|
1002
|
+
'proof_assessment_source': 'supervising_agent',
|
|
1003
|
+
'evidence_bundle': {
|
|
1004
|
+
'verification_mode': 'audio',
|
|
1005
|
+
'expected_path': '/sequencer',
|
|
1006
|
+
'after': {
|
|
1007
|
+
'observation': {'valid': True, 'telemetry_ready': True, 'reason': 'ok'},
|
|
1008
|
+
'supporting_artifacts': {
|
|
1009
|
+
'has_structured_payload': True,
|
|
1010
|
+
'proof_evidence_present': True,
|
|
1011
|
+
'proof_evidence_sample': '{"attack_ms_after":12,"passed":true}',
|
|
1012
|
+
},
|
|
1013
|
+
},
|
|
1014
|
+
},
|
|
1015
|
+
'finalized': True,
|
|
1016
|
+
'pr_url': 'https://github.com/example/repo/pull/1',
|
|
1017
|
+
'pr_number': '1',
|
|
1018
|
+
'marked_ready': True,
|
|
1019
|
+
'proof_assessment_comment_posted': True,
|
|
1020
|
+
'discord_notification': {'ok': True, 'pr_url': 'https://github.com/example/repo/pull/1'},
|
|
1021
|
+
})
|
|
1022
|
+
write_state(state_path, state)
|
|
1023
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
1024
|
+
sys.modules.pop('util', None)
|
|
1025
|
+
|
|
1026
|
+
try:
|
|
1027
|
+
load_module('ship_structured_after_evidence', SHIP_PATH)
|
|
1028
|
+
except SystemExit as exc:
|
|
1029
|
+
assert exc.code == 0 or str(exc) == '0', exc
|
|
1030
|
+
after_ship = json.loads(state_path.read_text())
|
|
1031
|
+
assert after_ship['stage'] == 'ship'
|
|
1032
|
+
assert after_ship['merge_recommendation'].startswith('ready_to_ship')
|
|
1033
|
+
assert after_ship['ship_report']['pr_url'] == 'https://github.com/example/repo/pull/1'
|
|
1034
|
+
assert after_ship['ship_report']['before_artifact_url'] == 'https://cdn.example.com/before.png'
|
|
1035
|
+
return {'ok': True, 'stage': after_ship['stage'], 'after_cdn': after_ship['after_cdn']}
|
|
1036
|
+
raise AssertionError('ship should have exited after finalized structured-evidence sync')
|
|
1037
|
+
finally:
|
|
1038
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
1039
|
+
|
|
1040
|
+
|
|
1041
|
+
def run_ship_discord_thread_target():
|
|
1042
|
+
sys.modules.pop('util', None)
|
|
1043
|
+
source = SHIP_PATH.read_text()
|
|
1044
|
+
helpers_source = source.split('\ns = load_state()', 1)[0]
|
|
1045
|
+
namespace = {'__file__': str(SHIP_PATH)}
|
|
1046
|
+
exec(compile(helpers_source, str(SHIP_PATH), 'exec'), namespace)
|
|
1047
|
+
|
|
1048
|
+
thread_target = namespace['discord_message_target']({
|
|
1049
|
+
'discord_channel': 'parent-channel-123',
|
|
1050
|
+
'discord_thread_id': 'thread-456',
|
|
1051
|
+
'discord_message_id': 'message-789',
|
|
1052
|
+
'discord_source_url': 'https://discord.com/channels/guild/thread-456/message-789',
|
|
1053
|
+
})
|
|
1054
|
+
assert thread_target['ok'] is True
|
|
1055
|
+
assert thread_target['target_channel_id'] == 'thread-456'
|
|
1056
|
+
assert thread_target['parent_channel_id'] == 'parent-channel-123'
|
|
1057
|
+
assert 'message_reference' not in thread_target
|
|
1058
|
+
|
|
1059
|
+
reply_target = namespace['discord_message_target']({
|
|
1060
|
+
'discord_channel': 'parent-channel-123',
|
|
1061
|
+
'discord_message_id': 'message-789',
|
|
1062
|
+
})
|
|
1063
|
+
assert reply_target['ok'] is True
|
|
1064
|
+
assert reply_target['target_channel_id'] == 'parent-channel-123'
|
|
1065
|
+
assert reply_target['message_reference']['message_id'] == 'message-789'
|
|
1066
|
+
assert reply_target['message_reference']['channel_id'] == 'parent-channel-123'
|
|
1067
|
+
|
|
1068
|
+
missing_target = namespace['discord_message_target']({'discord_message_id': 'message-789'})
|
|
1069
|
+
assert missing_target['ok'] is False
|
|
1070
|
+
assert 'discord_channel or discord_thread_id' in missing_target['reason']
|
|
1071
|
+
|
|
1072
|
+
return {'ok': True, 'thread_target': thread_target['target_channel_id'], 'reply_target': reply_target['target_channel_id']}
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
def run_ship_filters_tool_noise_when_staging():
|
|
1076
|
+
sys.modules.pop('util', None)
|
|
1077
|
+
source = SHIP_PATH.read_text()
|
|
1078
|
+
helpers_source = source.split('\ns = load_state()', 1)[0]
|
|
1079
|
+
namespace = {'__file__': str(SHIP_PATH)}
|
|
1080
|
+
exec(compile(helpers_source, str(SHIP_PATH), 'exec'), namespace)
|
|
1081
|
+
|
|
1082
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-stage-'))
|
|
1083
|
+
try:
|
|
1084
|
+
sp.run(['git', 'init', '-b', 'main'], cwd=tempdir, check=True, capture_output=True, text=True)
|
|
1085
|
+
sp.run(['git', 'config', 'user.email', 'test@example.com'], cwd=tempdir, check=True)
|
|
1086
|
+
sp.run(['git', 'config', 'user.name', 'Test User'], cwd=tempdir, check=True)
|
|
1087
|
+
(tempdir / 'tracked.txt').write_text('before\n')
|
|
1088
|
+
sp.run(['git', 'add', 'tracked.txt'], cwd=tempdir, check=True)
|
|
1089
|
+
sp.run(['git', 'commit', '-m', 'initial'], cwd=tempdir, check=True, capture_output=True, text=True)
|
|
1090
|
+
|
|
1091
|
+
(tempdir / 'tracked.txt').write_text('after\n')
|
|
1092
|
+
(tempdir / 'new-code.txt').write_text('new\n')
|
|
1093
|
+
(tempdir / '.codex').write_text('agent scratch\n')
|
|
1094
|
+
(tempdir / '.oc-smoke').mkdir()
|
|
1095
|
+
(tempdir / '.oc-smoke' / 'err').write_text('smoke noise\n')
|
|
1096
|
+
|
|
1097
|
+
status = sp.run(['git', 'status', '--porcelain'], cwd=tempdir, check=True, capture_output=True, text=True).stdout
|
|
1098
|
+
lines = namespace['committable_status_lines'](status)
|
|
1099
|
+
assert any('tracked.txt' in line for line in lines), lines
|
|
1100
|
+
assert any('new-code.txt' in line for line in lines), lines
|
|
1101
|
+
assert not any('.codex' in line for line in lines), lines
|
|
1102
|
+
assert not any('.oc-smoke' in line for line in lines), lines
|
|
1103
|
+
|
|
1104
|
+
staged = namespace['stage_committable_changes'](str(tempdir))
|
|
1105
|
+
assert 'tracked.txt' in staged, staged
|
|
1106
|
+
assert 'new-code.txt' in staged, staged
|
|
1107
|
+
assert '.codex' not in staged, staged
|
|
1108
|
+
assert '.oc-smoke/err' not in staged, staged
|
|
1109
|
+
|
|
1110
|
+
cached = sp.run(['git', 'diff', '--cached', '--name-only'], cwd=tempdir, check=True, capture_output=True, text=True).stdout.splitlines()
|
|
1111
|
+
assert 'tracked.txt' in cached, cached
|
|
1112
|
+
assert 'new-code.txt' in cached, cached
|
|
1113
|
+
assert '.codex' not in cached, cached
|
|
1114
|
+
assert '.oc-smoke/err' not in cached, cached
|
|
1115
|
+
|
|
1116
|
+
return {'ok': True, 'staged': staged}
|
|
1117
|
+
finally:
|
|
1118
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def run_ship_resolves_real_pr_branch():
|
|
1122
|
+
sys.modules.pop('util', None)
|
|
1123
|
+
source = SHIP_PATH.read_text()
|
|
1124
|
+
helpers_source = source.split('\ns = load_state()', 1)[0]
|
|
1125
|
+
|
|
1126
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-ship-branch-'))
|
|
1127
|
+
try:
|
|
1128
|
+
state_path = tempdir / 'state.json'
|
|
1129
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
1130
|
+
namespace = {'__file__': str(SHIP_PATH)}
|
|
1131
|
+
exec(compile(helpers_source, str(SHIP_PATH), 'exec'), namespace)
|
|
1132
|
+
state = {
|
|
1133
|
+
'branch': 'riddle-proof/rp_test-after',
|
|
1134
|
+
'target_branch': 'riddle-proof/rp_test-after',
|
|
1135
|
+
'after_worktree_branch': 'riddle-proof/rp_test-after',
|
|
1136
|
+
'pr_number': '257',
|
|
1137
|
+
'pr_url': 'https://github.com/example/repo/pull/257',
|
|
1138
|
+
}
|
|
1139
|
+
write_state(state_path, state)
|
|
1140
|
+
namespace['pr_head_branch'] = lambda repo_dir, pr_ref: 'ttt-status-polish-proof'
|
|
1141
|
+
branch = namespace['resolve_ship_branch'](state, str(tempdir))
|
|
1142
|
+
assert branch == 'ttt-status-polish-proof', branch
|
|
1143
|
+
after_state = json.loads(state_path.read_text())
|
|
1144
|
+
assert after_state['branch'] == 'ttt-status-polish-proof'
|
|
1145
|
+
assert after_state['target_branch'] == 'ttt-status-polish-proof'
|
|
1146
|
+
|
|
1147
|
+
temp_state = {
|
|
1148
|
+
'branch': 'riddle-proof/rp_test-after',
|
|
1149
|
+
'target_branch': 'riddle-proof/rp_test-after',
|
|
1150
|
+
'after_worktree_branch': 'riddle-proof/rp_test-after',
|
|
1151
|
+
'change_request': 'Ship a clean proof branch',
|
|
1152
|
+
'run_id': 'rp_test_audio_abcdef',
|
|
1153
|
+
}
|
|
1154
|
+
namespace['pr_head_branch'] = lambda repo_dir, pr_ref: ''
|
|
1155
|
+
recovered_branch = namespace['resolve_ship_branch'](temp_state, str(tempdir))
|
|
1156
|
+
assert recovered_branch.startswith('agent/openclaw/ship-a-clean-proof-branch-'), recovered_branch
|
|
1157
|
+
assert temp_state['ship_branch_recovered_from'] == 'riddle-proof/rp_test-after'
|
|
1158
|
+
|
|
1159
|
+
ambiguous_pr_state = {
|
|
1160
|
+
'branch': 'riddle-proof/rp_test-after',
|
|
1161
|
+
'target_branch': 'riddle-proof/rp_test-after',
|
|
1162
|
+
'after_worktree_branch': 'riddle-proof/rp_test-after',
|
|
1163
|
+
'pr_number': '999',
|
|
1164
|
+
'pr_url': 'https://github.com/example/repo/pull/999',
|
|
1165
|
+
}
|
|
1166
|
+
try:
|
|
1167
|
+
namespace['resolve_ship_branch'](ambiguous_pr_state, str(tempdir))
|
|
1168
|
+
except SystemExit as exc:
|
|
1169
|
+
assert 'temporary proof branch' in str(exc), exc
|
|
1170
|
+
else:
|
|
1171
|
+
raise AssertionError('temporary proof branch should still be rejected for unresolved existing PRs')
|
|
1172
|
+
return {'ok': True, 'branch': branch, 'recovered_branch': recovered_branch}
|
|
1173
|
+
finally:
|
|
1174
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
if __name__ == '__main__':
|
|
1178
|
+
payload = {
|
|
1179
|
+
'capture_artifact_enrichment': run_capture_artifact_enrichment(),
|
|
1180
|
+
'capture_diagnostics_redaction': run_capture_diagnostics_redact_sensitive_values(),
|
|
1181
|
+
'apply_auth_context': run_apply_auth_context_passes_supported_auth_payloads(),
|
|
1182
|
+
'verify_quality_ignores_proof_telemetry_console_text': run_verify_quality_ignores_proof_telemetry_console_text(),
|
|
1183
|
+
'recon_then_author_request': run_recon_then_author_request(),
|
|
1184
|
+
'recon_route_literal_preference': run_recon_prefers_route_literals_over_import_paths(),
|
|
1185
|
+
'author_applies_supervisor_packet': run_author_applies_supervisor_packet(),
|
|
1186
|
+
'verify_requests_supervisor_assessment': run_verify_requests_supervisor_assessment(),
|
|
1187
|
+
'verify_structured_evidence_without_screenshot': run_verify_structured_evidence_without_screenshot(),
|
|
1188
|
+
'verify_audio_requires_proof_evidence': run_verify_audio_requires_proof_evidence(),
|
|
1189
|
+
'verify_audio_rejects_failed_nested_proof_evidence': run_verify_audio_rejects_failed_nested_proof_evidence(),
|
|
1190
|
+
'verify_capture_retry': run_verify_capture_retry(),
|
|
1191
|
+
'missing_baseline_guard': run_verify_missing_baseline(),
|
|
1192
|
+
'ship_supervisor_gate': run_ship_missing_supervisor_gate(),
|
|
1193
|
+
'ship_structured_after_evidence': run_ship_accepts_structured_after_evidence(),
|
|
1194
|
+
'ship_discord_thread_target': run_ship_discord_thread_target(),
|
|
1195
|
+
'ship_filters_tool_noise_when_staging': run_ship_filters_tool_noise_when_staging(),
|
|
1196
|
+
'ship_resolves_real_pr_branch': run_ship_resolves_real_pr_branch(),
|
|
1197
|
+
}
|
|
1198
|
+
print(json.dumps(payload, indent=2))
|