@riddledc/riddle-proof 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/chunk-3MHFLQKG.js +853 -0
- package/dist/{chunk-LVP22WE4.js → chunk-5GZZZ6JA.js} +5 -1
- package/dist/engine-harness.cjs +2505 -22
- package/dist/engine-harness.js +1 -1
- package/dist/index.cjs +2512 -29
- package/dist/index.js +2 -2
- package/dist/openclaw.cjs +1 -1
- package/dist/openclaw.js +1 -1
- package/dist/proof-run-core.cjs +909 -0
- package/dist/proof-run-core.d.cts +280 -0
- package/dist/proof-run-core.d.ts +280 -0
- package/dist/proof-run-core.js +48 -0
- package/dist/proof-run-engine.cjs +2499 -0
- package/dist/proof-run-engine.d.cts +677 -0
- package/dist/proof-run-engine.d.ts +677 -0
- package/dist/proof-run-engine.js +1649 -0
- package/lib/workspace-core.mjs +391 -0
- package/package.json +15 -3
- package/runtime/lib/author.py +343 -0
- package/runtime/lib/implement.py +63 -0
- package/runtime/lib/preflight.py +246 -0
- package/runtime/lib/recon.py +1048 -0
- package/runtime/lib/riddle_core_call.mjs +151 -0
- package/runtime/lib/setup.py +387 -0
- package/runtime/lib/ship.py +834 -0
- package/runtime/lib/util.py +673 -0
- package/runtime/lib/verify.py +1223 -0
- package/runtime/pipelines/riddle-proof-author.lobster +28 -0
- package/runtime/pipelines/riddle-proof-implement.lobster +26 -0
- package/runtime/pipelines/riddle-proof-recon.lobster +79 -0
- package/runtime/pipelines/riddle-proof-setup.lobster +141 -0
- package/runtime/pipelines/riddle-proof-ship.lobster +36 -0
- package/runtime/pipelines/riddle-proof-verify.lobster +74 -0
- package/runtime/tests/recon_verify_smoke.py +1198 -0
|
@@ -0,0 +1,1223 @@
|
|
|
1
|
+
"""Verify: capture after evidence against the baseline already established in recon.
|
|
2
|
+
|
|
3
|
+
Verify no longer discovers baseline context.
|
|
4
|
+
It reuses recon-owned before / prod evidence and focuses on the after-proof.
|
|
5
|
+
It now treats capture quality as a first-class sub-loop: bad captures stay in verify,
|
|
6
|
+
while good captures produce a structured evidence packet that the supervising agent
|
|
7
|
+
must assess before the wrapper routes back into author/implement/recon work or ship.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json, os, sys
|
|
11
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
12
|
+
from util import (
|
|
13
|
+
append_capture_diagnostic,
|
|
14
|
+
apply_auth_context,
|
|
15
|
+
capture_static_preview,
|
|
16
|
+
enrich_capture_payload,
|
|
17
|
+
has_auth_context,
|
|
18
|
+
invoke,
|
|
19
|
+
invoke_retry,
|
|
20
|
+
load_state,
|
|
21
|
+
prepare_server_preview,
|
|
22
|
+
save_state,
|
|
23
|
+
should_use_static_preview,
|
|
24
|
+
summarize_capture_artifacts,
|
|
25
|
+
)
|
|
26
|
+
import subprocess as sp
|
|
27
|
+
|
|
28
|
+
MIN_BODY_TEXT_LENGTH = 50
|
|
29
|
+
MIN_INTERACTIVE_ELEMENTS = 1
|
|
30
|
+
HYDRATION_WAIT_MS = 1500
|
|
31
|
+
PAGE_STATE_PREFIX = 'RIDDLE_PROOF_STATE:'
|
|
32
|
+
PROOF_EVIDENCE_PREFIX = 'RIDDLE_PROOF_EVIDENCE:'
|
|
33
|
+
PROOF_EVIDENCE_LOG_PREFIX = 'RIDDLE_PROOF_EVIDENCE '
|
|
34
|
+
PROOF_EVIDENCE_PREFIXES = (PROOF_EVIDENCE_PREFIX, PROOF_EVIDENCE_LOG_PREFIX)
|
|
35
|
+
IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg', '.webp', '.gif')
|
|
36
|
+
STRUCTURED_FIRST_MODES = {
|
|
37
|
+
'audio', 'data', 'json', 'log', 'logs', 'metric', 'metrics',
|
|
38
|
+
'telemetry', 'text', 'api',
|
|
39
|
+
}
|
|
40
|
+
VISUAL_FIRST_MODES = {
|
|
41
|
+
'visual', 'render', 'interaction', 'ui', 'layout', 'screenshot',
|
|
42
|
+
'canvas', 'animation',
|
|
43
|
+
}
|
|
44
|
+
PROOF_EVIDENCE_REQUIRED_MODES = {'audio'}
|
|
45
|
+
MIN_VISUAL_DELTA_PERCENT = 0.5
|
|
46
|
+
MIN_VISUAL_CHANGED_PIXELS = 5000
|
|
47
|
+
VISUAL_DELTA_PERCENT_KEYS = {
|
|
48
|
+
'change_pct', 'change_percent', 'changed_percent', 'percent_changed',
|
|
49
|
+
'diff_percent', 'visual_delta_percent', 'pixel_change_percent',
|
|
50
|
+
}
|
|
51
|
+
VISUAL_DELTA_RATIO_KEYS = {
|
|
52
|
+
'change_ratio', 'changed_ratio', 'diff_ratio', 'visual_delta_ratio',
|
|
53
|
+
}
|
|
54
|
+
VISUAL_CHANGED_PIXEL_KEYS = {
|
|
55
|
+
'changed_pixels', 'changed_pixel_count', 'changedpixels',
|
|
56
|
+
'diff_pixels', 'pixel_delta', 'visual_delta_pixels',
|
|
57
|
+
}
|
|
58
|
+
VISUAL_TOTAL_PIXEL_KEYS = {
|
|
59
|
+
'total_pixels', 'total_pixel_count', 'pixel_count', 'totalpixels',
|
|
60
|
+
}
|
|
61
|
+
VISUAL_WIDTH_KEYS = {'width', 'image_width', 'screenshot_width'}
|
|
62
|
+
VISUAL_HEIGHT_KEYS = {'height', 'image_height', 'screenshot_height'}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def capture_script_saves_screenshot(script):
|
|
66
|
+
return 'saveScreenshot' in (script or '')
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def normalized_verification_mode(value):
|
|
70
|
+
return ((value or 'proof').strip().lower() or 'proof')
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def proof_evidence_required_for_mode(verification_mode):
|
|
74
|
+
return normalized_verification_mode(verification_mode) in PROOF_EVIDENCE_REQUIRED_MODES
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def screenshot_required_for_mode(verification_mode):
|
|
78
|
+
return normalized_verification_mode(verification_mode) in VISUAL_FIRST_MODES
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def auto_screenshot_for_mode(verification_mode):
|
|
82
|
+
return normalized_verification_mode(verification_mode) not in STRUCTURED_FIRST_MODES
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def payload_has_capture_artifacts(payload):
|
|
86
|
+
if not isinstance(payload, dict):
|
|
87
|
+
return False
|
|
88
|
+
if payload.get('outputs') or payload.get('screenshots') or payload.get('console'):
|
|
89
|
+
return True
|
|
90
|
+
result = payload.get('result')
|
|
91
|
+
if isinstance(result, dict) and result:
|
|
92
|
+
return True
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def capture_payload_error(payload):
|
|
97
|
+
if not isinstance(payload, dict):
|
|
98
|
+
return ''
|
|
99
|
+
if payload.get('ok') is False and not payload_has_capture_artifacts(payload):
|
|
100
|
+
for key in ('error', 'stderr', 'stdout'):
|
|
101
|
+
value = payload.get(key)
|
|
102
|
+
if value:
|
|
103
|
+
return str(value).strip()
|
|
104
|
+
return 'capture tool returned ok=false without artifacts'
|
|
105
|
+
return ''
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def abort_capture_failure(state, results, expected_path, message, raw_payload):
|
|
109
|
+
summary = 'After capture failed before usable proof artifacts were produced: ' + str(message).strip()
|
|
110
|
+
observation = {
|
|
111
|
+
'valid': False,
|
|
112
|
+
'reason': summary,
|
|
113
|
+
'telemetry_ready': False,
|
|
114
|
+
'details': {
|
|
115
|
+
'capture_tool_error': str(message).strip(),
|
|
116
|
+
'artifact_summary': summarize_capture_artifacts(raw_payload),
|
|
117
|
+
'observed_path': expected_path,
|
|
118
|
+
'observed_path_raw': expected_path,
|
|
119
|
+
},
|
|
120
|
+
}
|
|
121
|
+
results['after'] = {
|
|
122
|
+
'screenshots': [],
|
|
123
|
+
'raw': raw_payload,
|
|
124
|
+
'observation': observation,
|
|
125
|
+
'supporting_artifacts': collect_supporting_artifacts(raw_payload),
|
|
126
|
+
}
|
|
127
|
+
state['stage'] = 'verify'
|
|
128
|
+
state['after_cdn'] = ''
|
|
129
|
+
state['verify_results'] = results
|
|
130
|
+
state['verify_status'] = 'capture_error'
|
|
131
|
+
state['merge_recommendation'] = 'do-not-merge'
|
|
132
|
+
state['proof_assessment'] = {}
|
|
133
|
+
state['proof_assessment_source'] = None
|
|
134
|
+
state['proof_assessment_request'] = {}
|
|
135
|
+
state['verify_decision_request'] = {
|
|
136
|
+
'status': state['verify_status'],
|
|
137
|
+
'summary': summary,
|
|
138
|
+
'expected_path': expected_path,
|
|
139
|
+
'latest_observation': observation,
|
|
140
|
+
'capture_quality': {
|
|
141
|
+
'decision': 'capture_error',
|
|
142
|
+
'summary': summary,
|
|
143
|
+
'recommended_stage': None,
|
|
144
|
+
'continue_with_stage': None,
|
|
145
|
+
'reasons': [summary],
|
|
146
|
+
},
|
|
147
|
+
'next_stage_options': ['verify', 'recon'],
|
|
148
|
+
'recommended_stage': None,
|
|
149
|
+
'continue_with_stage': None,
|
|
150
|
+
'fields_agent_may_update': ['server_path', 'wait_for_selector'],
|
|
151
|
+
'instructions': [
|
|
152
|
+
'The capture tool failed before producing screenshots or structured evidence.',
|
|
153
|
+
'Fix the runtime/configuration problem before retrying verify.',
|
|
154
|
+
'Do not return to proof authoring unless the capture tool can run and produces low-quality evidence.',
|
|
155
|
+
],
|
|
156
|
+
}
|
|
157
|
+
state['verify_summary'] = summary
|
|
158
|
+
state['proof_summary'] = summary
|
|
159
|
+
state['evidence_notes'] = [
|
|
160
|
+
'Capture failed before usable proof evidence was produced.',
|
|
161
|
+
'This is a runtime or configuration failure, not a proof-authoring failure.',
|
|
162
|
+
]
|
|
163
|
+
save_state(state)
|
|
164
|
+
raise SystemExit(summary)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def build_probe_capture_script(base_script='', verification_mode='proof'):
|
|
168
|
+
pieces = []
|
|
169
|
+
script = (base_script or '').strip()
|
|
170
|
+
if script:
|
|
171
|
+
pieces.append(script.rstrip(';') + ';')
|
|
172
|
+
pieces.extend([
|
|
173
|
+
f'await page.waitForTimeout({HYDRATION_WAIT_MS});',
|
|
174
|
+
'const pageState = await page.evaluate(() => {',
|
|
175
|
+
' const textOf = (el) => ((el && el.innerText) || (el && el.textContent) || "").replace(/\\s+/g, " ").trim();',
|
|
176
|
+
' const isVisible = (el) => {',
|
|
177
|
+
' if (!el || !el.getBoundingClientRect) return false;',
|
|
178
|
+
' const rect = el.getBoundingClientRect();',
|
|
179
|
+
' const style = window.getComputedStyle(el);',
|
|
180
|
+
' return rect.width > 1 && rect.height > 1 && style.visibility !== "hidden" && style.display !== "none";',
|
|
181
|
+
' };',
|
|
182
|
+
' const textList = (selector, limit) => Array.from(document.querySelectorAll(selector)).filter(isVisible).map((el) => textOf(el).slice(0, 160)).filter(Boolean).slice(0, limit);',
|
|
183
|
+
' const links = Array.from(document.querySelectorAll("a[href]")).filter(isVisible).map((el) => ({ text: textOf(el).slice(0, 120), href: el.getAttribute("href") || "" })).filter((item) => item.text || item.href).slice(0, 12);',
|
|
184
|
+
' const largeVisibleElements = Array.from(document.body ? document.body.querySelectorAll("main, section, article, [role=main], canvas, button, a, h1, h2, h3, [data-testid], [class], [id]") : []).filter(isVisible).map((el) => {',
|
|
185
|
+
' const rect = el.getBoundingClientRect();',
|
|
186
|
+
' const className = typeof el.className === "string" ? el.className : "";',
|
|
187
|
+
' return { tag: el.tagName.toLowerCase(), id: el.id || "", className: className.slice(0, 120), text: textOf(el).slice(0, 120), area: Math.round(rect.width * rect.height) };',
|
|
188
|
+
' }).sort((a, b) => b.area - a.area).slice(0, 10);',
|
|
189
|
+
' const visibleText = document.body ? textOf(document.body) : "";',
|
|
190
|
+
' return {',
|
|
191
|
+
' bodyTextLength: visibleText.length,',
|
|
192
|
+
' visibleTextSample: visibleText.slice(0, 800),',
|
|
193
|
+
' interactiveElements: document.querySelectorAll("button, input, [role=button], canvas, a[href]").length,',
|
|
194
|
+
' visibleInteractiveElements: Array.from(document.querySelectorAll("button, input, [role=button], canvas, a[href]")).filter(isVisible).length,',
|
|
195
|
+
' headings: textList("h1, h2, [role=heading]", 8),',
|
|
196
|
+
' buttons: textList("button, [role=button]", 12),',
|
|
197
|
+
' links,',
|
|
198
|
+
' canvasCount: document.querySelectorAll("canvas").length,',
|
|
199
|
+
' largeVisibleElements,',
|
|
200
|
+
' pathname: window.location.pathname,',
|
|
201
|
+
' title: document.title,',
|
|
202
|
+
' };',
|
|
203
|
+
'});',
|
|
204
|
+
'console.log(' + json.dumps(PAGE_STATE_PREFIX) + ' + JSON.stringify(pageState));',
|
|
205
|
+
'let __riddleProofEvidenceValue = null;',
|
|
206
|
+
'try {',
|
|
207
|
+
' __riddleProofEvidenceValue = await page.evaluate(() => {',
|
|
208
|
+
' const root = (typeof window !== "undefined" && window) || (typeof globalThis !== "undefined" && globalThis) || (typeof self !== "undefined" && self) || {};',
|
|
209
|
+
' return root.__riddleProofEvidence ?? root.riddleProofEvidence ?? null;',
|
|
210
|
+
' });',
|
|
211
|
+
'} catch {}',
|
|
212
|
+
'if (__riddleProofEvidenceValue === null || __riddleProofEvidenceValue === undefined) {',
|
|
213
|
+
' const __riddleProofEvidenceRoot = (typeof globalThis !== "undefined" && globalThis) || (typeof window !== "undefined" && window) || (typeof self !== "undefined" && self) || {};',
|
|
214
|
+
' __riddleProofEvidenceValue = __riddleProofEvidenceRoot.__riddleProofEvidence ?? __riddleProofEvidenceRoot.riddleProofEvidence ?? null;',
|
|
215
|
+
'}',
|
|
216
|
+
'if (__riddleProofEvidenceValue !== null && __riddleProofEvidenceValue !== undefined) {',
|
|
217
|
+
' try { console.log(' + json.dumps(PROOF_EVIDENCE_PREFIX) + ' + JSON.stringify(__riddleProofEvidenceValue)); }',
|
|
218
|
+
' catch (err) { console.log(' + json.dumps(PROOF_EVIDENCE_PREFIX) + ' + JSON.stringify({ serialization_error: String(err) })); }',
|
|
219
|
+
'}',
|
|
220
|
+
])
|
|
221
|
+
if auto_screenshot_for_mode(verification_mode) and not capture_script_saves_screenshot(script):
|
|
222
|
+
pieces.append("await saveScreenshot('after-proof');")
|
|
223
|
+
pieces.append('return { pageState, proofEvidence: __riddleProofEvidenceValue };')
|
|
224
|
+
return ' '.join(pieces)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def extract_screenshot_url(payload, preferred_label=''):
|
|
228
|
+
preferred_names = []
|
|
229
|
+
label = (preferred_label or '').strip()
|
|
230
|
+
if label:
|
|
231
|
+
preferred_names = [
|
|
232
|
+
label,
|
|
233
|
+
label + '.png',
|
|
234
|
+
label + '.jpg',
|
|
235
|
+
label + '.jpeg',
|
|
236
|
+
label + '.webp',
|
|
237
|
+
label + '.gif',
|
|
238
|
+
]
|
|
239
|
+
outputs = payload.get('outputs') or []
|
|
240
|
+
for item in outputs:
|
|
241
|
+
name = item.get('name', '')
|
|
242
|
+
if name in preferred_names and 'error' not in name:
|
|
243
|
+
return item.get('url', '')
|
|
244
|
+
for item in outputs:
|
|
245
|
+
name = item.get('name', '')
|
|
246
|
+
if name.endswith(IMAGE_EXTENSIONS) and 'error' not in name:
|
|
247
|
+
return item.get('url', '')
|
|
248
|
+
screenshots = payload.get('screenshots') or []
|
|
249
|
+
for item in screenshots:
|
|
250
|
+
name = item.get('name', '')
|
|
251
|
+
if name in preferred_names and 'error' not in name:
|
|
252
|
+
return item.get('url', '')
|
|
253
|
+
if screenshots:
|
|
254
|
+
return screenshots[0].get('url', '')
|
|
255
|
+
return ''
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def iter_console_messages(console):
|
|
259
|
+
if isinstance(console, list):
|
|
260
|
+
for entry in console:
|
|
261
|
+
if isinstance(entry, str):
|
|
262
|
+
yield entry
|
|
263
|
+
elif isinstance(entry, dict):
|
|
264
|
+
text = entry.get('text') or entry.get('message') or ''
|
|
265
|
+
if isinstance(text, str):
|
|
266
|
+
yield text
|
|
267
|
+
return
|
|
268
|
+
|
|
269
|
+
if isinstance(console, dict):
|
|
270
|
+
entries = console.get('entries') or {}
|
|
271
|
+
if isinstance(entries, dict):
|
|
272
|
+
for bucket in ('log', 'info', 'warn', 'error'):
|
|
273
|
+
values = entries.get(bucket) or []
|
|
274
|
+
if not isinstance(values, list):
|
|
275
|
+
continue
|
|
276
|
+
for entry in values:
|
|
277
|
+
if isinstance(entry, str):
|
|
278
|
+
yield entry
|
|
279
|
+
elif isinstance(entry, dict):
|
|
280
|
+
text = entry.get('text') or entry.get('message') or ''
|
|
281
|
+
if isinstance(text, str):
|
|
282
|
+
yield text
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def is_proof_telemetry_console_message(text):
|
|
286
|
+
return isinstance(text, str) and (
|
|
287
|
+
text.startswith(PAGE_STATE_PREFIX)
|
|
288
|
+
or proof_evidence_console_payload(text) is not None
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def proof_evidence_console_payload(text):
|
|
293
|
+
if not isinstance(text, str):
|
|
294
|
+
return None
|
|
295
|
+
for prefix in PROOF_EVIDENCE_PREFIXES:
|
|
296
|
+
if text.startswith(prefix):
|
|
297
|
+
return text[len(prefix):].strip()
|
|
298
|
+
return None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def extract_page_state(payload):
|
|
302
|
+
for text in iter_console_messages(payload.get('console') or []):
|
|
303
|
+
if isinstance(text, str) and text.startswith(PAGE_STATE_PREFIX):
|
|
304
|
+
try:
|
|
305
|
+
return json.loads(text[len(PAGE_STATE_PREFIX):])
|
|
306
|
+
except Exception:
|
|
307
|
+
continue
|
|
308
|
+
result = payload.get('result') or {}
|
|
309
|
+
if isinstance(result, dict):
|
|
310
|
+
page_state = result.get('pageState')
|
|
311
|
+
if isinstance(page_state, dict):
|
|
312
|
+
return page_state
|
|
313
|
+
return None
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def extract_proof_evidence(payload):
|
|
317
|
+
evidence = []
|
|
318
|
+
for text in iter_console_messages(payload.get('console') or []):
|
|
319
|
+
raw_evidence = proof_evidence_console_payload(text)
|
|
320
|
+
if raw_evidence is not None:
|
|
321
|
+
try:
|
|
322
|
+
evidence.append(json.loads(raw_evidence))
|
|
323
|
+
except Exception:
|
|
324
|
+
continue
|
|
325
|
+
|
|
326
|
+
result = payload.get('result') or {}
|
|
327
|
+
if isinstance(result, dict):
|
|
328
|
+
for key in ('proofEvidence', 'proof_evidence', 'evidence', 'metrics', 'logs', 'analysis'):
|
|
329
|
+
value = result.get(key)
|
|
330
|
+
if value not in (None, ''):
|
|
331
|
+
evidence.append(value)
|
|
332
|
+
|
|
333
|
+
if not evidence:
|
|
334
|
+
return None
|
|
335
|
+
if len(evidence) == 1:
|
|
336
|
+
return evidence[0]
|
|
337
|
+
return evidence
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def first_failed_proof_evidence(value):
|
|
341
|
+
if isinstance(value, dict):
|
|
342
|
+
if value.get('proof_evidence_present') is False:
|
|
343
|
+
return value
|
|
344
|
+
for item in value.values():
|
|
345
|
+
failed = first_failed_proof_evidence(item)
|
|
346
|
+
if failed is not None:
|
|
347
|
+
return failed
|
|
348
|
+
elif isinstance(value, list):
|
|
349
|
+
for item in value:
|
|
350
|
+
failed = first_failed_proof_evidence(item)
|
|
351
|
+
if failed is not None:
|
|
352
|
+
return failed
|
|
353
|
+
return None
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def failed_proof_evidence_summary(proof_evidence):
|
|
357
|
+
failed = first_failed_proof_evidence(proof_evidence)
|
|
358
|
+
if not isinstance(failed, dict):
|
|
359
|
+
return ''
|
|
360
|
+
failed_checks = []
|
|
361
|
+
checks = failed.get('checks')
|
|
362
|
+
if isinstance(checks, dict):
|
|
363
|
+
failed_checks = [str(key) for key, value in checks.items() if value is False]
|
|
364
|
+
summary = 'Audio proof evidence explicitly reports proof_evidence_present=false.'
|
|
365
|
+
if failed_checks:
|
|
366
|
+
summary += ' Failed checks: ' + ', '.join(failed_checks[:8]) + '.'
|
|
367
|
+
evidence_summary = str(failed.get('evidence_summary') or '').strip()
|
|
368
|
+
if evidence_summary:
|
|
369
|
+
summary += ' Evidence summary: ' + evidence_summary[:300]
|
|
370
|
+
return summary
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def compact_value(value, limit=1200):
|
|
374
|
+
try:
|
|
375
|
+
text = json.dumps(value, sort_keys=True)
|
|
376
|
+
except Exception:
|
|
377
|
+
text = str(value)
|
|
378
|
+
if len(text) <= limit:
|
|
379
|
+
return text
|
|
380
|
+
return text[:limit - 20].rstrip() + '...'
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def normalize_metric_key(value):
|
|
384
|
+
return ''.join(ch if ch.isalnum() else '_' for ch in str(value or '').strip().lower()).strip('_')
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def metric_number(value):
|
|
388
|
+
if isinstance(value, bool) or value is None:
|
|
389
|
+
return None
|
|
390
|
+
if isinstance(value, (int, float)):
|
|
391
|
+
return float(value)
|
|
392
|
+
if isinstance(value, str):
|
|
393
|
+
text = value.strip().rstrip('%').replace(',', '')
|
|
394
|
+
if not text:
|
|
395
|
+
return None
|
|
396
|
+
try:
|
|
397
|
+
return float(text)
|
|
398
|
+
except Exception:
|
|
399
|
+
return None
|
|
400
|
+
return None
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def find_metric_value(value, key_names, depth=0):
|
|
404
|
+
if depth > 7:
|
|
405
|
+
return None
|
|
406
|
+
if isinstance(value, dict):
|
|
407
|
+
for raw_key, raw_value in value.items():
|
|
408
|
+
if normalize_metric_key(raw_key) in key_names:
|
|
409
|
+
number = metric_number(raw_value)
|
|
410
|
+
if number is not None:
|
|
411
|
+
return number
|
|
412
|
+
for raw_value in value.values():
|
|
413
|
+
number = find_metric_value(raw_value, key_names, depth + 1)
|
|
414
|
+
if number is not None:
|
|
415
|
+
return number
|
|
416
|
+
elif isinstance(value, list):
|
|
417
|
+
for item in value[:60]:
|
|
418
|
+
number = find_metric_value(item, key_names, depth + 1)
|
|
419
|
+
if number is not None:
|
|
420
|
+
return number
|
|
421
|
+
return None
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def extract_visual_delta(payload):
|
|
425
|
+
payload = enrich_capture_payload(payload)
|
|
426
|
+
result = payload.get('result') if isinstance(payload, dict) else {}
|
|
427
|
+
proof_json = payload.get('_proof_json') if isinstance(payload, dict) else {}
|
|
428
|
+
proof_evidence = extract_proof_evidence(payload)
|
|
429
|
+
candidates = [
|
|
430
|
+
payload if isinstance(payload, dict) else {},
|
|
431
|
+
result if isinstance(result, dict) else {},
|
|
432
|
+
proof_json if isinstance(proof_json, dict) else {},
|
|
433
|
+
proof_evidence,
|
|
434
|
+
]
|
|
435
|
+
|
|
436
|
+
percent = None
|
|
437
|
+
ratio = None
|
|
438
|
+
changed_pixels = None
|
|
439
|
+
total_pixels = None
|
|
440
|
+
width = None
|
|
441
|
+
height = None
|
|
442
|
+
for candidate in candidates:
|
|
443
|
+
if candidate is None:
|
|
444
|
+
continue
|
|
445
|
+
if percent is None:
|
|
446
|
+
percent = find_metric_value(candidate, VISUAL_DELTA_PERCENT_KEYS)
|
|
447
|
+
if ratio is None:
|
|
448
|
+
ratio = find_metric_value(candidate, VISUAL_DELTA_RATIO_KEYS)
|
|
449
|
+
if changed_pixels is None:
|
|
450
|
+
changed_pixels = find_metric_value(candidate, VISUAL_CHANGED_PIXEL_KEYS)
|
|
451
|
+
if total_pixels is None:
|
|
452
|
+
total_pixels = find_metric_value(candidate, VISUAL_TOTAL_PIXEL_KEYS)
|
|
453
|
+
if width is None:
|
|
454
|
+
width = find_metric_value(candidate, VISUAL_WIDTH_KEYS)
|
|
455
|
+
if height is None:
|
|
456
|
+
height = find_metric_value(candidate, VISUAL_HEIGHT_KEYS)
|
|
457
|
+
|
|
458
|
+
if percent is None and ratio is not None:
|
|
459
|
+
percent = ratio * 100 if 0 <= ratio <= 1 else ratio
|
|
460
|
+
if total_pixels is None and width and height:
|
|
461
|
+
total_pixels = width * height
|
|
462
|
+
if percent is None and changed_pixels is not None and total_pixels:
|
|
463
|
+
percent = (changed_pixels / total_pixels) * 100
|
|
464
|
+
|
|
465
|
+
if percent is None and changed_pixels is None:
|
|
466
|
+
return {
|
|
467
|
+
'status': 'unmeasured',
|
|
468
|
+
'passed': None,
|
|
469
|
+
'change_percent': None,
|
|
470
|
+
'changed_pixels': None,
|
|
471
|
+
'total_pixels': int(total_pixels) if total_pixels else None,
|
|
472
|
+
'min_change_percent': MIN_VISUAL_DELTA_PERCENT,
|
|
473
|
+
'min_changed_pixels': MIN_VISUAL_CHANGED_PIXELS,
|
|
474
|
+
'reason': 'No measured before/after visual delta was found in proof evidence.',
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
percent_pass = percent is not None and percent >= MIN_VISUAL_DELTA_PERCENT
|
|
478
|
+
pixel_pass = changed_pixels is not None and changed_pixels >= MIN_VISUAL_CHANGED_PIXELS
|
|
479
|
+
passed = percent_pass or pixel_pass
|
|
480
|
+
return {
|
|
481
|
+
'status': 'measured',
|
|
482
|
+
'passed': bool(passed),
|
|
483
|
+
'change_percent': round(percent, 4) if percent is not None else None,
|
|
484
|
+
'changed_pixels': int(changed_pixels) if changed_pixels is not None else None,
|
|
485
|
+
'total_pixels': int(total_pixels) if total_pixels is not None else None,
|
|
486
|
+
'min_change_percent': MIN_VISUAL_DELTA_PERCENT,
|
|
487
|
+
'min_changed_pixels': MIN_VISUAL_CHANGED_PIXELS,
|
|
488
|
+
'reason': (
|
|
489
|
+
'Measured visual delta clears the legibility threshold.'
|
|
490
|
+
if passed else
|
|
491
|
+
'Measured visual delta is below the legibility threshold; capture success alone is not proof.'
|
|
492
|
+
),
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def visual_delta_applies(verification_mode):
|
|
497
|
+
return screenshot_required_for_mode(verification_mode)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def list_value(value):
|
|
501
|
+
return value if isinstance(value, list) else []
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def semantic_anchor_count(page_state):
|
|
505
|
+
if not isinstance(page_state, dict):
|
|
506
|
+
return 0
|
|
507
|
+
headings = [item for item in list_value(page_state.get('headings')) if str(item).strip()]
|
|
508
|
+
buttons = [item for item in list_value(page_state.get('buttons')) if str(item).strip()]
|
|
509
|
+
links = [
|
|
510
|
+
item for item in list_value(page_state.get('links'))
|
|
511
|
+
if isinstance(item, dict) and (str(item.get('text') or '').strip() or str(item.get('href') or '').strip())
|
|
512
|
+
]
|
|
513
|
+
large = [
|
|
514
|
+
item for item in list_value(page_state.get('largeVisibleElements'))
|
|
515
|
+
if isinstance(item, dict) and (str(item.get('text') or '').strip() or item.get('tag') == 'canvas')
|
|
516
|
+
]
|
|
517
|
+
return len(headings) + len(buttons) + len(links) + len(large) + int(page_state.get('canvasCount') or 0)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def has_enriched_page_state(page_state):
|
|
521
|
+
return isinstance(page_state, dict) and any(
|
|
522
|
+
key in page_state
|
|
523
|
+
for key in ('visibleTextSample', 'headings', 'buttons', 'links', 'canvasCount', 'largeVisibleElements')
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def normalize_observed_path(value):
|
|
528
|
+
path = (value or '').strip()
|
|
529
|
+
if not path:
|
|
530
|
+
return ''
|
|
531
|
+
path = path.split('?', 1)[0].split('#', 1)[0]
|
|
532
|
+
if not path.startswith('/'):
|
|
533
|
+
path = '/' + path.lstrip('/')
|
|
534
|
+
parts = path.split('/')
|
|
535
|
+
if len(parts) >= 4 and parts[1] == 's':
|
|
536
|
+
path = '/' + '/'.join(parts[3:])
|
|
537
|
+
return path.rstrip('/') or '/'
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def collect_supporting_artifacts(payload):
|
|
541
|
+
payload = enrich_capture_payload(payload)
|
|
542
|
+
outputs = payload.get('outputs') or []
|
|
543
|
+
image_outputs = []
|
|
544
|
+
data_outputs = []
|
|
545
|
+
other_outputs = []
|
|
546
|
+
for item in outputs:
|
|
547
|
+
name = item.get('name', '') or ''
|
|
548
|
+
record = {
|
|
549
|
+
'name': name,
|
|
550
|
+
'url': item.get('url', ''),
|
|
551
|
+
}
|
|
552
|
+
if name.endswith(IMAGE_EXTENSIONS):
|
|
553
|
+
image_outputs.append(record)
|
|
554
|
+
elif name.endswith(('.json', '.jsonl', '.csv', '.txt', '.md', '.html')):
|
|
555
|
+
data_outputs.append(record)
|
|
556
|
+
else:
|
|
557
|
+
other_outputs.append(record)
|
|
558
|
+
|
|
559
|
+
result = payload.get('result') or {}
|
|
560
|
+
result_keys = list(result.keys()) if isinstance(result, dict) else []
|
|
561
|
+
structured_result_keys = [k for k in result_keys if k not in ('pageState', 'page_state')]
|
|
562
|
+
console_entries = payload.get('console') or []
|
|
563
|
+
proof_evidence = extract_proof_evidence(payload)
|
|
564
|
+
|
|
565
|
+
return {
|
|
566
|
+
'image_outputs': image_outputs,
|
|
567
|
+
'data_outputs': data_outputs,
|
|
568
|
+
'other_outputs': other_outputs,
|
|
569
|
+
'result_keys': result_keys,
|
|
570
|
+
'structured_result_keys': structured_result_keys,
|
|
571
|
+
'console_entries': len(console_entries),
|
|
572
|
+
'proof_evidence_present': proof_evidence is not None,
|
|
573
|
+
'proof_evidence_sample': compact_value(proof_evidence) if proof_evidence is not None else '',
|
|
574
|
+
'has_structured_payload': bool(data_outputs or structured_result_keys or proof_evidence is not None),
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
579
|
+
payload = enrich_capture_payload(payload)
|
|
580
|
+
mode = normalized_verification_mode(verification_mode)
|
|
581
|
+
supporting = collect_supporting_artifacts(payload)
|
|
582
|
+
structured_ready = bool(supporting.get('has_structured_payload'))
|
|
583
|
+
screenshot_required = screenshot_required_for_mode(mode)
|
|
584
|
+
details = {
|
|
585
|
+
'verification_mode': mode,
|
|
586
|
+
'capture_tool_error': capture_payload_error(payload),
|
|
587
|
+
'has_screenshot': False,
|
|
588
|
+
'screenshot_required': screenshot_required,
|
|
589
|
+
'structured_evidence_present': structured_ready,
|
|
590
|
+
'proof_evidence_present': bool(supporting.get('proof_evidence_present')),
|
|
591
|
+
'proof_evidence_sample': supporting.get('proof_evidence_sample', ''),
|
|
592
|
+
'body_text_length': 0,
|
|
593
|
+
'interactive_elements': 0,
|
|
594
|
+
'visible_interactive_elements': 0,
|
|
595
|
+
'has_errors': False,
|
|
596
|
+
'observed_path': '',
|
|
597
|
+
'observed_path_raw': '',
|
|
598
|
+
'title': '',
|
|
599
|
+
'visible_text_sample': '',
|
|
600
|
+
'headings': [],
|
|
601
|
+
'buttons': [],
|
|
602
|
+
'links': [],
|
|
603
|
+
'canvas_count': 0,
|
|
604
|
+
'large_visible_elements': [],
|
|
605
|
+
'semantic_anchor_count': 0,
|
|
606
|
+
'capture_error_messages': [],
|
|
607
|
+
'artifact_summary': summarize_capture_artifacts(payload),
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
screenshot_url = extract_screenshot_url(payload)
|
|
611
|
+
details['has_screenshot'] = bool(screenshot_url)
|
|
612
|
+
|
|
613
|
+
page_state = extract_page_state(payload)
|
|
614
|
+
if isinstance(page_state, dict):
|
|
615
|
+
raw_observed_path = page_state.get('pathname', '')
|
|
616
|
+
details.update({
|
|
617
|
+
'body_text_length': page_state.get('bodyTextLength', 0),
|
|
618
|
+
'interactive_elements': page_state.get('interactiveElements', 0),
|
|
619
|
+
'visible_interactive_elements': page_state.get('visibleInteractiveElements', page_state.get('interactiveElements', 0)),
|
|
620
|
+
'observed_path': normalize_observed_path(raw_observed_path),
|
|
621
|
+
'observed_path_raw': raw_observed_path,
|
|
622
|
+
'title': page_state.get('title', ''),
|
|
623
|
+
'visible_text_sample': page_state.get('visibleTextSample', ''),
|
|
624
|
+
'headings': list_value(page_state.get('headings'))[:8],
|
|
625
|
+
'buttons': list_value(page_state.get('buttons'))[:12],
|
|
626
|
+
'links': list_value(page_state.get('links'))[:12],
|
|
627
|
+
'canvas_count': page_state.get('canvasCount', 0),
|
|
628
|
+
'large_visible_elements': list_value(page_state.get('largeVisibleElements'))[:10],
|
|
629
|
+
'semantic_anchor_count': semantic_anchor_count(page_state),
|
|
630
|
+
})
|
|
631
|
+
elif screenshot_url:
|
|
632
|
+
details.update({
|
|
633
|
+
'body_text_length': MIN_BODY_TEXT_LENGTH + 100,
|
|
634
|
+
'interactive_elements': MIN_INTERACTIVE_ELEMENTS + 1,
|
|
635
|
+
'visible_interactive_elements': MIN_INTERACTIVE_ELEMENTS + 1,
|
|
636
|
+
'observed_path': expected_path,
|
|
637
|
+
'observed_path_raw': expected_path,
|
|
638
|
+
})
|
|
639
|
+
else:
|
|
640
|
+
details.update({
|
|
641
|
+
'observed_path': expected_path,
|
|
642
|
+
'observed_path_raw': expected_path,
|
|
643
|
+
})
|
|
644
|
+
|
|
645
|
+
console = payload.get('console') or []
|
|
646
|
+
for text in iter_console_messages(console):
|
|
647
|
+
if is_proof_telemetry_console_message(text):
|
|
648
|
+
continue
|
|
649
|
+
if isinstance(text, str) and ('error' in text.lower() or 'failed' in text.lower()):
|
|
650
|
+
details['has_errors'] = True
|
|
651
|
+
if len(details['capture_error_messages']) < 3:
|
|
652
|
+
details['capture_error_messages'].append(text[:500])
|
|
653
|
+
break
|
|
654
|
+
proof_json = payload.get('_proof_json') or {}
|
|
655
|
+
if isinstance(proof_json, dict) and proof_json.get('script_error'):
|
|
656
|
+
details['has_errors'] = True
|
|
657
|
+
details['capture_error_messages'].append(str(proof_json.get('script_error'))[:500])
|
|
658
|
+
|
|
659
|
+
reasons = []
|
|
660
|
+
if details['capture_tool_error']:
|
|
661
|
+
reasons.append('capture tool failed: ' + details['capture_tool_error'])
|
|
662
|
+
if screenshot_required and not details['has_screenshot']:
|
|
663
|
+
reasons.append('no screenshot in capture for visual verification mode')
|
|
664
|
+
if not details['has_screenshot'] and not structured_ready:
|
|
665
|
+
reasons.append('no screenshot or structured proof evidence in capture')
|
|
666
|
+
|
|
667
|
+
should_enforce_visual_readiness = screenshot_required or (details['has_screenshot'] and not structured_ready)
|
|
668
|
+
if should_enforce_visual_readiness and details['body_text_length'] < MIN_BODY_TEXT_LENGTH:
|
|
669
|
+
reasons.append(f'blank/near-blank page (text length: {details["body_text_length"]})')
|
|
670
|
+
if should_enforce_visual_readiness and details['interactive_elements'] < MIN_INTERACTIVE_ELEMENTS:
|
|
671
|
+
reasons.append(f'not interactive enough ({details["interactive_elements"]} interactive elements)')
|
|
672
|
+
if should_enforce_visual_readiness and has_enriched_page_state(page_state) and details['semantic_anchor_count'] < 1:
|
|
673
|
+
reasons.append('no visible semantic UI anchors in page capture')
|
|
674
|
+
if details['has_errors']:
|
|
675
|
+
reasons.append('page has console/runtime errors')
|
|
676
|
+
|
|
677
|
+
observed_path = normalize_observed_path(details.get('observed_path'))
|
|
678
|
+
normalized_expected = (expected_path or '').rstrip('/') or '/'
|
|
679
|
+
if isinstance(page_state, dict) and expected_path and observed_path and observed_path != normalized_expected:
|
|
680
|
+
raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
|
|
681
|
+
reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
|
|
682
|
+
|
|
683
|
+
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1
|
|
684
|
+
visual_ready = (
|
|
685
|
+
details['has_screenshot']
|
|
686
|
+
and details['body_text_length'] >= MIN_BODY_TEXT_LENGTH
|
|
687
|
+
and details['interactive_elements'] >= MIN_INTERACTIVE_ELEMENTS
|
|
688
|
+
and semantic_ready
|
|
689
|
+
and not details['has_errors']
|
|
690
|
+
)
|
|
691
|
+
telemetry_ready = (visual_ready or structured_ready) and not details['has_errors']
|
|
692
|
+
|
|
693
|
+
return {
|
|
694
|
+
'valid': len(reasons) == 0 and telemetry_ready,
|
|
695
|
+
'reason': '; '.join(reasons) if reasons else 'ok',
|
|
696
|
+
'telemetry_ready': telemetry_ready,
|
|
697
|
+
'details': details,
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def build_capture_retry_decision(after_observation, required_baseline_present, proof_evidence_blocker=''):
|
|
702
|
+
reasons = []
|
|
703
|
+
if not required_baseline_present:
|
|
704
|
+
reasons.append('Recon baseline is missing, so verify should return to recon instead of guessing a new reference context.')
|
|
705
|
+
return {
|
|
706
|
+
'decision': 'needs_recon',
|
|
707
|
+
'summary': 'Verify is blocked on a missing recon baseline.',
|
|
708
|
+
'recommended_stage': 'recon',
|
|
709
|
+
'continue_with_stage': 'recon',
|
|
710
|
+
'reasons': reasons,
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
if proof_evidence_blocker:
|
|
714
|
+
reasons.append(proof_evidence_blocker)
|
|
715
|
+
decision = 'missing_proof_evidence'
|
|
716
|
+
if 'proof_evidence_present=false' in proof_evidence_blocker:
|
|
717
|
+
decision = 'failed_proof_evidence'
|
|
718
|
+
reasons.append('The capture reached usable page context, but the proof evidence explicitly failed its own required audio gate.')
|
|
719
|
+
else:
|
|
720
|
+
reasons.append('The capture reached usable page context, but the proof script did not emit the structured evidence required for this verification mode.')
|
|
721
|
+
reasons.append('Return to author so the capture script can expose passing proof evidence before verify asks for a supervising-agent judgment.')
|
|
722
|
+
return {
|
|
723
|
+
'decision': decision,
|
|
724
|
+
'summary': proof_evidence_blocker,
|
|
725
|
+
'recommended_stage': 'author',
|
|
726
|
+
'continue_with_stage': 'author',
|
|
727
|
+
'reasons': reasons,
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
reason = after_observation.get('reason') or 'after capture is not usable yet'
|
|
731
|
+
reasons.append('The after evidence is not usable yet: ' + reason)
|
|
732
|
+
recommended_stage = 'recon' if 'wrong route' in reason else 'author'
|
|
733
|
+
if recommended_stage == 'recon':
|
|
734
|
+
reasons.append('The capture appears to be on the wrong route or baseline context, so recon should refresh the reference path.')
|
|
735
|
+
else:
|
|
736
|
+
reasons.append('The capture plan itself needs revision, so author should tighten the proof script or framing inputs.')
|
|
737
|
+
return {
|
|
738
|
+
'decision': 'revise_capture',
|
|
739
|
+
'summary': 'Verify needs another internal capture iteration before the evidence can be judged.',
|
|
740
|
+
'recommended_stage': recommended_stage,
|
|
741
|
+
'continue_with_stage': recommended_stage,
|
|
742
|
+
'reasons': reasons,
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def compact_semantic_list(value, limit):
|
|
747
|
+
if not isinstance(value, list):
|
|
748
|
+
return []
|
|
749
|
+
return value[:limit]
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def semantic_observation(label, observation):
|
|
753
|
+
if not isinstance(observation, dict):
|
|
754
|
+
observation = {}
|
|
755
|
+
details = observation.get('details') or {}
|
|
756
|
+
if not isinstance(details, dict):
|
|
757
|
+
details = {}
|
|
758
|
+
valid = observation.get('valid')
|
|
759
|
+
if valid is None:
|
|
760
|
+
valid = observation.get('ok')
|
|
761
|
+
return {
|
|
762
|
+
'label': label,
|
|
763
|
+
'valid': bool(valid),
|
|
764
|
+
'reason': observation.get('reason', ''),
|
|
765
|
+
'telemetry_ready': bool(observation.get('telemetry_ready')),
|
|
766
|
+
'url': observation.get('url', ''),
|
|
767
|
+
'capture_url': observation.get('capture_url', ''),
|
|
768
|
+
'observed_path': details.get('observed_path', ''),
|
|
769
|
+
'observed_path_raw': details.get('observed_path_raw', ''),
|
|
770
|
+
'title': details.get('title', ''),
|
|
771
|
+
'visible_text_sample': details.get('visible_text_sample', ''),
|
|
772
|
+
'headings': compact_semantic_list(details.get('headings'), 8),
|
|
773
|
+
'buttons': compact_semantic_list(details.get('buttons'), 12),
|
|
774
|
+
'links': compact_semantic_list(details.get('links'), 12),
|
|
775
|
+
'canvas_count': details.get('canvas_count', 0),
|
|
776
|
+
'interactive_elements': details.get('interactive_elements', 0),
|
|
777
|
+
'visible_interactive_elements': details.get('visible_interactive_elements', 0),
|
|
778
|
+
'semantic_anchor_count': details.get('semantic_anchor_count', 0),
|
|
779
|
+
'large_visible_elements': compact_semantic_list(details.get('large_visible_elements'), 10),
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
def build_semantic_context(state, results, after_observation, expected_path):
|
|
784
|
+
baseline = (results.get('baseline') or {}) if isinstance(results, dict) else {}
|
|
785
|
+
before = (baseline.get('before') or {}) if isinstance(baseline, dict) else {}
|
|
786
|
+
prod = (baseline.get('prod') or {}) if isinstance(baseline, dict) else {}
|
|
787
|
+
before_semantic = semantic_observation('before', before.get('observation') or {})
|
|
788
|
+
prod_semantic = semantic_observation('prod', prod.get('observation') or {})
|
|
789
|
+
after_semantic = semantic_observation('after', after_observation)
|
|
790
|
+
return {
|
|
791
|
+
'expected_path': expected_path,
|
|
792
|
+
'reference': state.get('requested_reference') or state.get('reference', 'both'),
|
|
793
|
+
'requested_change': state.get('change_request', ''),
|
|
794
|
+
'success_criteria': (state.get('success_criteria') or '').strip(),
|
|
795
|
+
'route': {
|
|
796
|
+
'expected_path': expected_path,
|
|
797
|
+
'before_observed_path': before_semantic.get('observed_path') or before.get('path') or '',
|
|
798
|
+
'prod_observed_path': prod_semantic.get('observed_path') or prod.get('path') or '',
|
|
799
|
+
'after_observed_path': after_semantic.get('observed_path') or '',
|
|
800
|
+
},
|
|
801
|
+
'before': before_semantic,
|
|
802
|
+
'prod': prod_semantic,
|
|
803
|
+
'after': after_semantic,
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def build_evidence_bundle(state, results, after_payload, after_observation, required_baseline_present, expected_path):
|
|
808
|
+
supporting = collect_supporting_artifacts(after_payload)
|
|
809
|
+
proof_evidence = extract_proof_evidence(after_payload)
|
|
810
|
+
visual_delta = (
|
|
811
|
+
extract_visual_delta(after_payload)
|
|
812
|
+
if visual_delta_applies(state.get('verification_mode'))
|
|
813
|
+
else {'status': 'not_applicable', 'passed': None, 'reason': 'Verification mode does not require visual delta gating.'}
|
|
814
|
+
)
|
|
815
|
+
semantic_context = build_semantic_context(state, results, after_observation, expected_path)
|
|
816
|
+
return {
|
|
817
|
+
'verification_mode': normalized_verification_mode(state.get('verification_mode')),
|
|
818
|
+
'reference': state.get('requested_reference') or state.get('reference', 'both'),
|
|
819
|
+
'expected_path': expected_path,
|
|
820
|
+
'required_baseline_present': required_baseline_present,
|
|
821
|
+
'baseline': results.get('baseline') or {},
|
|
822
|
+
'semantic_context': semantic_context,
|
|
823
|
+
'after': {
|
|
824
|
+
'screenshot_url': state.get('after_cdn') or '',
|
|
825
|
+
'observation': after_observation,
|
|
826
|
+
'supporting_artifacts': supporting,
|
|
827
|
+
'proof_evidence': proof_evidence,
|
|
828
|
+
'proof_evidence_sample': compact_value(proof_evidence) if proof_evidence is not None else '',
|
|
829
|
+
'visual_delta': visual_delta,
|
|
830
|
+
},
|
|
831
|
+
'proof_evidence': proof_evidence,
|
|
832
|
+
'proof_evidence_sample': compact_value(proof_evidence) if proof_evidence is not None else '',
|
|
833
|
+
'success_criteria': (state.get('success_criteria') or '').strip(),
|
|
834
|
+
'assertions': state.get('parsed_assertions') or None,
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def build_supervisor_assessment_request(state, payload, after_observation, required_baseline_present, expected_path, evidence_bundle=None):
|
|
839
|
+
verification_mode = ((state.get('verification_mode') or 'proof').strip().lower() or 'proof')
|
|
840
|
+
supporting = collect_supporting_artifacts(payload)
|
|
841
|
+
has_assertions = bool(state.get('parsed_assertions'))
|
|
842
|
+
has_success_criteria = bool((state.get('success_criteria') or '').strip())
|
|
843
|
+
evidence_basis = []
|
|
844
|
+
if required_baseline_present:
|
|
845
|
+
evidence_basis.append('recon-baseline')
|
|
846
|
+
if after_observation.get('valid'):
|
|
847
|
+
evidence_basis.append('after-capture')
|
|
848
|
+
if supporting['image_outputs']:
|
|
849
|
+
evidence_basis.append('screenshots')
|
|
850
|
+
if supporting['has_structured_payload']:
|
|
851
|
+
evidence_basis.append('structured-artifacts')
|
|
852
|
+
visual_delta = ((evidence_bundle or {}).get('after') or {}).get('visual_delta') or {}
|
|
853
|
+
if visual_delta.get('status') == 'measured':
|
|
854
|
+
evidence_basis.append('visual-delta')
|
|
855
|
+
semantic_context = ((evidence_bundle or {}).get('semantic_context') or {})
|
|
856
|
+
if semantic_context:
|
|
857
|
+
evidence_basis.append('semantic-context')
|
|
858
|
+
if has_assertions:
|
|
859
|
+
evidence_basis.append('assertions')
|
|
860
|
+
if has_success_criteria:
|
|
861
|
+
evidence_basis.append('success-criteria')
|
|
862
|
+
|
|
863
|
+
return {
|
|
864
|
+
'status': 'needs_supervising_agent_assessment',
|
|
865
|
+
'verification_mode': verification_mode,
|
|
866
|
+
'expected_path': expected_path,
|
|
867
|
+
'required_baseline_present': required_baseline_present,
|
|
868
|
+
'after_observation': after_observation,
|
|
869
|
+
'supporting_artifacts': supporting,
|
|
870
|
+
'visual_delta': visual_delta,
|
|
871
|
+
'semantic_context': semantic_context,
|
|
872
|
+
'evidence_bundle': evidence_bundle or {},
|
|
873
|
+
'evidence_basis': evidence_basis,
|
|
874
|
+
'instructions': [
|
|
875
|
+
'The supervising agent owns proof assessment. Inspect the recon baseline(s), after evidence, and any structured artifacts together.',
|
|
876
|
+
'Decide whether the evidence is ready_to_ship or should continue internally through author, implement, or recon.',
|
|
877
|
+
'Do not mark ready_to_ship if the before/prod baseline is blank, shell-only, generic, or not visibly tied to the requested feature.',
|
|
878
|
+
'Use semantic_context.route plus headings/buttons/text anchors to ground route and content judgment before treating a screenshot as wrong-route.',
|
|
879
|
+
'For visual/UI modes, use screenshots plus after_observation.details.visible_text_sample, headings, buttons, links, canvas_count, and large_visible_elements to explain what the proof actually shows.',
|
|
880
|
+
'For visual/UI polish, capture success is not proof. If visual_delta.status=measured and visual_delta.passed=false, choose needs_implementation or needs_richer_proof instead of ready_to_ship.',
|
|
881
|
+
'If visual_delta.status=unmeasured for visual/UI proof, only choose ready_to_ship when the screenshots and page-state details let you name a clearly legible before/after change; otherwise request richer proof or another implementation pass.',
|
|
882
|
+
'For data/audio/log/metrics/custom modes, judge the structured evidence bundle and proof_evidence_sample directly; screenshots are optional supporting context.',
|
|
883
|
+
'The summary must name the concrete change, the target route/UI, what changed in after evidence, and why the stop condition is satisfied.',
|
|
884
|
+
'Only set escalation_target=human when you conclude the workflow has hit a real wall or is not converging.',
|
|
885
|
+
'Pass the judgment back via proof_assessment_json and resume the workflow.',
|
|
886
|
+
],
|
|
887
|
+
'response_schema': {
|
|
888
|
+
'decision': 'ready_to_ship | needs_richer_proof | revise_capture | needs_recon | needs_implementation',
|
|
889
|
+
'summary': 'string',
|
|
890
|
+
'recommended_stage': 'author | implement | recon | ship | verify',
|
|
891
|
+
'continue_with_stage': 'author | implement | recon | ship | verify',
|
|
892
|
+
'escalation_target': 'agent | human',
|
|
893
|
+
'reasons': ['string'],
|
|
894
|
+
'source': 'supervising_agent',
|
|
895
|
+
},
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
s = load_state()
|
|
900
|
+
capture_script = (s.get('capture_script') or '').strip()
|
|
901
|
+
if not capture_script:
|
|
902
|
+
raise SystemExit('capture_script not set in state. Recon should finish homework first, then verify should receive the real capture plan.')
|
|
903
|
+
|
|
904
|
+
if s.get('implementation_status') not in ('changes_detected', 'completed'):
|
|
905
|
+
raise SystemExit('Implementation not recorded. Make the code changes and run riddle-proof-implement before verify.')
|
|
906
|
+
|
|
907
|
+
mode = s.get('mode', 'server')
|
|
908
|
+
reference = s.get('requested_reference') or s.get('reference', 'both')
|
|
909
|
+
prod_url = (s.get('prod_url') or '').strip()
|
|
910
|
+
after_dir = s.get('after_worktree', '').strip()
|
|
911
|
+
if not after_dir or not os.path.exists(after_dir):
|
|
912
|
+
raise SystemExit('after_worktree not found. Run setup first.')
|
|
913
|
+
|
|
914
|
+
build_cmd = s.get('build_command', 'npm run build')
|
|
915
|
+
recon_baselines = ((s.get('recon_results') or {}).get('baselines') or {})
|
|
916
|
+
expected_path = (
|
|
917
|
+
(recon_baselines.get('before') or {}).get('path')
|
|
918
|
+
or (recon_baselines.get('prod') or {}).get('path')
|
|
919
|
+
or ((s.get('recon_hypothesis') or {}).get('target_path'))
|
|
920
|
+
or s.get('server_path')
|
|
921
|
+
or '/'
|
|
922
|
+
)
|
|
923
|
+
verification_mode = normalized_verification_mode(s.get('verification_mode'))
|
|
924
|
+
probe_capture_script = build_probe_capture_script(capture_script, verification_mode)
|
|
925
|
+
results = {
|
|
926
|
+
'baseline': {
|
|
927
|
+
'reference': reference,
|
|
928
|
+
'before': None,
|
|
929
|
+
'prod': None,
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
# Refresh auth tokens
|
|
934
|
+
if s.get('use_auth', '').lower() in ('true', '1', 'yes'):
|
|
935
|
+
print('Refreshing auth tokens...')
|
|
936
|
+
try:
|
|
937
|
+
auth = invoke('auth_cognito_tokens', {}, timeout=60)
|
|
938
|
+
if auth.get('ok') and auth.get('localStorage'):
|
|
939
|
+
merged_local_storage = dict(auth['localStorage'])
|
|
940
|
+
merged_local_storage.update(s.get('auth_explicit_localStorage') or {})
|
|
941
|
+
s['auth_localStorage'] = merged_local_storage
|
|
942
|
+
print('Auth tokens refreshed.')
|
|
943
|
+
else:
|
|
944
|
+
raise SystemExit('auth_cognito_tokens failed for use_auth=true: ' + str(auth.get('error', '')))
|
|
945
|
+
except Exception as e:
|
|
946
|
+
raise SystemExit('auth_cognito_tokens failed for use_auth=true: ' + str(e))
|
|
947
|
+
|
|
948
|
+
existing_before = (s.get('before_cdn') or '').strip()
|
|
949
|
+
existing_prod = (s.get('prod_cdn') or '').strip()
|
|
950
|
+
if reference in ('before', 'both') and not existing_before:
|
|
951
|
+
raise SystemExit('Recon baseline missing: before_cdn is empty. Run recon again and confirm the before baseline succeeds before verify.')
|
|
952
|
+
if reference in ('prod', 'both') and prod_url and not existing_prod:
|
|
953
|
+
raise SystemExit('Recon baseline missing: prod_cdn is empty. Run recon again and confirm the prod baseline succeeds before verify.')
|
|
954
|
+
if reference == 'prod' and not prod_url:
|
|
955
|
+
raise SystemExit('reference is "prod" but no prod_url provided.')
|
|
956
|
+
|
|
957
|
+
if existing_before:
|
|
958
|
+
before_baseline = recon_baselines.get('before', {})
|
|
959
|
+
results['baseline']['before'] = {
|
|
960
|
+
'url': existing_before,
|
|
961
|
+
'source': 'recon',
|
|
962
|
+
'path': before_baseline.get('path', s.get('server_path', '')),
|
|
963
|
+
'observation': before_baseline.get('observation') or {},
|
|
964
|
+
}
|
|
965
|
+
if existing_prod:
|
|
966
|
+
prod_baseline = recon_baselines.get('prod', {})
|
|
967
|
+
results['baseline']['prod'] = {
|
|
968
|
+
'url': existing_prod,
|
|
969
|
+
'source': 'recon',
|
|
970
|
+
'path': prod_baseline.get('path', s.get('server_path', '')),
|
|
971
|
+
'observation': prod_baseline.get('observation') or {},
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
print('Verify will reuse recon baseline(s).')
|
|
975
|
+
if existing_before:
|
|
976
|
+
print('Before baseline: ' + existing_before)
|
|
977
|
+
if existing_prod:
|
|
978
|
+
print('Prod baseline: ' + existing_prod)
|
|
979
|
+
|
|
980
|
+
# AFTER (always from after worktree)
|
|
981
|
+
print('Cleaning after .next cache...')
|
|
982
|
+
sp.run('rm -rf .next', shell=True, cwd=after_dir, capture_output=True)
|
|
983
|
+
|
|
984
|
+
print('Building after worktree...')
|
|
985
|
+
br = sp.run(build_cmd, shell=True, cwd=after_dir, capture_output=True, text=True, timeout=600)
|
|
986
|
+
if br.returncode != 0:
|
|
987
|
+
raise SystemExit('After build failed: ' + br.stderr[:500])
|
|
988
|
+
|
|
989
|
+
after_payload = {}
|
|
990
|
+
static_reason = should_use_static_preview(after_dir, s) if mode == 'server' else ''
|
|
991
|
+
if mode == 'server' and not static_reason:
|
|
992
|
+
build_dir, server_command, server_exclude = prepare_server_preview(after_dir, s)
|
|
993
|
+
|
|
994
|
+
server_args = {
|
|
995
|
+
'directory': build_dir,
|
|
996
|
+
'image': s['server_image'],
|
|
997
|
+
'command': server_command,
|
|
998
|
+
'port': int(s['server_port']),
|
|
999
|
+
'wait_until': 'domcontentloaded',
|
|
1000
|
+
'readiness_timeout': 180,
|
|
1001
|
+
'timeout': 300,
|
|
1002
|
+
'env': {'PORT': str(s['server_port']), 'HOSTNAME': '0.0.0.0'},
|
|
1003
|
+
'exclude': server_exclude,
|
|
1004
|
+
}
|
|
1005
|
+
if s.get('server_path'):
|
|
1006
|
+
server_args['path'] = s['server_path']
|
|
1007
|
+
server_args['readiness_path'] = '/' if has_auth_context(s) else s['server_path']
|
|
1008
|
+
if s.get('color_scheme'):
|
|
1009
|
+
server_args['color_scheme'] = s['color_scheme']
|
|
1010
|
+
if s.get('wait_for_selector'):
|
|
1011
|
+
server_args['wait_for_selector'] = s['wait_for_selector']
|
|
1012
|
+
apply_auth_context(s, server_args)
|
|
1013
|
+
server_args['script'] = probe_capture_script
|
|
1014
|
+
|
|
1015
|
+
print('Running after server preview from: ' + build_dir)
|
|
1016
|
+
shot = invoke_retry('riddle_server_preview', server_args, retries=3, timeout=420)
|
|
1017
|
+
append_capture_diagnostic(s, 'after', 'riddle_server_preview', server_args, shot)
|
|
1018
|
+
after_payload = shot
|
|
1019
|
+
capture_error = capture_payload_error(after_payload)
|
|
1020
|
+
if capture_error:
|
|
1021
|
+
abort_capture_failure(s, results, expected_path, capture_error, after_payload)
|
|
1022
|
+
url = extract_screenshot_url(shot, 'after-proof')
|
|
1023
|
+
if not url:
|
|
1024
|
+
print('WARNING: After server preview no screenshot.')
|
|
1025
|
+
results['after'] = {'screenshots': [{'url': url}] if url else [], 'raw': shot}
|
|
1026
|
+
s['after_cdn'] = url
|
|
1027
|
+
|
|
1028
|
+
else:
|
|
1029
|
+
if static_reason:
|
|
1030
|
+
print('Static preview fallback for after capture: ' + static_reason)
|
|
1031
|
+
old_id = s.get('after_preview_id', '')
|
|
1032
|
+
if old_id:
|
|
1033
|
+
invoke('riddle_preview_delete', {'id': old_id}, timeout=30)
|
|
1034
|
+
capture = capture_static_preview(s, after_dir, 'after', probe_capture_script, timeout=300, target_path=s.get('server_path', ''))
|
|
1035
|
+
s['after_preview_id'] = capture.get('preview_id', '')
|
|
1036
|
+
after_payload = ((capture.get('raw') or {}).get('capture') or (capture.get('raw') or {}) or capture)
|
|
1037
|
+
append_capture_diagnostic(
|
|
1038
|
+
s,
|
|
1039
|
+
'after',
|
|
1040
|
+
'riddle_static_preview',
|
|
1041
|
+
{'target_path': s.get('server_path', ''), 'static_fallback_reason': static_reason},
|
|
1042
|
+
after_payload,
|
|
1043
|
+
)
|
|
1044
|
+
capture_error = capture_payload_error(after_payload)
|
|
1045
|
+
if capture_error:
|
|
1046
|
+
abort_capture_failure(s, results, expected_path, capture_error, after_payload)
|
|
1047
|
+
results['after'] = {'screenshots': [{'url': capture.get('url', '')}] if capture.get('url') else [], 'raw': capture.get('raw')}
|
|
1048
|
+
s['after_cdn'] = capture.get('url', '')
|
|
1049
|
+
|
|
1050
|
+
after_observation = evaluate_capture_quality(after_payload, expected_path, verification_mode)
|
|
1051
|
+
results['after']['observation'] = after_observation
|
|
1052
|
+
results['after']['supporting_artifacts'] = collect_supporting_artifacts(after_payload)
|
|
1053
|
+
|
|
1054
|
+
# Structured proof summary
|
|
1055
|
+
s['verify_results'] = results
|
|
1056
|
+
s['stage'] = 'verify'
|
|
1057
|
+
assertions = s.get('parsed_assertions')
|
|
1058
|
+
if assertions:
|
|
1059
|
+
s['assertion_status'] = 'specified'
|
|
1060
|
+
else:
|
|
1061
|
+
s['assertion_status'] = 'not_specified'
|
|
1062
|
+
|
|
1063
|
+
summary_lines = []
|
|
1064
|
+
summary_lines.append('Original request: ' + s.get('change_request', ''))
|
|
1065
|
+
summary_lines.append('Verification mode: ' + s.get('verification_mode', 'proof'))
|
|
1066
|
+
if s.get('success_criteria'):
|
|
1067
|
+
summary_lines.append('Success criteria: ' + s['success_criteria'])
|
|
1068
|
+
if s.get('implementation_summary'):
|
|
1069
|
+
summary_lines.append('Implementation summary: ' + s['implementation_summary'])
|
|
1070
|
+
if s.get('implementation_notes'):
|
|
1071
|
+
summary_lines.append('Implementation notes: ' + s['implementation_notes'])
|
|
1072
|
+
if isinstance(s.get('changed_files'), list) and s.get('changed_files'):
|
|
1073
|
+
summary_lines.append('Changed files: ' + ', '.join(str(item) for item in s['changed_files'][:12]))
|
|
1074
|
+
if s.get('proof_plan'):
|
|
1075
|
+
summary_lines.append('Authored proof plan: ' + s['proof_plan'])
|
|
1076
|
+
if s.get('supervisor_author_summary'):
|
|
1077
|
+
summary_lines.append('Supervisor author summary: ' + str(s['supervisor_author_summary']))
|
|
1078
|
+
if s.get('supervisor_author_rationale'):
|
|
1079
|
+
rationale = s['supervisor_author_rationale']
|
|
1080
|
+
if isinstance(rationale, list):
|
|
1081
|
+
summary_lines.append('Supervisor author rationale: ' + '; '.join(str(item) for item in rationale[:4]))
|
|
1082
|
+
else:
|
|
1083
|
+
summary_lines.append('Supervisor author rationale: ' + str(rationale))
|
|
1084
|
+
if existing_before:
|
|
1085
|
+
summary_lines.append('Before baseline (recon): ' + existing_before)
|
|
1086
|
+
if existing_prod:
|
|
1087
|
+
summary_lines.append('Prod baseline (recon): ' + existing_prod)
|
|
1088
|
+
summary_lines.append('After screenshot: ' + (s.get('after_cdn') or '(none)'))
|
|
1089
|
+
summary_lines.append('Expected proof path from recon: ' + expected_path)
|
|
1090
|
+
summary_lines.append('After observation: ' + after_observation['reason'])
|
|
1091
|
+
supporting = results['after'].get('supporting_artifacts') or {}
|
|
1092
|
+
if supporting.get('has_structured_payload'):
|
|
1093
|
+
basis = []
|
|
1094
|
+
if supporting.get('structured_result_keys'):
|
|
1095
|
+
basis.append('result keys: ' + ', '.join(str(item) for item in supporting.get('structured_result_keys', [])[:8]))
|
|
1096
|
+
if supporting.get('data_outputs'):
|
|
1097
|
+
basis.append('data outputs: ' + ', '.join(str((item or {}).get('name', '')) for item in supporting.get('data_outputs', [])[:8]))
|
|
1098
|
+
if supporting.get('proof_evidence_present'):
|
|
1099
|
+
basis.append('proof evidence: ' + str(supporting.get('proof_evidence_sample', ''))[:400])
|
|
1100
|
+
summary_lines.append('Structured after evidence: ' + ('; '.join(basis) if basis else 'present'))
|
|
1101
|
+
observed_path = (after_observation.get('details') or {}).get('observed_path') or expected_path
|
|
1102
|
+
summary_lines.append('Observed after path: ' + observed_path)
|
|
1103
|
+
details = after_observation.get('details') or {}
|
|
1104
|
+
if details.get('headings'):
|
|
1105
|
+
summary_lines.append('Visible headings: ' + '; '.join(str(item) for item in details.get('headings', [])[:6]))
|
|
1106
|
+
if details.get('buttons'):
|
|
1107
|
+
summary_lines.append('Visible buttons: ' + '; '.join(str(item) for item in details.get('buttons', [])[:8]))
|
|
1108
|
+
if details.get('visible_text_sample'):
|
|
1109
|
+
summary_lines.append('Visible text sample: ' + str(details['visible_text_sample'])[:500])
|
|
1110
|
+
if assertions:
|
|
1111
|
+
if isinstance(assertions, list):
|
|
1112
|
+
summary_lines.append('Assertions supplied: ' + str(len(assertions)))
|
|
1113
|
+
elif isinstance(assertions, dict):
|
|
1114
|
+
summary_lines.append('Assertions supplied: yes')
|
|
1115
|
+
|
|
1116
|
+
required_baseline_present = True
|
|
1117
|
+
if reference in ('before', 'both'):
|
|
1118
|
+
required_baseline_present = required_baseline_present and bool(existing_before)
|
|
1119
|
+
if reference in ('prod', 'both') and prod_url:
|
|
1120
|
+
required_baseline_present = required_baseline_present and bool(existing_prod)
|
|
1121
|
+
|
|
1122
|
+
evidence_bundle = build_evidence_bundle(s, results, after_payload, after_observation, required_baseline_present, expected_path)
|
|
1123
|
+
s['evidence_bundle'] = evidence_bundle
|
|
1124
|
+
visual_delta = ((evidence_bundle.get('after') or {}).get('visual_delta') or {})
|
|
1125
|
+
if visual_delta.get('status') != 'not_applicable':
|
|
1126
|
+
summary_lines.append('Visual delta gate: ' + compact_value(visual_delta, limit=700))
|
|
1127
|
+
|
|
1128
|
+
proof_evidence_blocker = ''
|
|
1129
|
+
if proof_evidence_required_for_mode(s.get('verification_mode')):
|
|
1130
|
+
proof_evidence = evidence_bundle.get('proof_evidence')
|
|
1131
|
+
if proof_evidence is None:
|
|
1132
|
+
proof_evidence_blocker = (
|
|
1133
|
+
'Audio verification requires proof_evidence_present=true, but the after capture did not emit structured proof evidence.'
|
|
1134
|
+
)
|
|
1135
|
+
else:
|
|
1136
|
+
proof_evidence_blocker = failed_proof_evidence_summary(proof_evidence)
|
|
1137
|
+
if proof_evidence_blocker:
|
|
1138
|
+
summary_lines.append('Structured proof evidence gate: ' + proof_evidence_blocker)
|
|
1139
|
+
|
|
1140
|
+
has_good_evidence = required_baseline_present and after_observation.get('valid') and not proof_evidence_blocker
|
|
1141
|
+
|
|
1142
|
+
if has_good_evidence:
|
|
1143
|
+
supervisor_request = build_supervisor_assessment_request(s, after_payload, after_observation, required_baseline_present, expected_path, evidence_bundle)
|
|
1144
|
+
s['verify_status'] = 'evidence_captured'
|
|
1145
|
+
s['merge_recommendation'] = 'pending-supervisor-judgment'
|
|
1146
|
+
s['proof_assessment'] = {}
|
|
1147
|
+
s['proof_assessment_source'] = None
|
|
1148
|
+
s['proof_assessment_request'] = supervisor_request
|
|
1149
|
+
s['verify_decision_request'] = {
|
|
1150
|
+
'status': s['verify_status'],
|
|
1151
|
+
'summary': 'Verify captured usable evidence and is waiting for supervising-agent proof assessment.',
|
|
1152
|
+
'expected_path': expected_path,
|
|
1153
|
+
'latest_observation': after_observation,
|
|
1154
|
+
'next_stage_options': ['verify', 'author', 'implement', 'ship', 'recon'],
|
|
1155
|
+
'recommended_stage': None,
|
|
1156
|
+
'continue_with_stage': None,
|
|
1157
|
+
'fields_agent_may_update': ['proof_assessment_json', 'capture_script', 'server_path', 'wait_for_selector', 'implementation_notes', 'proof_plan', 'assertions_json'],
|
|
1158
|
+
'assessment_request': supervisor_request,
|
|
1159
|
+
'instructions': [
|
|
1160
|
+
'Inspect the recon baseline(s), after evidence, and any structured artifacts together.',
|
|
1161
|
+
'The supervising agent should return a proof_assessment_json payload describing ship vs continued internal iteration.',
|
|
1162
|
+
'Do not escalate to the human unless the supervising agent concludes the workflow is genuinely stuck or not converging.',
|
|
1163
|
+
],
|
|
1164
|
+
}
|
|
1165
|
+
summary_lines.append('Proof assessment: awaiting supervising agent judgment')
|
|
1166
|
+
summary_lines.append('Proof next stage: supervising agent decides after reviewing the evidence packet')
|
|
1167
|
+
else:
|
|
1168
|
+
capture_retry = build_capture_retry_decision(after_observation, required_baseline_present, proof_evidence_blocker)
|
|
1169
|
+
s['verify_status'] = 'capture_incomplete'
|
|
1170
|
+
s['merge_recommendation'] = 'do-not-merge'
|
|
1171
|
+
s['proof_assessment'] = {}
|
|
1172
|
+
s['proof_assessment_source'] = None
|
|
1173
|
+
s['proof_assessment_request'] = {}
|
|
1174
|
+
s['verify_decision_request'] = {
|
|
1175
|
+
'status': s['verify_status'],
|
|
1176
|
+
'summary': capture_retry['summary'],
|
|
1177
|
+
'expected_path': expected_path,
|
|
1178
|
+
'latest_observation': after_observation,
|
|
1179
|
+
'capture_quality': capture_retry,
|
|
1180
|
+
'next_stage_options': ['author', 'verify', 'implement', 'recon'],
|
|
1181
|
+
'recommended_stage': capture_retry.get('recommended_stage') or 'author',
|
|
1182
|
+
'continue_with_stage': capture_retry.get('continue_with_stage') or 'author',
|
|
1183
|
+
'fields_agent_may_update': ['capture_script', 'server_path', 'wait_for_selector', 'proof_plan'],
|
|
1184
|
+
'instructions': [
|
|
1185
|
+
'The after-proof is missing or low quality, so return to author when the capture plan itself needs revision.',
|
|
1186
|
+
'Adjust capture_script, server_path, wait_for_selector, and/or proof_plan before rerunning verify.',
|
|
1187
|
+
'If the baseline itself is wrong, return to recon instead of forcing verify to rediscover context.',
|
|
1188
|
+
],
|
|
1189
|
+
}
|
|
1190
|
+
summary_lines.append('Proof assessment: not yet possible because the after capture is still incomplete')
|
|
1191
|
+
summary_lines.append('Proof next stage: ' + str(capture_retry.get('recommended_stage') or 'author'))
|
|
1192
|
+
|
|
1193
|
+
s['verify_summary'] = '\n'.join(summary_lines)
|
|
1194
|
+
s['proof_summary'] = s['verify_summary']
|
|
1195
|
+
s['evidence_notes'] = [
|
|
1196
|
+
'Review recon baseline(s), after evidence, and any supervising-agent proof assessment together.',
|
|
1197
|
+
'Proof evidence can be screenshots, structured metrics/logs/artifacts, assertions, or a mix.',
|
|
1198
|
+
'Treat screenshots as supporting proof, not the only proof source.',
|
|
1199
|
+
'Only merge if the evidence satisfies the stated success criteria for the chosen verification_mode.',
|
|
1200
|
+
]
|
|
1201
|
+
|
|
1202
|
+
save_state(s)
|
|
1203
|
+
|
|
1204
|
+
assessment_status = 'awaiting_supervising_agent' if s.get('verify_status') == 'evidence_captured' else 'capture_incomplete'
|
|
1205
|
+
|
|
1206
|
+
print()
|
|
1207
|
+
print('=' * 50)
|
|
1208
|
+
print('EVIDENCE')
|
|
1209
|
+
print('=' * 50)
|
|
1210
|
+
print('BEFORE: ' + (existing_before or '(none)'))
|
|
1211
|
+
if existing_prod:
|
|
1212
|
+
print('PROD: ' + existing_prod)
|
|
1213
|
+
print('AFTER SCREENSHOT: ' + s.get('after_cdn', '(none)'))
|
|
1214
|
+
if supporting.get('has_structured_payload'):
|
|
1215
|
+
print('AFTER STRUCTURED EVIDENCE: yes')
|
|
1216
|
+
print('VERIFY STATUS: ' + s.get('verify_status', 'unknown'))
|
|
1217
|
+
print('ASSERTION STATUS: ' + s.get('assertion_status', 'unknown'))
|
|
1218
|
+
print('MERGE RECOMMENDATION: ' + s.get('merge_recommendation', ''))
|
|
1219
|
+
print('PROOF ASSESSMENT: ' + assessment_status)
|
|
1220
|
+
print()
|
|
1221
|
+
print('PROOF SUMMARY:')
|
|
1222
|
+
print(s.get('proof_summary', ''))
|
|
1223
|
+
print(json.dumps({'ok': True, 'merge_recommendation': s.get('merge_recommendation', ''), 'verify_status': s.get('verify_status', ''), 'proof_assessment': assessment_status}))
|