@riddledc/riddle-proof 0.5.41 → 0.5.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-4YCWZVBN.js → chunk-U4VNN2CT.js} +2 -1
- package/dist/{chunk-ZBM67XGB.js → chunk-VSLU6XNI.js} +1 -1
- package/dist/{chunk-DWQK7J4U.js → chunk-XNRMMQAY.js} +3 -1
- package/dist/diagnostics.cjs +3 -1
- package/dist/diagnostics.js +1 -1
- package/dist/engine-harness.cjs +2 -1
- package/dist/engine-harness.js +2 -2
- package/dist/index.cjs +5 -2
- package/dist/index.js +3 -3
- package/dist/proof-run-core.cjs +2 -1
- package/dist/proof-run-core.js +1 -1
- package/dist/proof-run-engine.cjs +2 -1
- package/dist/proof-run-engine.js +1 -1
- package/package.json +1 -1
- package/runtime/lib/author.py +1 -0
- package/runtime/lib/recon.py +63 -6
- package/runtime/lib/riddle_core_call.mjs +22 -0
- package/runtime/lib/ship.py +3 -1
- package/runtime/lib/util.py +6 -1
- package/runtime/lib/verify.py +210 -9
- package/runtime/tests/recon_verify_smoke.py +144 -1
|
@@ -371,7 +371,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
371
371
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
372
372
|
const status = String(visualDelta.status || "missing");
|
|
373
373
|
if (status === "unmeasured") {
|
|
374
|
-
|
|
374
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
375
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
375
376
|
}
|
|
376
377
|
if (status === "measured" && visualDelta.passed === false) {
|
|
377
378
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
|
@@ -96,14 +96,16 @@ function summarizeCaptureArtifacts(payload) {
|
|
|
96
96
|
const record = isRecord(payload) ? payload : {};
|
|
97
97
|
const artifactJson = isRecord(record._artifact_json) ? record._artifact_json : {};
|
|
98
98
|
const proofJson = isRecord(record._proof_json) ? record._proof_json : isRecord(artifactJson["proof.json"]) ? artifactJson["proof.json"] : {};
|
|
99
|
+
const visualDiffJson = isRecord(record.visual_diff) ? record.visual_diff : isRecord(record.visualDiff) ? record.visualDiff : isRecord(artifactJson["visual-diff.json"]) ? artifactJson["visual-diff.json"] : {};
|
|
99
100
|
const consoleJson = isRecord(record.console) ? record.console : isRecord(artifactJson["console.json"]) ? artifactJson["console.json"] : {};
|
|
100
101
|
const result = isRecord(record.result) ? record.result : isRecord(proofJson.result) ? proofJson.result : isRecord(proofJson.script_result) ? proofJson.script_result : isRecord(proofJson.return_value) ? proofJson.return_value : isRecord(proofJson.value) ? proofJson.value : {};
|
|
101
102
|
const consoleSummary = isRecord(consoleJson.summary) ? redactForProofDiagnostics(consoleJson.summary, { string_limit: 500 }) : void 0;
|
|
103
|
+
const resultKeys = /* @__PURE__ */ new Set([...sortedKeys(result), ...sortedKeys(visualDiffJson)]);
|
|
102
104
|
return {
|
|
103
105
|
outputs: artifactItems(record.outputs).slice(0, 20).map((item) => artifactSummary(item, "outputs")),
|
|
104
106
|
screenshots: artifactItems(record.screenshots).slice(0, 10).map((item) => artifactSummary(item, "screenshots")),
|
|
105
107
|
artifacts: artifactItems(record.artifacts).slice(0, 20).map((item) => artifactSummary(item, "artifacts")),
|
|
106
|
-
result_keys:
|
|
108
|
+
result_keys: Array.from(resultKeys).sort(),
|
|
107
109
|
artifact_json: sortedKeys(artifactJson),
|
|
108
110
|
artifact_errors: artifactErrorMap(record._artifact_errors),
|
|
109
111
|
proof_script_error: Boolean(proofJson.script_error),
|
package/dist/diagnostics.cjs
CHANGED
|
@@ -127,14 +127,16 @@ function summarizeCaptureArtifacts(payload) {
|
|
|
127
127
|
const record = isRecord(payload) ? payload : {};
|
|
128
128
|
const artifactJson = isRecord(record._artifact_json) ? record._artifact_json : {};
|
|
129
129
|
const proofJson = isRecord(record._proof_json) ? record._proof_json : isRecord(artifactJson["proof.json"]) ? artifactJson["proof.json"] : {};
|
|
130
|
+
const visualDiffJson = isRecord(record.visual_diff) ? record.visual_diff : isRecord(record.visualDiff) ? record.visualDiff : isRecord(artifactJson["visual-diff.json"]) ? artifactJson["visual-diff.json"] : {};
|
|
130
131
|
const consoleJson = isRecord(record.console) ? record.console : isRecord(artifactJson["console.json"]) ? artifactJson["console.json"] : {};
|
|
131
132
|
const result = isRecord(record.result) ? record.result : isRecord(proofJson.result) ? proofJson.result : isRecord(proofJson.script_result) ? proofJson.script_result : isRecord(proofJson.return_value) ? proofJson.return_value : isRecord(proofJson.value) ? proofJson.value : {};
|
|
132
133
|
const consoleSummary = isRecord(consoleJson.summary) ? redactForProofDiagnostics(consoleJson.summary, { string_limit: 500 }) : void 0;
|
|
134
|
+
const resultKeys = /* @__PURE__ */ new Set([...sortedKeys(result), ...sortedKeys(visualDiffJson)]);
|
|
133
135
|
return {
|
|
134
136
|
outputs: artifactItems(record.outputs).slice(0, 20).map((item) => artifactSummary(item, "outputs")),
|
|
135
137
|
screenshots: artifactItems(record.screenshots).slice(0, 10).map((item) => artifactSummary(item, "screenshots")),
|
|
136
138
|
artifacts: artifactItems(record.artifacts).slice(0, 20).map((item) => artifactSummary(item, "artifacts")),
|
|
137
|
-
result_keys:
|
|
139
|
+
result_keys: Array.from(resultKeys).sort(),
|
|
138
140
|
artifact_json: sortedKeys(artifactJson),
|
|
139
141
|
artifact_errors: artifactErrorMap(record._artifact_errors),
|
|
140
142
|
proof_script_error: Boolean(proofJson.script_error),
|
package/dist/diagnostics.js
CHANGED
package/dist/engine-harness.cjs
CHANGED
|
@@ -379,7 +379,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
379
379
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
380
380
|
const status = String(visualDelta.status || "missing");
|
|
381
381
|
if (status === "unmeasured") {
|
|
382
|
-
|
|
382
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
383
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
383
384
|
}
|
|
384
385
|
if (status === "measured" && visualDelta.passed === false) {
|
|
385
386
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/engine-harness.js
CHANGED
|
@@ -2,11 +2,11 @@ import {
|
|
|
2
2
|
createDisabledRiddleProofAgentAdapter,
|
|
3
3
|
readRiddleProofRunStatus,
|
|
4
4
|
runRiddleProofEngineHarness
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-VSLU6XNI.js";
|
|
6
6
|
import "./chunk-RI25RGQP.js";
|
|
7
7
|
import "./chunk-7S7O3NKF.js";
|
|
8
8
|
import "./chunk-W7VTDN4T.js";
|
|
9
|
-
import "./chunk-
|
|
9
|
+
import "./chunk-U4VNN2CT.js";
|
|
10
10
|
export {
|
|
11
11
|
createDisabledRiddleProofAgentAdapter,
|
|
12
12
|
readRiddleProofRunStatus,
|
package/dist/index.cjs
CHANGED
|
@@ -379,7 +379,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
379
379
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
380
380
|
const status = String(visualDelta.status || "missing");
|
|
381
381
|
if (status === "unmeasured") {
|
|
382
|
-
|
|
382
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
383
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
383
384
|
}
|
|
384
385
|
if (status === "measured" && visualDelta.passed === false) {
|
|
385
386
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
|
@@ -5171,14 +5172,16 @@ function summarizeCaptureArtifacts(payload) {
|
|
|
5171
5172
|
const record = isRecord(payload) ? payload : {};
|
|
5172
5173
|
const artifactJson = isRecord(record._artifact_json) ? record._artifact_json : {};
|
|
5173
5174
|
const proofJson = isRecord(record._proof_json) ? record._proof_json : isRecord(artifactJson["proof.json"]) ? artifactJson["proof.json"] : {};
|
|
5175
|
+
const visualDiffJson = isRecord(record.visual_diff) ? record.visual_diff : isRecord(record.visualDiff) ? record.visualDiff : isRecord(artifactJson["visual-diff.json"]) ? artifactJson["visual-diff.json"] : {};
|
|
5174
5176
|
const consoleJson = isRecord(record.console) ? record.console : isRecord(artifactJson["console.json"]) ? artifactJson["console.json"] : {};
|
|
5175
5177
|
const result = isRecord(record.result) ? record.result : isRecord(proofJson.result) ? proofJson.result : isRecord(proofJson.script_result) ? proofJson.script_result : isRecord(proofJson.return_value) ? proofJson.return_value : isRecord(proofJson.value) ? proofJson.value : {};
|
|
5176
5178
|
const consoleSummary = isRecord(consoleJson.summary) ? redactForProofDiagnostics(consoleJson.summary, { string_limit: 500 }) : void 0;
|
|
5179
|
+
const resultKeys = /* @__PURE__ */ new Set([...sortedKeys(result), ...sortedKeys(visualDiffJson)]);
|
|
5177
5180
|
return {
|
|
5178
5181
|
outputs: artifactItems(record.outputs).slice(0, 20).map((item) => artifactSummary(item, "outputs")),
|
|
5179
5182
|
screenshots: artifactItems(record.screenshots).slice(0, 10).map((item) => artifactSummary(item, "screenshots")),
|
|
5180
5183
|
artifacts: artifactItems(record.artifacts).slice(0, 20).map((item) => artifactSummary(item, "artifacts")),
|
|
5181
|
-
result_keys:
|
|
5184
|
+
result_keys: Array.from(resultKeys).sort(),
|
|
5182
5185
|
artifact_json: sortedKeys(artifactJson),
|
|
5183
5186
|
artifact_errors: artifactErrorMap(record._artifact_errors),
|
|
5184
5187
|
proof_script_error: Boolean(proofJson.script_error),
|
package/dist/index.js
CHANGED
|
@@ -20,12 +20,12 @@ import {
|
|
|
20
20
|
createCaptureDiagnostic,
|
|
21
21
|
redactForProofDiagnostics,
|
|
22
22
|
summarizeCaptureArtifacts
|
|
23
|
-
} from "./chunk-
|
|
23
|
+
} from "./chunk-XNRMMQAY.js";
|
|
24
24
|
import {
|
|
25
25
|
createDisabledRiddleProofAgentAdapter,
|
|
26
26
|
readRiddleProofRunStatus,
|
|
27
27
|
runRiddleProofEngineHarness
|
|
28
|
-
} from "./chunk-
|
|
28
|
+
} from "./chunk-VSLU6XNI.js";
|
|
29
29
|
import {
|
|
30
30
|
RIDDLE_PROOF_CHECKPOINT_PACKET_VERSION,
|
|
31
31
|
RIDDLE_PROOF_CHECKPOINT_RESPONSE_VERSION,
|
|
@@ -67,7 +67,7 @@ import {
|
|
|
67
67
|
extractPlayabilityEvidence,
|
|
68
68
|
isRiddleProofPlayabilityMode
|
|
69
69
|
} from "./chunk-NSWT3VSV.js";
|
|
70
|
-
import "./chunk-
|
|
70
|
+
import "./chunk-U4VNN2CT.js";
|
|
71
71
|
export {
|
|
72
72
|
DEFAULT_DIAGNOSTIC_ARRAY_LIMIT,
|
|
73
73
|
DEFAULT_DIAGNOSTIC_HISTORY_LIMIT,
|
package/dist/proof-run-core.cjs
CHANGED
|
@@ -430,7 +430,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
430
430
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
431
431
|
const status = String(visualDelta.status || "missing");
|
|
432
432
|
if (status === "unmeasured") {
|
|
433
|
-
|
|
433
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
434
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
434
435
|
}
|
|
435
436
|
if (status === "measured" && visualDelta.passed === false) {
|
|
436
437
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/proof-run-core.js
CHANGED
|
@@ -412,7 +412,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
412
412
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
413
413
|
const status = String(visualDelta.status || "missing");
|
|
414
414
|
if (status === "unmeasured") {
|
|
415
|
-
|
|
415
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
416
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
416
417
|
}
|
|
417
418
|
if (status === "measured" && visualDelta.passed === false) {
|
|
418
419
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/proof-run-engine.js
CHANGED
package/package.json
CHANGED
package/runtime/lib/author.py
CHANGED
|
@@ -154,6 +154,7 @@ def author_request_payload(state, reference, baselines, current_plan, hypothesis
|
|
|
154
154
|
'For playable/gameplay proof, start the experience, send keyboard or pointer input, sample state before/after, measure non-HUD playfield/canvas pixel deltas across time, and set window.__riddleProofEvidence.playability or playability_evidence with version riddle-proof.playability.v1.',
|
|
155
155
|
'For data/audio/log/metric/custom proof, screenshots are optional; set window.__riddleProofEvidence inside page.evaluate to a JSON-serializable object with the measured observations the verifier should judge.',
|
|
156
156
|
'Do not assign globalThis.__riddleProofEvidence, window.__riddleProofEvidence, or self.__riddleProofEvidence outside page.evaluate; the Riddle worker context may not expose those globals safely.',
|
|
157
|
+
'When page.evaluate needs data from the outer Playwright script, pass exactly one serializable argument object; do not pass multiple positional arguments.',
|
|
157
158
|
'Do not begin capture_script with page.goto unless an in-app navigation is genuinely required after the preview opens the target route.',
|
|
158
159
|
'Only escalate to the human after the supervising agent concludes the workflow is genuinely stuck or not converging.',
|
|
159
160
|
],
|
package/runtime/lib/recon.py
CHANGED
|
@@ -33,6 +33,7 @@ import subprocess as sp
|
|
|
33
33
|
MAX_RECON_ATTEMPTS = 4
|
|
34
34
|
MIN_BODY_TEXT_LENGTH = 50
|
|
35
35
|
MIN_INTERACTIVE_ELEMENTS = 1
|
|
36
|
+
MIN_CANVAS_AREA = 50000
|
|
36
37
|
HYDRATION_WAIT_MS = 1500
|
|
37
38
|
PAGE_STATE_PREFIX = 'RIDDLE_PROOF_STATE:'
|
|
38
39
|
PROOF_EVIDENCE_PREFIX = 'RIDDLE_PROOF_EVIDENCE:'
|
|
@@ -423,6 +424,45 @@ def has_enriched_page_state(page_state):
|
|
|
423
424
|
)
|
|
424
425
|
|
|
425
426
|
|
|
427
|
+
def numeric_value(value):
|
|
428
|
+
if isinstance(value, bool) or value is None:
|
|
429
|
+
return None
|
|
430
|
+
if isinstance(value, (int, float)):
|
|
431
|
+
return float(value)
|
|
432
|
+
if isinstance(value, str):
|
|
433
|
+
text = value.strip().rstrip('%').replace(',', '')
|
|
434
|
+
if not text:
|
|
435
|
+
return None
|
|
436
|
+
try:
|
|
437
|
+
return float(text)
|
|
438
|
+
except Exception:
|
|
439
|
+
return None
|
|
440
|
+
return None
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def canvas_capture_signal(page_state):
|
|
444
|
+
if not isinstance(page_state, dict):
|
|
445
|
+
return {
|
|
446
|
+
'canvas_ready': False,
|
|
447
|
+
'canvas_count': 0,
|
|
448
|
+
'large_canvas_area': 0,
|
|
449
|
+
}
|
|
450
|
+
large_canvas_area = 0
|
|
451
|
+
for item in list_value(page_state.get('largeVisibleElements')):
|
|
452
|
+
if not isinstance(item, dict) or item.get('tag') != 'canvas':
|
|
453
|
+
continue
|
|
454
|
+
area = numeric_value(item.get('area')) or 0
|
|
455
|
+
if area > large_canvas_area:
|
|
456
|
+
large_canvas_area = area
|
|
457
|
+
canvas_count = int(numeric_value(page_state.get('canvasCount')) or 0)
|
|
458
|
+
return {
|
|
459
|
+
'canvas_ready': bool(canvas_count > 0 and large_canvas_area >= MIN_CANVAS_AREA),
|
|
460
|
+
'canvas_count': canvas_count,
|
|
461
|
+
'large_canvas_area': int(large_canvas_area),
|
|
462
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
|
|
426
466
|
def normalize_observed_path(value):
|
|
427
467
|
raw = str(value or '').strip()
|
|
428
468
|
if not raw:
|
|
@@ -538,6 +578,9 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
538
578
|
'buttons': [],
|
|
539
579
|
'links': [],
|
|
540
580
|
'canvas_count': 0,
|
|
581
|
+
'canvas_capture_ready': False,
|
|
582
|
+
'large_canvas_area': 0,
|
|
583
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
541
584
|
'large_visible_elements': [],
|
|
542
585
|
'semantic_anchor_count': 0,
|
|
543
586
|
'capture_error_messages': [],
|
|
@@ -572,6 +615,12 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
572
615
|
'large_visible_elements': list_value(page_state.get('largeVisibleElements'))[:10],
|
|
573
616
|
'semantic_anchor_count': semantic_anchor_count(page_state),
|
|
574
617
|
})
|
|
618
|
+
canvas_signal = canvas_capture_signal(page_state)
|
|
619
|
+
details.update({
|
|
620
|
+
'canvas_capture_ready': canvas_signal['canvas_ready'],
|
|
621
|
+
'large_canvas_area': canvas_signal['large_canvas_area'],
|
|
622
|
+
'min_canvas_area': canvas_signal['min_canvas_area'],
|
|
623
|
+
})
|
|
575
624
|
else:
|
|
576
625
|
details.update({
|
|
577
626
|
'body_text_length': MIN_BODY_TEXT_LENGTH + 100,
|
|
@@ -596,11 +645,20 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
596
645
|
details['capture_error_messages'].append(str(proof_json.get('script_error'))[:500])
|
|
597
646
|
|
|
598
647
|
reasons = []
|
|
599
|
-
|
|
648
|
+
canvas_ready = bool(details.get('canvas_capture_ready'))
|
|
649
|
+
body_text_ready = details['body_text_length'] >= MIN_BODY_TEXT_LENGTH or canvas_ready
|
|
650
|
+
interactive_ready = details['interactive_elements'] >= MIN_INTERACTIVE_ELEMENTS or canvas_ready
|
|
651
|
+
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1 or canvas_ready
|
|
652
|
+
details['body_text_ready'] = body_text_ready
|
|
653
|
+
details['interactive_ready'] = interactive_ready
|
|
654
|
+
details['semantic_ready'] = semantic_ready
|
|
655
|
+
details['canvas_capture_override'] = canvas_ready
|
|
656
|
+
|
|
657
|
+
if not body_text_ready:
|
|
600
658
|
reasons.append(f'blank/near-blank page (text length: {details["body_text_length"]})')
|
|
601
|
-
if
|
|
659
|
+
if not interactive_ready:
|
|
602
660
|
reasons.append(f'not interactive enough ({details["interactive_elements"]} interactive elements)')
|
|
603
|
-
if has_enriched_page_state(page_state) and
|
|
661
|
+
if has_enriched_page_state(page_state) and not semantic_ready:
|
|
604
662
|
reasons.append('no visible semantic UI anchors in page capture')
|
|
605
663
|
if details['has_errors']:
|
|
606
664
|
reasons.append('page has console/runtime errors')
|
|
@@ -610,11 +668,10 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
610
668
|
raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
|
|
611
669
|
reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
|
|
612
670
|
|
|
613
|
-
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1
|
|
614
671
|
telemetry_ready = (
|
|
615
672
|
details['has_screenshot']
|
|
616
|
-
and
|
|
617
|
-
and
|
|
673
|
+
and body_text_ready
|
|
674
|
+
and interactive_ready
|
|
618
675
|
and semantic_ready
|
|
619
676
|
and not details['has_errors']
|
|
620
677
|
)
|
|
@@ -136,6 +136,28 @@ async function run(tool, args) {
|
|
|
136
136
|
});
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
if (tool === "riddle_visual_diff") {
|
|
140
|
+
const {
|
|
141
|
+
async,
|
|
142
|
+
options,
|
|
143
|
+
stealth,
|
|
144
|
+
timeout_sec,
|
|
145
|
+
...visualDiffOptions
|
|
146
|
+
} = args;
|
|
147
|
+
const script = `return await visualDiff(${JSON.stringify(visualDiffOptions)});`;
|
|
148
|
+
return core.runWithDefaults(config, {
|
|
149
|
+
url: args.url_before,
|
|
150
|
+
script,
|
|
151
|
+
options: { ...(options ?? {}), returnResult: true },
|
|
152
|
+
stealth,
|
|
153
|
+
timeout_sec: timeout_sec ?? 60,
|
|
154
|
+
async,
|
|
155
|
+
}, {
|
|
156
|
+
include: ["result", "console", "visual_diff"],
|
|
157
|
+
returnAsync: !!async,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
139
161
|
if (tool === "riddle_run") {
|
|
140
162
|
return core.runWithDefaults(config, args.payload ?? args, {
|
|
141
163
|
include: ["screenshot", "console", "result"],
|
package/runtime/lib/ship.py
CHANGED
|
@@ -352,7 +352,9 @@ def visual_delta_ship_blocker(state):
|
|
|
352
352
|
return ''
|
|
353
353
|
status = str(visual_delta.get('status') or 'missing')
|
|
354
354
|
if status == 'unmeasured':
|
|
355
|
-
|
|
355
|
+
reason = str(visual_delta.get('reason') or '').strip()
|
|
356
|
+
detail = f': {reason}' if reason else ''
|
|
357
|
+
return 'visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof' + detail
|
|
356
358
|
if status == 'measured' and visual_delta.get('passed') is False:
|
|
357
359
|
return 'visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof'
|
|
358
360
|
reason = str(visual_delta.get('reason') or '').strip()
|
package/runtime/lib/util.py
CHANGED
|
@@ -14,6 +14,7 @@ RIDDLE_DIRECT_TOOLS = {
|
|
|
14
14
|
'riddle_build_preview',
|
|
15
15
|
'riddle_build_preview_status',
|
|
16
16
|
'riddle_script',
|
|
17
|
+
'riddle_visual_diff',
|
|
17
18
|
'riddle_run',
|
|
18
19
|
}
|
|
19
20
|
CAPTURE_ARTIFACT_JSON_LIMIT = 256 * 1024
|
|
@@ -767,7 +768,7 @@ def enrich_capture_payload(payload):
|
|
|
767
768
|
artifact_json = dict(enriched.get('_artifact_json') or {})
|
|
768
769
|
artifact_errors = dict(enriched.get('_artifact_errors') or {})
|
|
769
770
|
|
|
770
|
-
for name in ('console.json', 'proof.json'):
|
|
771
|
+
for name in ('console.json', 'proof.json', 'visual-diff.json'):
|
|
771
772
|
if name in artifact_json or name in artifact_errors:
|
|
772
773
|
continue
|
|
773
774
|
item = capture_output_item(enriched, name)
|
|
@@ -798,6 +799,10 @@ def enrich_capture_payload(payload):
|
|
|
798
799
|
enriched['result'] = result
|
|
799
800
|
break
|
|
800
801
|
|
|
802
|
+
visual_diff_json = artifact_json.get('visual-diff.json')
|
|
803
|
+
if isinstance(visual_diff_json, dict) and not enriched.get('visual_diff'):
|
|
804
|
+
enriched['visual_diff'] = visual_diff_json
|
|
805
|
+
|
|
801
806
|
return enriched
|
|
802
807
|
|
|
803
808
|
|
package/runtime/lib/verify.py
CHANGED
|
@@ -33,6 +33,7 @@ import subprocess as sp
|
|
|
33
33
|
|
|
34
34
|
MIN_BODY_TEXT_LENGTH = 50
|
|
35
35
|
MIN_INTERACTIVE_ELEMENTS = 1
|
|
36
|
+
MIN_CANVAS_AREA = 50000
|
|
36
37
|
HYDRATION_WAIT_MS = 1500
|
|
37
38
|
PAGE_STATE_PREFIX = 'RIDDLE_PROOF_STATE:'
|
|
38
39
|
PROOF_EVIDENCE_PREFIX = 'RIDDLE_PROOF_EVIDENCE:'
|
|
@@ -54,13 +55,24 @@ MIN_VISUAL_CHANGED_PIXELS = 5000
|
|
|
54
55
|
VISUAL_DELTA_PERCENT_KEYS = {
|
|
55
56
|
'change_pct', 'change_percent', 'changed_percent', 'percent_changed',
|
|
56
57
|
'diff_percent', 'visual_delta_percent', 'pixel_change_percent',
|
|
58
|
+
'changepercent', 'changedpercent', 'percentchanged', 'diffpercent',
|
|
59
|
+
'visualdeltapercent', 'pixelchangepercent', 'difference_percent',
|
|
60
|
+
'differencepercent', 'diff_percentage', 'diffpercentage',
|
|
61
|
+
'mismatch_percent', 'mismatchpercent', 'percentage', 'percent',
|
|
57
62
|
}
|
|
58
63
|
VISUAL_DELTA_RATIO_KEYS = {
|
|
59
64
|
'change_ratio', 'changed_ratio', 'diff_ratio', 'visual_delta_ratio',
|
|
65
|
+
'changeratio', 'changedratio', 'diffratio', 'visualdeltaratio',
|
|
66
|
+
'difference_ratio', 'differenceratio', 'mismatch_ratio', 'mismatchratio',
|
|
60
67
|
}
|
|
61
68
|
VISUAL_CHANGED_PIXEL_KEYS = {
|
|
62
69
|
'changed_pixels', 'changed_pixel_count', 'changedpixels',
|
|
63
70
|
'diff_pixels', 'pixel_delta', 'visual_delta_pixels',
|
|
71
|
+
'diffpixels', 'pixeldelta', 'visualdeltapixels', 'different_pixels',
|
|
72
|
+
'differentpixels', 'mismatch_pixels', 'mismatchpixels', 'pixels_changed',
|
|
73
|
+
'pixelschanged', 'diff_pixel_count', 'diffpixelcount',
|
|
74
|
+
'pixel_diff_count', 'pixeldiffcount', 'different_pixel_count',
|
|
75
|
+
'differentpixelcount', 'num_different_pixels', 'numdifferentpixels',
|
|
64
76
|
}
|
|
65
77
|
VISUAL_TOTAL_PIXEL_KEYS = {
|
|
66
78
|
'total_pixels', 'total_pixel_count', 'pixel_count', 'totalpixels',
|
|
@@ -799,11 +811,19 @@ def extract_visual_delta(payload):
|
|
|
799
811
|
payload = enrich_capture_payload(payload)
|
|
800
812
|
result = payload.get('result') if isinstance(payload, dict) else {}
|
|
801
813
|
proof_json = payload.get('_proof_json') if isinstance(payload, dict) else {}
|
|
814
|
+
artifact_json = payload.get('_artifact_json') if isinstance(payload, dict) else {}
|
|
802
815
|
proof_evidence = extract_proof_evidence(payload)
|
|
816
|
+
screenshot_url = extract_screenshot_url(payload)
|
|
817
|
+
artifact_summary = summarize_capture_artifacts(payload)
|
|
803
818
|
candidates = [
|
|
804
819
|
payload if isinstance(payload, dict) else {},
|
|
805
820
|
result if isinstance(result, dict) else {},
|
|
806
821
|
proof_json if isinstance(proof_json, dict) else {},
|
|
822
|
+
payload.get('visual_diff') if isinstance(payload, dict) and isinstance(payload.get('visual_diff'), dict) else {},
|
|
823
|
+
payload.get('visualDiff') if isinstance(payload, dict) and isinstance(payload.get('visualDiff'), dict) else {},
|
|
824
|
+
result.get('visual_diff') if isinstance(result, dict) and isinstance(result.get('visual_diff'), dict) else {},
|
|
825
|
+
result.get('visualDiff') if isinstance(result, dict) and isinstance(result.get('visualDiff'), dict) else {},
|
|
826
|
+
artifact_json.get('visual-diff.json') if isinstance(artifact_json, dict) and isinstance(artifact_json.get('visual-diff.json'), dict) else {},
|
|
807
827
|
proof_evidence,
|
|
808
828
|
]
|
|
809
829
|
|
|
@@ -837,6 +857,23 @@ def extract_visual_delta(payload):
|
|
|
837
857
|
percent = (changed_pixels / total_pixels) * 100
|
|
838
858
|
|
|
839
859
|
if percent is None and changed_pixels is None:
|
|
860
|
+
has_artifacts = bool(
|
|
861
|
+
screenshot_url
|
|
862
|
+
or artifact_summary.get('outputs')
|
|
863
|
+
or artifact_summary.get('screenshots')
|
|
864
|
+
or artifact_summary.get('artifact_json')
|
|
865
|
+
)
|
|
866
|
+
reason = 'No measured before/after visual delta was found in proof evidence.'
|
|
867
|
+
if screenshot_url:
|
|
868
|
+
reason = (
|
|
869
|
+
'After screenshot artifact is present, but no measured before/after visual delta metrics were emitted; '
|
|
870
|
+
'the comparator did not run or did not publish change_percent/changed_pixels.'
|
|
871
|
+
)
|
|
872
|
+
elif has_artifacts:
|
|
873
|
+
reason = (
|
|
874
|
+
'Capture artifacts are present, but no measured before/after visual delta metrics were emitted; '
|
|
875
|
+
'the comparator did not run or did not publish change_percent/changed_pixels.'
|
|
876
|
+
)
|
|
840
877
|
return {
|
|
841
878
|
'status': 'unmeasured',
|
|
842
879
|
'passed': None,
|
|
@@ -845,7 +882,19 @@ def extract_visual_delta(payload):
|
|
|
845
882
|
'total_pixels': int(total_pixels) if total_pixels else None,
|
|
846
883
|
'min_change_percent': MIN_VISUAL_DELTA_PERCENT,
|
|
847
884
|
'min_changed_pixels': MIN_VISUAL_CHANGED_PIXELS,
|
|
848
|
-
'reason':
|
|
885
|
+
'reason': reason,
|
|
886
|
+
'diagnostic': {
|
|
887
|
+
'after_screenshot_present': bool(screenshot_url),
|
|
888
|
+
'proof_evidence_present': proof_evidence is not None,
|
|
889
|
+
'artifact_output_count': len(artifact_summary.get('outputs') or []),
|
|
890
|
+
'artifact_screenshot_count': len(artifact_summary.get('screenshots') or []),
|
|
891
|
+
'artifact_json': list(artifact_summary.get('artifact_json') or []),
|
|
892
|
+
'expected_metric_keys': {
|
|
893
|
+
'percent': sorted(VISUAL_DELTA_PERCENT_KEYS),
|
|
894
|
+
'changed_pixels': sorted(VISUAL_CHANGED_PIXEL_KEYS),
|
|
895
|
+
'total_pixels': sorted(VISUAL_TOTAL_PIXEL_KEYS),
|
|
896
|
+
},
|
|
897
|
+
},
|
|
849
898
|
}
|
|
850
899
|
|
|
851
900
|
percent_pass = percent is not None and percent >= MIN_VISUAL_DELTA_PERCENT
|
|
@@ -867,6 +916,114 @@ def extract_visual_delta(payload):
|
|
|
867
916
|
}
|
|
868
917
|
|
|
869
918
|
|
|
919
|
+
def visual_delta_baseline_candidate(state, results):
|
|
920
|
+
reference = str(state.get('requested_reference') or state.get('reference') or 'both').strip().lower()
|
|
921
|
+
baseline = (results.get('baseline') or {}) if isinstance(results, dict) else {}
|
|
922
|
+
candidates = []
|
|
923
|
+
if reference in ('before', 'both', ''):
|
|
924
|
+
candidates.append(('before', (baseline.get('before') or {}).get('url') if isinstance(baseline.get('before'), dict) else ''))
|
|
925
|
+
if reference in ('prod', 'production', 'both'):
|
|
926
|
+
candidates.append(('prod', (baseline.get('prod') or {}).get('url') if isinstance(baseline.get('prod'), dict) else ''))
|
|
927
|
+
candidates.extend([
|
|
928
|
+
('before', state.get('before_cdn') or ''),
|
|
929
|
+
('prod', state.get('prod_cdn') or ''),
|
|
930
|
+
])
|
|
931
|
+
for label, url in candidates:
|
|
932
|
+
text = str(url or '').strip()
|
|
933
|
+
if text:
|
|
934
|
+
return {'label': label, 'url': text}
|
|
935
|
+
return {'label': '', 'url': ''}
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
def add_visual_delta_diagnostic(visual_delta, key, value):
|
|
939
|
+
updated = dict(visual_delta or {})
|
|
940
|
+
diagnostic = dict(updated.get('diagnostic') or {})
|
|
941
|
+
diagnostic[key] = value
|
|
942
|
+
updated['diagnostic'] = diagnostic
|
|
943
|
+
return updated
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def measure_visual_delta_against_baseline(state, results, after_payload, current_visual_delta):
|
|
947
|
+
if not visual_delta_applies(state.get('verification_mode')):
|
|
948
|
+
return current_visual_delta
|
|
949
|
+
if isinstance(current_visual_delta, dict) and current_visual_delta.get('status') == 'measured':
|
|
950
|
+
return current_visual_delta
|
|
951
|
+
|
|
952
|
+
baseline = visual_delta_baseline_candidate(state, results)
|
|
953
|
+
before_url = baseline.get('url') or ''
|
|
954
|
+
after_url = extract_screenshot_url(after_payload, 'after-proof') or str(state.get('after_cdn') or '').strip()
|
|
955
|
+
if not before_url or not after_url:
|
|
956
|
+
missing = []
|
|
957
|
+
if not before_url:
|
|
958
|
+
missing.append('baseline screenshot')
|
|
959
|
+
if not after_url:
|
|
960
|
+
missing.append('after screenshot')
|
|
961
|
+
return add_visual_delta_diagnostic(
|
|
962
|
+
current_visual_delta,
|
|
963
|
+
'visual_diff_fallback',
|
|
964
|
+
{
|
|
965
|
+
'status': 'skipped',
|
|
966
|
+
'reason': 'missing ' + ' and '.join(missing),
|
|
967
|
+
'baseline_label': baseline.get('label') or '',
|
|
968
|
+
'before_url_present': bool(before_url),
|
|
969
|
+
'after_url_present': bool(after_url),
|
|
970
|
+
},
|
|
971
|
+
)
|
|
972
|
+
|
|
973
|
+
args = {
|
|
974
|
+
'url_before': before_url,
|
|
975
|
+
'url_after': after_url,
|
|
976
|
+
'delay_ms': 250,
|
|
977
|
+
'timeout_sec': 60,
|
|
978
|
+
}
|
|
979
|
+
try:
|
|
980
|
+
payload = invoke_retry('riddle_visual_diff', args, retries=2, timeout=180)
|
|
981
|
+
append_capture_diagnostic(state, 'visual_delta', 'riddle_visual_diff', args, payload)
|
|
982
|
+
except Exception as exc:
|
|
983
|
+
return add_visual_delta_diagnostic(
|
|
984
|
+
current_visual_delta,
|
|
985
|
+
'visual_diff_fallback',
|
|
986
|
+
{
|
|
987
|
+
'status': 'error',
|
|
988
|
+
'error': type(exc).__name__ + ': ' + str(exc)[:300],
|
|
989
|
+
'baseline_label': baseline.get('label') or '',
|
|
990
|
+
},
|
|
991
|
+
)
|
|
992
|
+
|
|
993
|
+
measured = extract_visual_delta(payload)
|
|
994
|
+
if isinstance(measured, dict) and measured.get('status') == 'measured':
|
|
995
|
+
measured['source'] = 'riddle_visual_diff'
|
|
996
|
+
measured['comparison'] = {
|
|
997
|
+
'baseline_label': baseline.get('label') or '',
|
|
998
|
+
'before_url': before_url,
|
|
999
|
+
'after_url': after_url,
|
|
1000
|
+
}
|
|
1001
|
+
if isinstance(payload, dict):
|
|
1002
|
+
diff_url = extract_screenshot_url(payload, 'visual-diff') or extract_screenshot_url(payload, 'diff')
|
|
1003
|
+
if diff_url:
|
|
1004
|
+
measured['comparison']['diff_url'] = diff_url
|
|
1005
|
+
measured['artifact_summary'] = summarize_capture_artifacts(payload)
|
|
1006
|
+
return measured
|
|
1007
|
+
|
|
1008
|
+
updated = add_visual_delta_diagnostic(
|
|
1009
|
+
current_visual_delta,
|
|
1010
|
+
'visual_diff_fallback',
|
|
1011
|
+
{
|
|
1012
|
+
'status': 'unmeasured',
|
|
1013
|
+
'baseline_label': baseline.get('label') or '',
|
|
1014
|
+
'before_url': before_url,
|
|
1015
|
+
'after_url': after_url,
|
|
1016
|
+
'payload_ok': payload.get('ok') if isinstance(payload, dict) else None,
|
|
1017
|
+
'payload_error': str(payload.get('error') or payload.get('stderr') or '')[:500] if isinstance(payload, dict) else '',
|
|
1018
|
+
'artifact_summary': summarize_capture_artifacts(payload) if isinstance(payload, dict) else {},
|
|
1019
|
+
},
|
|
1020
|
+
)
|
|
1021
|
+
reason = str(updated.get('reason') or '').strip()
|
|
1022
|
+
suffix = ' Riddle visual_diff fallback ran but did not publish a recognizable numeric delta.'
|
|
1023
|
+
updated['reason'] = (reason + suffix).strip() if reason else suffix.strip()
|
|
1024
|
+
return updated
|
|
1025
|
+
|
|
1026
|
+
|
|
870
1027
|
def visual_delta_applies(verification_mode):
|
|
871
1028
|
return screenshot_required_for_mode(verification_mode)
|
|
872
1029
|
|
|
@@ -891,7 +1048,8 @@ def visual_delta_blocker_for_mode(verification_mode, visual_delta):
|
|
|
891
1048
|
status = str(visual_delta.get('status') or 'missing')
|
|
892
1049
|
reason = str(visual_delta.get('reason') or '').strip()
|
|
893
1050
|
if status == 'unmeasured':
|
|
894
|
-
|
|
1051
|
+
detail = f' Reason: {reason}' if reason else ''
|
|
1052
|
+
return 'visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof; capture a measured before/after visual delta or choose needs_richer_proof.' + detail
|
|
895
1053
|
if status == 'measured' and isinstance(visual_delta, dict) and visual_delta.get('passed') is False:
|
|
896
1054
|
return 'visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof; the measured change did not clear the threshold.'
|
|
897
1055
|
if reason:
|
|
@@ -926,6 +1084,29 @@ def has_enriched_page_state(page_state):
|
|
|
926
1084
|
)
|
|
927
1085
|
|
|
928
1086
|
|
|
1087
|
+
def canvas_capture_signal(page_state):
|
|
1088
|
+
if not isinstance(page_state, dict):
|
|
1089
|
+
return {
|
|
1090
|
+
'canvas_ready': False,
|
|
1091
|
+
'canvas_count': 0,
|
|
1092
|
+
'large_canvas_area': 0,
|
|
1093
|
+
}
|
|
1094
|
+
large_canvas_area = 0
|
|
1095
|
+
for item in list_value(page_state.get('largeVisibleElements')):
|
|
1096
|
+
if not isinstance(item, dict) or item.get('tag') != 'canvas':
|
|
1097
|
+
continue
|
|
1098
|
+
area = metric_number(item.get('area')) or 0
|
|
1099
|
+
if area > large_canvas_area:
|
|
1100
|
+
large_canvas_area = area
|
|
1101
|
+
canvas_count = int(metric_number(page_state.get('canvasCount')) or 0)
|
|
1102
|
+
return {
|
|
1103
|
+
'canvas_ready': bool(canvas_count > 0 and large_canvas_area >= MIN_CANVAS_AREA),
|
|
1104
|
+
'canvas_count': canvas_count,
|
|
1105
|
+
'large_canvas_area': int(large_canvas_area),
|
|
1106
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
|
|
929
1110
|
def normalize_observed_path(value):
|
|
930
1111
|
raw = str(value or '').strip()
|
|
931
1112
|
if not raw:
|
|
@@ -1135,6 +1316,7 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1135
1316
|
mode = normalized_verification_mode(verification_mode)
|
|
1136
1317
|
supporting = collect_supporting_artifacts(payload)
|
|
1137
1318
|
structured_ready = bool(supporting.get('has_structured_payload'))
|
|
1319
|
+
playability_ready = mode in PLAYABILITY_MODES and bool(supporting.get('playability_ready'))
|
|
1138
1320
|
screenshot_required = screenshot_required_for_mode(mode)
|
|
1139
1321
|
details = {
|
|
1140
1322
|
'verification_mode': mode,
|
|
@@ -1143,6 +1325,10 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1143
1325
|
'screenshot_required': screenshot_required,
|
|
1144
1326
|
'structured_evidence_present': structured_ready,
|
|
1145
1327
|
'proof_evidence_present': bool(supporting.get('proof_evidence_present')),
|
|
1328
|
+
'playability_ready': playability_ready,
|
|
1329
|
+
'canvas_capture_ready': False,
|
|
1330
|
+
'large_canvas_area': 0,
|
|
1331
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
1146
1332
|
'proof_evidence_sample': supporting.get('proof_evidence_sample', ''),
|
|
1147
1333
|
'body_text_length': 0,
|
|
1148
1334
|
'interactive_elements': 0,
|
|
@@ -1183,6 +1369,12 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1183
1369
|
'large_visible_elements': list_value(page_state.get('largeVisibleElements'))[:10],
|
|
1184
1370
|
'semantic_anchor_count': semantic_anchor_count(page_state),
|
|
1185
1371
|
})
|
|
1372
|
+
canvas_signal = canvas_capture_signal(page_state)
|
|
1373
|
+
details.update({
|
|
1374
|
+
'canvas_capture_ready': canvas_signal['canvas_ready'],
|
|
1375
|
+
'large_canvas_area': canvas_signal['large_canvas_area'],
|
|
1376
|
+
'min_canvas_area': canvas_signal['min_canvas_area'],
|
|
1377
|
+
})
|
|
1186
1378
|
elif screenshot_url:
|
|
1187
1379
|
details.update({
|
|
1188
1380
|
'body_text_length': MIN_BODY_TEXT_LENGTH + 100,
|
|
@@ -1220,11 +1412,20 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1220
1412
|
reasons.append('no screenshot or structured proof evidence in capture')
|
|
1221
1413
|
|
|
1222
1414
|
should_enforce_visual_readiness = screenshot_required or (details['has_screenshot'] and not structured_ready)
|
|
1223
|
-
|
|
1415
|
+
canvas_ready = bool(details.get('canvas_capture_ready'))
|
|
1416
|
+
body_text_ready = details['body_text_length'] >= MIN_BODY_TEXT_LENGTH or canvas_ready or playability_ready
|
|
1417
|
+
interactive_ready = details['interactive_elements'] >= MIN_INTERACTIVE_ELEMENTS or canvas_ready or playability_ready
|
|
1418
|
+
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1 or canvas_ready or playability_ready
|
|
1419
|
+
details['body_text_ready'] = body_text_ready
|
|
1420
|
+
details['interactive_ready'] = interactive_ready
|
|
1421
|
+
details['semantic_ready'] = semantic_ready
|
|
1422
|
+
details['canvas_or_playability_override'] = bool(should_enforce_visual_readiness and (canvas_ready or playability_ready))
|
|
1423
|
+
|
|
1424
|
+
if should_enforce_visual_readiness and not body_text_ready:
|
|
1224
1425
|
reasons.append(f'blank/near-blank page (text length: {details["body_text_length"]})')
|
|
1225
|
-
if should_enforce_visual_readiness and
|
|
1426
|
+
if should_enforce_visual_readiness and not interactive_ready:
|
|
1226
1427
|
reasons.append(f'not interactive enough ({details["interactive_elements"]} interactive elements)')
|
|
1227
|
-
if should_enforce_visual_readiness and has_enriched_page_state(page_state) and
|
|
1428
|
+
if should_enforce_visual_readiness and has_enriched_page_state(page_state) and not semantic_ready:
|
|
1228
1429
|
reasons.append('no visible semantic UI anchors in page capture')
|
|
1229
1430
|
if details['has_errors']:
|
|
1230
1431
|
reasons.append('page has console/runtime errors')
|
|
@@ -1234,15 +1435,14 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1234
1435
|
raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
|
|
1235
1436
|
reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
|
|
1236
1437
|
|
|
1237
|
-
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1
|
|
1238
1438
|
visual_ready = (
|
|
1239
1439
|
details['has_screenshot']
|
|
1240
|
-
and
|
|
1241
|
-
and
|
|
1440
|
+
and body_text_ready
|
|
1441
|
+
and interactive_ready
|
|
1242
1442
|
and semantic_ready
|
|
1243
1443
|
and not details['has_errors']
|
|
1244
1444
|
)
|
|
1245
|
-
telemetry_ready = (visual_ready or structured_ready) and not details['has_errors']
|
|
1445
|
+
telemetry_ready = (visual_ready or structured_ready or playability_ready) and not details['has_errors']
|
|
1246
1446
|
|
|
1247
1447
|
return {
|
|
1248
1448
|
'valid': len(reasons) == 0 and telemetry_ready,
|
|
@@ -1368,6 +1568,7 @@ def build_evidence_bundle(state, results, after_payload, after_observation, requ
|
|
|
1368
1568
|
if visual_delta_applies(state.get('verification_mode'))
|
|
1369
1569
|
else {'status': 'not_applicable', 'passed': None, 'reason': 'Verification mode does not require visual delta gating.'}
|
|
1370
1570
|
)
|
|
1571
|
+
visual_delta = measure_visual_delta_against_baseline(state, results, after_payload, visual_delta)
|
|
1371
1572
|
semantic_context = build_semantic_context(state, results, after_observation, expected_path)
|
|
1372
1573
|
artifact_contract = artifact_contract_for_mode(state.get('verification_mode'))
|
|
1373
1574
|
artifact_production = artifact_production_summary(after_payload, supporting)
|
|
@@ -566,8 +566,150 @@ def run_verify_quality_ignores_proof_telemetry_console_text():
|
|
|
566
566
|
})
|
|
567
567
|
assert unmeasured_delta['status'] == 'unmeasured'
|
|
568
568
|
assert unmeasured_delta['passed'] is None
|
|
569
|
+
assert unmeasured_delta['diagnostic']['after_screenshot_present'] is True
|
|
570
|
+
assert 'After screenshot artifact is present' in unmeasured_delta['reason']
|
|
569
571
|
|
|
570
|
-
|
|
572
|
+
visual_diff_delta = namespace['extract_visual_delta']({
|
|
573
|
+
'ok': True,
|
|
574
|
+
'outputs': [{'name': 'visual-diff.json', 'url': 'https://cdn.example.com/visual-diff.json'}],
|
|
575
|
+
'_artifact_json': {
|
|
576
|
+
'visual-diff.json': {
|
|
577
|
+
'changePercent': '1.45',
|
|
578
|
+
'diffPixelCount': 14094,
|
|
579
|
+
'totalPixels': 972000,
|
|
580
|
+
},
|
|
581
|
+
},
|
|
582
|
+
})
|
|
583
|
+
assert visual_diff_delta['status'] == 'measured'
|
|
584
|
+
assert visual_diff_delta['passed'] is True
|
|
585
|
+
assert visual_diff_delta['change_percent'] == 1.45
|
|
586
|
+
assert visual_diff_delta['changed_pixels'] == 14094
|
|
587
|
+
|
|
588
|
+
fallback_calls = []
|
|
589
|
+
|
|
590
|
+
def fake_invoke_retry(tool, args, retries=3, timeout=180):
|
|
591
|
+
fallback_calls.append({'tool': tool, 'args': args, 'retries': retries, 'timeout': timeout})
|
|
592
|
+
return {
|
|
593
|
+
'ok': True,
|
|
594
|
+
'visual_diff': {
|
|
595
|
+
'diffPercentage': 0.82,
|
|
596
|
+
'differentPixels': 7970,
|
|
597
|
+
'totalPixels': 972000,
|
|
598
|
+
},
|
|
599
|
+
'outputs': [{'name': 'visual-diff.png', 'url': 'https://cdn.example.com/visual-diff.png'}],
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
namespace['invoke_retry'] = fake_invoke_retry
|
|
603
|
+
namespace['append_capture_diagnostic'] = lambda *args, **kwargs: None
|
|
604
|
+
fallback_delta = namespace['measure_visual_delta_against_baseline'](
|
|
605
|
+
{'verification_mode': 'visual', 'requested_reference': 'before', 'before_cdn': 'https://cdn.example.com/before.png'},
|
|
606
|
+
{'baseline': {'before': {'url': 'https://cdn.example.com/before.png'}}},
|
|
607
|
+
{'ok': True, 'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after.png'}]},
|
|
608
|
+
unmeasured_delta,
|
|
609
|
+
)
|
|
610
|
+
assert fallback_delta['status'] == 'measured'
|
|
611
|
+
assert fallback_delta['source'] == 'riddle_visual_diff'
|
|
612
|
+
assert fallback_delta['comparison']['before_url'] == 'https://cdn.example.com/before.png'
|
|
613
|
+
assert fallback_delta['comparison']['after_url'] == 'https://cdn.example.com/after.png'
|
|
614
|
+
assert fallback_calls[0]['tool'] == 'riddle_visual_diff'
|
|
615
|
+
|
|
616
|
+
canvas_payload = {
|
|
617
|
+
'bodyTextLength': 7,
|
|
618
|
+
'visibleTextSample': 'Luge',
|
|
619
|
+
'interactiveElements': 1,
|
|
620
|
+
'visibleInteractiveElements': 1,
|
|
621
|
+
'pathname': '/games/luge-run',
|
|
622
|
+
'title': 'Luge Run',
|
|
623
|
+
'headings': [],
|
|
624
|
+
'buttons': [],
|
|
625
|
+
'links': [],
|
|
626
|
+
'canvasCount': 1,
|
|
627
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': '', 'area': 420000}],
|
|
628
|
+
}
|
|
629
|
+
canvas_quality = namespace['evaluate_capture_quality']({
|
|
630
|
+
'ok': True,
|
|
631
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
632
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
633
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps(canvas_payload)],
|
|
634
|
+
}, '/games/luge-run', 'visual')
|
|
635
|
+
assert canvas_quality['valid'] is True, canvas_quality
|
|
636
|
+
assert canvas_quality['details']['canvas_capture_ready'] is True
|
|
637
|
+
assert canvas_quality['details']['body_text_ready'] is True
|
|
638
|
+
assert 'blank/near-blank' not in canvas_quality['reason']
|
|
639
|
+
|
|
640
|
+
playability_payload = {
|
|
641
|
+
**canvas_payload,
|
|
642
|
+
'interactiveElements': 0,
|
|
643
|
+
'visibleInteractiveElements': 0,
|
|
644
|
+
'largeVisibleElements': [],
|
|
645
|
+
}
|
|
646
|
+
playability_evidence = {
|
|
647
|
+
'input_events': [{'type': 'pointerdown'}],
|
|
648
|
+
'state_delta': {'changed': True, 'changed_keys': ['distance']},
|
|
649
|
+
'canvas_delta': {'changed_pixels': 18000},
|
|
650
|
+
'time_delta_ms': 1300,
|
|
651
|
+
}
|
|
652
|
+
playable_quality = namespace['evaluate_capture_quality']({
|
|
653
|
+
'ok': True,
|
|
654
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
655
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
656
|
+
'console': [
|
|
657
|
+
'RIDDLE_PROOF_STATE:' + json.dumps(playability_payload),
|
|
658
|
+
'RIDDLE_PROOF_EVIDENCE:' + json.dumps({'playability': playability_evidence}),
|
|
659
|
+
],
|
|
660
|
+
}, '/games/luge-run', 'playable')
|
|
661
|
+
assert playable_quality['valid'] is True, playable_quality
|
|
662
|
+
assert playable_quality['details']['playability_ready'] is True
|
|
663
|
+
assert playable_quality['details']['interactive_ready'] is True
|
|
664
|
+
|
|
665
|
+
return {
|
|
666
|
+
'ok': True,
|
|
667
|
+
'telemetry_valid': quality['valid'],
|
|
668
|
+
'weak_delta_passed': weak_delta['passed'],
|
|
669
|
+
'canvas_valid': canvas_quality['valid'],
|
|
670
|
+
'playable_valid': playable_quality['valid'],
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def run_recon_quality_accepts_canvas_first_routes():
|
|
675
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-recon-canvas-quality-'))
|
|
676
|
+
state_path = tempdir / 'state.json'
|
|
677
|
+
try:
|
|
678
|
+
write_state(state_path, {
|
|
679
|
+
'after_worktree': str(tempdir),
|
|
680
|
+
'before_worktree': str(tempdir),
|
|
681
|
+
})
|
|
682
|
+
with temporary_env(RIDDLE_PROOF_STATE_FILE=str(state_path)):
|
|
683
|
+
sys.modules.pop('util', None)
|
|
684
|
+
source = RECON_PATH.read_text()
|
|
685
|
+
helpers_source = source.split('\ndef clean_next_cache', 1)[0]
|
|
686
|
+
namespace = {'__file__': str(RECON_PATH)}
|
|
687
|
+
exec(compile(helpers_source, str(RECON_PATH), 'exec'), namespace)
|
|
688
|
+
canvas_payload = {
|
|
689
|
+
'bodyTextLength': 4,
|
|
690
|
+
'visibleTextSample': 'Game',
|
|
691
|
+
'interactiveElements': 1,
|
|
692
|
+
'visibleInteractiveElements': 1,
|
|
693
|
+
'pathname': '/games/luge-run',
|
|
694
|
+
'title': 'Luge Run',
|
|
695
|
+
'headings': [],
|
|
696
|
+
'buttons': [],
|
|
697
|
+
'links': [],
|
|
698
|
+
'canvasCount': 1,
|
|
699
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': '', 'area': 420000}],
|
|
700
|
+
}
|
|
701
|
+
quality = namespace['evaluate_capture_quality']({
|
|
702
|
+
'ok': True,
|
|
703
|
+
'screenshots': [{'name': 'before.png', 'url': 'https://cdn.example.com/before.png'}],
|
|
704
|
+
'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/before.png'}],
|
|
705
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps(canvas_payload)],
|
|
706
|
+
}, '/games/luge-run')
|
|
707
|
+
assert quality['valid'] is True, quality
|
|
708
|
+
assert quality['details']['canvas_capture_ready'] is True
|
|
709
|
+
assert 'blank/near-blank' not in quality['reason']
|
|
710
|
+
return {'ok': True, 'valid': quality['valid']}
|
|
711
|
+
finally:
|
|
712
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
571
713
|
|
|
572
714
|
|
|
573
715
|
def load_util_with_fake(fake: FakeRiddle):
|
|
@@ -1992,6 +2134,7 @@ if __name__ == '__main__':
|
|
|
1992
2134
|
'implement_records_detection_when_changes_missing': run_implement_records_detection_when_changes_missing(),
|
|
1993
2135
|
'implement_ignores_tool_noise_when_detecting_changes': run_implement_ignores_tool_noise_when_detecting_changes(),
|
|
1994
2136
|
'verify_quality_ignores_proof_telemetry_console_text': run_verify_quality_ignores_proof_telemetry_console_text(),
|
|
2137
|
+
'recon_quality_accepts_canvas_first_routes': run_recon_quality_accepts_canvas_first_routes(),
|
|
1995
2138
|
'recon_then_author_request': run_recon_then_author_request(),
|
|
1996
2139
|
'recon_preserves_query_route': run_recon_preserves_query_route(),
|
|
1997
2140
|
'recon_route_literal_preference': run_recon_prefers_route_literals_over_import_paths(),
|