@riddledc/riddle-proof 0.5.41 → 0.5.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-4YCWZVBN.js → chunk-U4VNN2CT.js} +2 -1
- package/dist/{chunk-ZBM67XGB.js → chunk-VSLU6XNI.js} +1 -1
- package/dist/engine-harness.cjs +2 -1
- package/dist/engine-harness.js +2 -2
- package/dist/index.cjs +2 -1
- package/dist/index.js +2 -2
- package/dist/proof-run-core.cjs +2 -1
- package/dist/proof-run-core.js +1 -1
- package/dist/proof-run-engine.cjs +2 -1
- package/dist/proof-run-engine.js +1 -1
- package/package.json +1 -1
- package/runtime/lib/recon.py +63 -6
- package/runtime/lib/ship.py +3 -1
- package/runtime/lib/verify.py +84 -9
- package/runtime/tests/recon_verify_smoke.py +100 -1
|
@@ -371,7 +371,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
371
371
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
372
372
|
const status = String(visualDelta.status || "missing");
|
|
373
373
|
if (status === "unmeasured") {
|
|
374
|
-
|
|
374
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
375
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
375
376
|
}
|
|
376
377
|
if (status === "measured" && visualDelta.passed === false) {
|
|
377
378
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/engine-harness.cjs
CHANGED
|
@@ -379,7 +379,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
379
379
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
380
380
|
const status = String(visualDelta.status || "missing");
|
|
381
381
|
if (status === "unmeasured") {
|
|
382
|
-
|
|
382
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
383
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
383
384
|
}
|
|
384
385
|
if (status === "measured" && visualDelta.passed === false) {
|
|
385
386
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/engine-harness.js
CHANGED
|
@@ -2,11 +2,11 @@ import {
|
|
|
2
2
|
createDisabledRiddleProofAgentAdapter,
|
|
3
3
|
readRiddleProofRunStatus,
|
|
4
4
|
runRiddleProofEngineHarness
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-VSLU6XNI.js";
|
|
6
6
|
import "./chunk-RI25RGQP.js";
|
|
7
7
|
import "./chunk-7S7O3NKF.js";
|
|
8
8
|
import "./chunk-W7VTDN4T.js";
|
|
9
|
-
import "./chunk-
|
|
9
|
+
import "./chunk-U4VNN2CT.js";
|
|
10
10
|
export {
|
|
11
11
|
createDisabledRiddleProofAgentAdapter,
|
|
12
12
|
readRiddleProofRunStatus,
|
package/dist/index.cjs
CHANGED
|
@@ -379,7 +379,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
379
379
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
380
380
|
const status = String(visualDelta.status || "missing");
|
|
381
381
|
if (status === "unmeasured") {
|
|
382
|
-
|
|
382
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
383
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
383
384
|
}
|
|
384
385
|
if (status === "measured" && visualDelta.passed === false) {
|
|
385
386
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/index.js
CHANGED
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
createDisabledRiddleProofAgentAdapter,
|
|
26
26
|
readRiddleProofRunStatus,
|
|
27
27
|
runRiddleProofEngineHarness
|
|
28
|
-
} from "./chunk-
|
|
28
|
+
} from "./chunk-VSLU6XNI.js";
|
|
29
29
|
import {
|
|
30
30
|
RIDDLE_PROOF_CHECKPOINT_PACKET_VERSION,
|
|
31
31
|
RIDDLE_PROOF_CHECKPOINT_RESPONSE_VERSION,
|
|
@@ -67,7 +67,7 @@ import {
|
|
|
67
67
|
extractPlayabilityEvidence,
|
|
68
68
|
isRiddleProofPlayabilityMode
|
|
69
69
|
} from "./chunk-NSWT3VSV.js";
|
|
70
|
-
import "./chunk-
|
|
70
|
+
import "./chunk-U4VNN2CT.js";
|
|
71
71
|
export {
|
|
72
72
|
DEFAULT_DIAGNOSTIC_ARRAY_LIMIT,
|
|
73
73
|
DEFAULT_DIAGNOSTIC_HISTORY_LIMIT,
|
package/dist/proof-run-core.cjs
CHANGED
|
@@ -430,7 +430,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
430
430
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
431
431
|
const status = String(visualDelta.status || "missing");
|
|
432
432
|
if (status === "unmeasured") {
|
|
433
|
-
|
|
433
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
434
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
434
435
|
}
|
|
435
436
|
if (status === "measured" && visualDelta.passed === false) {
|
|
436
437
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/proof-run-core.js
CHANGED
|
@@ -412,7 +412,8 @@ function visualDeltaShipGateReason(state = {}) {
|
|
|
412
412
|
if (visualDelta.status === "measured" && visualDelta.passed === true) return null;
|
|
413
413
|
const status = String(visualDelta.status || "missing");
|
|
414
414
|
if (status === "unmeasured") {
|
|
415
|
-
|
|
415
|
+
const reason2 = String(visualDelta.reason || "").trim();
|
|
416
|
+
return reason2 ? `visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof: ${reason2}` : "visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof";
|
|
416
417
|
}
|
|
417
418
|
if (status === "measured" && visualDelta.passed === false) {
|
|
418
419
|
return "visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof";
|
package/dist/proof-run-engine.js
CHANGED
package/package.json
CHANGED
package/runtime/lib/recon.py
CHANGED
|
@@ -33,6 +33,7 @@ import subprocess as sp
|
|
|
33
33
|
MAX_RECON_ATTEMPTS = 4
|
|
34
34
|
MIN_BODY_TEXT_LENGTH = 50
|
|
35
35
|
MIN_INTERACTIVE_ELEMENTS = 1
|
|
36
|
+
MIN_CANVAS_AREA = 50000
|
|
36
37
|
HYDRATION_WAIT_MS = 1500
|
|
37
38
|
PAGE_STATE_PREFIX = 'RIDDLE_PROOF_STATE:'
|
|
38
39
|
PROOF_EVIDENCE_PREFIX = 'RIDDLE_PROOF_EVIDENCE:'
|
|
@@ -423,6 +424,45 @@ def has_enriched_page_state(page_state):
|
|
|
423
424
|
)
|
|
424
425
|
|
|
425
426
|
|
|
427
|
+
def numeric_value(value):
|
|
428
|
+
if isinstance(value, bool) or value is None:
|
|
429
|
+
return None
|
|
430
|
+
if isinstance(value, (int, float)):
|
|
431
|
+
return float(value)
|
|
432
|
+
if isinstance(value, str):
|
|
433
|
+
text = value.strip().rstrip('%').replace(',', '')
|
|
434
|
+
if not text:
|
|
435
|
+
return None
|
|
436
|
+
try:
|
|
437
|
+
return float(text)
|
|
438
|
+
except Exception:
|
|
439
|
+
return None
|
|
440
|
+
return None
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def canvas_capture_signal(page_state):
|
|
444
|
+
if not isinstance(page_state, dict):
|
|
445
|
+
return {
|
|
446
|
+
'canvas_ready': False,
|
|
447
|
+
'canvas_count': 0,
|
|
448
|
+
'large_canvas_area': 0,
|
|
449
|
+
}
|
|
450
|
+
large_canvas_area = 0
|
|
451
|
+
for item in list_value(page_state.get('largeVisibleElements')):
|
|
452
|
+
if not isinstance(item, dict) or item.get('tag') != 'canvas':
|
|
453
|
+
continue
|
|
454
|
+
area = numeric_value(item.get('area')) or 0
|
|
455
|
+
if area > large_canvas_area:
|
|
456
|
+
large_canvas_area = area
|
|
457
|
+
canvas_count = int(numeric_value(page_state.get('canvasCount')) or 0)
|
|
458
|
+
return {
|
|
459
|
+
'canvas_ready': bool(canvas_count > 0 and large_canvas_area >= MIN_CANVAS_AREA),
|
|
460
|
+
'canvas_count': canvas_count,
|
|
461
|
+
'large_canvas_area': int(large_canvas_area),
|
|
462
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
|
|
426
466
|
def normalize_observed_path(value):
|
|
427
467
|
raw = str(value or '').strip()
|
|
428
468
|
if not raw:
|
|
@@ -538,6 +578,9 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
538
578
|
'buttons': [],
|
|
539
579
|
'links': [],
|
|
540
580
|
'canvas_count': 0,
|
|
581
|
+
'canvas_capture_ready': False,
|
|
582
|
+
'large_canvas_area': 0,
|
|
583
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
541
584
|
'large_visible_elements': [],
|
|
542
585
|
'semantic_anchor_count': 0,
|
|
543
586
|
'capture_error_messages': [],
|
|
@@ -572,6 +615,12 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
572
615
|
'large_visible_elements': list_value(page_state.get('largeVisibleElements'))[:10],
|
|
573
616
|
'semantic_anchor_count': semantic_anchor_count(page_state),
|
|
574
617
|
})
|
|
618
|
+
canvas_signal = canvas_capture_signal(page_state)
|
|
619
|
+
details.update({
|
|
620
|
+
'canvas_capture_ready': canvas_signal['canvas_ready'],
|
|
621
|
+
'large_canvas_area': canvas_signal['large_canvas_area'],
|
|
622
|
+
'min_canvas_area': canvas_signal['min_canvas_area'],
|
|
623
|
+
})
|
|
575
624
|
else:
|
|
576
625
|
details.update({
|
|
577
626
|
'body_text_length': MIN_BODY_TEXT_LENGTH + 100,
|
|
@@ -596,11 +645,20 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
596
645
|
details['capture_error_messages'].append(str(proof_json.get('script_error'))[:500])
|
|
597
646
|
|
|
598
647
|
reasons = []
|
|
599
|
-
|
|
648
|
+
canvas_ready = bool(details.get('canvas_capture_ready'))
|
|
649
|
+
body_text_ready = details['body_text_length'] >= MIN_BODY_TEXT_LENGTH or canvas_ready
|
|
650
|
+
interactive_ready = details['interactive_elements'] >= MIN_INTERACTIVE_ELEMENTS or canvas_ready
|
|
651
|
+
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1 or canvas_ready
|
|
652
|
+
details['body_text_ready'] = body_text_ready
|
|
653
|
+
details['interactive_ready'] = interactive_ready
|
|
654
|
+
details['semantic_ready'] = semantic_ready
|
|
655
|
+
details['canvas_capture_override'] = canvas_ready
|
|
656
|
+
|
|
657
|
+
if not body_text_ready:
|
|
600
658
|
reasons.append(f'blank/near-blank page (text length: {details["body_text_length"]})')
|
|
601
|
-
if
|
|
659
|
+
if not interactive_ready:
|
|
602
660
|
reasons.append(f'not interactive enough ({details["interactive_elements"]} interactive elements)')
|
|
603
|
-
if has_enriched_page_state(page_state) and
|
|
661
|
+
if has_enriched_page_state(page_state) and not semantic_ready:
|
|
604
662
|
reasons.append('no visible semantic UI anchors in page capture')
|
|
605
663
|
if details['has_errors']:
|
|
606
664
|
reasons.append('page has console/runtime errors')
|
|
@@ -610,11 +668,10 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
610
668
|
raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
|
|
611
669
|
reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
|
|
612
670
|
|
|
613
|
-
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1
|
|
614
671
|
telemetry_ready = (
|
|
615
672
|
details['has_screenshot']
|
|
616
|
-
and
|
|
617
|
-
and
|
|
673
|
+
and body_text_ready
|
|
674
|
+
and interactive_ready
|
|
618
675
|
and semantic_ready
|
|
619
676
|
and not details['has_errors']
|
|
620
677
|
)
|
package/runtime/lib/ship.py
CHANGED
|
@@ -352,7 +352,9 @@ def visual_delta_ship_blocker(state):
|
|
|
352
352
|
return ''
|
|
353
353
|
status = str(visual_delta.get('status') or 'missing')
|
|
354
354
|
if status == 'unmeasured':
|
|
355
|
-
|
|
355
|
+
reason = str(visual_delta.get('reason') or '').strip()
|
|
356
|
+
detail = f': {reason}' if reason else ''
|
|
357
|
+
return 'visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof' + detail
|
|
356
358
|
if status == 'measured' and visual_delta.get('passed') is False:
|
|
357
359
|
return 'visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof'
|
|
358
360
|
reason = str(visual_delta.get('reason') or '').strip()
|
package/runtime/lib/verify.py
CHANGED
|
@@ -33,6 +33,7 @@ import subprocess as sp
|
|
|
33
33
|
|
|
34
34
|
MIN_BODY_TEXT_LENGTH = 50
|
|
35
35
|
MIN_INTERACTIVE_ELEMENTS = 1
|
|
36
|
+
MIN_CANVAS_AREA = 50000
|
|
36
37
|
HYDRATION_WAIT_MS = 1500
|
|
37
38
|
PAGE_STATE_PREFIX = 'RIDDLE_PROOF_STATE:'
|
|
38
39
|
PROOF_EVIDENCE_PREFIX = 'RIDDLE_PROOF_EVIDENCE:'
|
|
@@ -800,6 +801,8 @@ def extract_visual_delta(payload):
|
|
|
800
801
|
result = payload.get('result') if isinstance(payload, dict) else {}
|
|
801
802
|
proof_json = payload.get('_proof_json') if isinstance(payload, dict) else {}
|
|
802
803
|
proof_evidence = extract_proof_evidence(payload)
|
|
804
|
+
screenshot_url = extract_screenshot_url(payload)
|
|
805
|
+
artifact_summary = summarize_capture_artifacts(payload)
|
|
803
806
|
candidates = [
|
|
804
807
|
payload if isinstance(payload, dict) else {},
|
|
805
808
|
result if isinstance(result, dict) else {},
|
|
@@ -837,6 +840,23 @@ def extract_visual_delta(payload):
|
|
|
837
840
|
percent = (changed_pixels / total_pixels) * 100
|
|
838
841
|
|
|
839
842
|
if percent is None and changed_pixels is None:
|
|
843
|
+
has_artifacts = bool(
|
|
844
|
+
screenshot_url
|
|
845
|
+
or artifact_summary.get('outputs')
|
|
846
|
+
or artifact_summary.get('screenshots')
|
|
847
|
+
or artifact_summary.get('artifact_json')
|
|
848
|
+
)
|
|
849
|
+
reason = 'No measured before/after visual delta was found in proof evidence.'
|
|
850
|
+
if screenshot_url:
|
|
851
|
+
reason = (
|
|
852
|
+
'After screenshot artifact is present, but no measured before/after visual delta metrics were emitted; '
|
|
853
|
+
'the comparator did not run or did not publish change_percent/changed_pixels.'
|
|
854
|
+
)
|
|
855
|
+
elif has_artifacts:
|
|
856
|
+
reason = (
|
|
857
|
+
'Capture artifacts are present, but no measured before/after visual delta metrics were emitted; '
|
|
858
|
+
'the comparator did not run or did not publish change_percent/changed_pixels.'
|
|
859
|
+
)
|
|
840
860
|
return {
|
|
841
861
|
'status': 'unmeasured',
|
|
842
862
|
'passed': None,
|
|
@@ -845,7 +865,19 @@ def extract_visual_delta(payload):
|
|
|
845
865
|
'total_pixels': int(total_pixels) if total_pixels else None,
|
|
846
866
|
'min_change_percent': MIN_VISUAL_DELTA_PERCENT,
|
|
847
867
|
'min_changed_pixels': MIN_VISUAL_CHANGED_PIXELS,
|
|
848
|
-
'reason':
|
|
868
|
+
'reason': reason,
|
|
869
|
+
'diagnostic': {
|
|
870
|
+
'after_screenshot_present': bool(screenshot_url),
|
|
871
|
+
'proof_evidence_present': proof_evidence is not None,
|
|
872
|
+
'artifact_output_count': len(artifact_summary.get('outputs') or []),
|
|
873
|
+
'artifact_screenshot_count': len(artifact_summary.get('screenshots') or []),
|
|
874
|
+
'artifact_json': list(artifact_summary.get('artifact_json') or []),
|
|
875
|
+
'expected_metric_keys': {
|
|
876
|
+
'percent': sorted(VISUAL_DELTA_PERCENT_KEYS),
|
|
877
|
+
'changed_pixels': sorted(VISUAL_CHANGED_PIXEL_KEYS),
|
|
878
|
+
'total_pixels': sorted(VISUAL_TOTAL_PIXEL_KEYS),
|
|
879
|
+
},
|
|
880
|
+
},
|
|
849
881
|
}
|
|
850
882
|
|
|
851
883
|
percent_pass = percent is not None and percent >= MIN_VISUAL_DELTA_PERCENT
|
|
@@ -891,7 +923,8 @@ def visual_delta_blocker_for_mode(verification_mode, visual_delta):
|
|
|
891
923
|
status = str(visual_delta.get('status') or 'missing')
|
|
892
924
|
reason = str(visual_delta.get('reason') or '').strip()
|
|
893
925
|
if status == 'unmeasured':
|
|
894
|
-
|
|
926
|
+
detail = f' Reason: {reason}' if reason else ''
|
|
927
|
+
return 'visual_delta.status=unmeasured blocks ready_to_ship for visual/UI proof; capture a measured before/after visual delta or choose needs_richer_proof.' + detail
|
|
895
928
|
if status == 'measured' and isinstance(visual_delta, dict) and visual_delta.get('passed') is False:
|
|
896
929
|
return 'visual_delta.status=measured but visual_delta.passed=false blocks ready_to_ship for visual/UI proof; the measured change did not clear the threshold.'
|
|
897
930
|
if reason:
|
|
@@ -926,6 +959,29 @@ def has_enriched_page_state(page_state):
|
|
|
926
959
|
)
|
|
927
960
|
|
|
928
961
|
|
|
962
|
+
def canvas_capture_signal(page_state):
|
|
963
|
+
if not isinstance(page_state, dict):
|
|
964
|
+
return {
|
|
965
|
+
'canvas_ready': False,
|
|
966
|
+
'canvas_count': 0,
|
|
967
|
+
'large_canvas_area': 0,
|
|
968
|
+
}
|
|
969
|
+
large_canvas_area = 0
|
|
970
|
+
for item in list_value(page_state.get('largeVisibleElements')):
|
|
971
|
+
if not isinstance(item, dict) or item.get('tag') != 'canvas':
|
|
972
|
+
continue
|
|
973
|
+
area = metric_number(item.get('area')) or 0
|
|
974
|
+
if area > large_canvas_area:
|
|
975
|
+
large_canvas_area = area
|
|
976
|
+
canvas_count = int(metric_number(page_state.get('canvasCount')) or 0)
|
|
977
|
+
return {
|
|
978
|
+
'canvas_ready': bool(canvas_count > 0 and large_canvas_area >= MIN_CANVAS_AREA),
|
|
979
|
+
'canvas_count': canvas_count,
|
|
980
|
+
'large_canvas_area': int(large_canvas_area),
|
|
981
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
|
|
929
985
|
def normalize_observed_path(value):
|
|
930
986
|
raw = str(value or '').strip()
|
|
931
987
|
if not raw:
|
|
@@ -1135,6 +1191,7 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1135
1191
|
mode = normalized_verification_mode(verification_mode)
|
|
1136
1192
|
supporting = collect_supporting_artifacts(payload)
|
|
1137
1193
|
structured_ready = bool(supporting.get('has_structured_payload'))
|
|
1194
|
+
playability_ready = mode in PLAYABILITY_MODES and bool(supporting.get('playability_ready'))
|
|
1138
1195
|
screenshot_required = screenshot_required_for_mode(mode)
|
|
1139
1196
|
details = {
|
|
1140
1197
|
'verification_mode': mode,
|
|
@@ -1143,6 +1200,10 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1143
1200
|
'screenshot_required': screenshot_required,
|
|
1144
1201
|
'structured_evidence_present': structured_ready,
|
|
1145
1202
|
'proof_evidence_present': bool(supporting.get('proof_evidence_present')),
|
|
1203
|
+
'playability_ready': playability_ready,
|
|
1204
|
+
'canvas_capture_ready': False,
|
|
1205
|
+
'large_canvas_area': 0,
|
|
1206
|
+
'min_canvas_area': MIN_CANVAS_AREA,
|
|
1146
1207
|
'proof_evidence_sample': supporting.get('proof_evidence_sample', ''),
|
|
1147
1208
|
'body_text_length': 0,
|
|
1148
1209
|
'interactive_elements': 0,
|
|
@@ -1183,6 +1244,12 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1183
1244
|
'large_visible_elements': list_value(page_state.get('largeVisibleElements'))[:10],
|
|
1184
1245
|
'semantic_anchor_count': semantic_anchor_count(page_state),
|
|
1185
1246
|
})
|
|
1247
|
+
canvas_signal = canvas_capture_signal(page_state)
|
|
1248
|
+
details.update({
|
|
1249
|
+
'canvas_capture_ready': canvas_signal['canvas_ready'],
|
|
1250
|
+
'large_canvas_area': canvas_signal['large_canvas_area'],
|
|
1251
|
+
'min_canvas_area': canvas_signal['min_canvas_area'],
|
|
1252
|
+
})
|
|
1186
1253
|
elif screenshot_url:
|
|
1187
1254
|
details.update({
|
|
1188
1255
|
'body_text_length': MIN_BODY_TEXT_LENGTH + 100,
|
|
@@ -1220,11 +1287,20 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1220
1287
|
reasons.append('no screenshot or structured proof evidence in capture')
|
|
1221
1288
|
|
|
1222
1289
|
should_enforce_visual_readiness = screenshot_required or (details['has_screenshot'] and not structured_ready)
|
|
1223
|
-
|
|
1290
|
+
canvas_ready = bool(details.get('canvas_capture_ready'))
|
|
1291
|
+
body_text_ready = details['body_text_length'] >= MIN_BODY_TEXT_LENGTH or canvas_ready or playability_ready
|
|
1292
|
+
interactive_ready = details['interactive_elements'] >= MIN_INTERACTIVE_ELEMENTS or canvas_ready or playability_ready
|
|
1293
|
+
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1 or canvas_ready or playability_ready
|
|
1294
|
+
details['body_text_ready'] = body_text_ready
|
|
1295
|
+
details['interactive_ready'] = interactive_ready
|
|
1296
|
+
details['semantic_ready'] = semantic_ready
|
|
1297
|
+
details['canvas_or_playability_override'] = bool(should_enforce_visual_readiness and (canvas_ready or playability_ready))
|
|
1298
|
+
|
|
1299
|
+
if should_enforce_visual_readiness and not body_text_ready:
|
|
1224
1300
|
reasons.append(f'blank/near-blank page (text length: {details["body_text_length"]})')
|
|
1225
|
-
if should_enforce_visual_readiness and
|
|
1301
|
+
if should_enforce_visual_readiness and not interactive_ready:
|
|
1226
1302
|
reasons.append(f'not interactive enough ({details["interactive_elements"]} interactive elements)')
|
|
1227
|
-
if should_enforce_visual_readiness and has_enriched_page_state(page_state) and
|
|
1303
|
+
if should_enforce_visual_readiness and has_enriched_page_state(page_state) and not semantic_ready:
|
|
1228
1304
|
reasons.append('no visible semantic UI anchors in page capture')
|
|
1229
1305
|
if details['has_errors']:
|
|
1230
1306
|
reasons.append('page has console/runtime errors')
|
|
@@ -1234,15 +1310,14 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
1234
1310
|
raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
|
|
1235
1311
|
reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
|
|
1236
1312
|
|
|
1237
|
-
semantic_ready = (not has_enriched_page_state(page_state)) or details['semantic_anchor_count'] >= 1
|
|
1238
1313
|
visual_ready = (
|
|
1239
1314
|
details['has_screenshot']
|
|
1240
|
-
and
|
|
1241
|
-
and
|
|
1315
|
+
and body_text_ready
|
|
1316
|
+
and interactive_ready
|
|
1242
1317
|
and semantic_ready
|
|
1243
1318
|
and not details['has_errors']
|
|
1244
1319
|
)
|
|
1245
|
-
telemetry_ready = (visual_ready or structured_ready) and not details['has_errors']
|
|
1320
|
+
telemetry_ready = (visual_ready or structured_ready or playability_ready) and not details['has_errors']
|
|
1246
1321
|
|
|
1247
1322
|
return {
|
|
1248
1323
|
'valid': len(reasons) == 0 and telemetry_ready,
|
|
@@ -566,8 +566,106 @@ def run_verify_quality_ignores_proof_telemetry_console_text():
|
|
|
566
566
|
})
|
|
567
567
|
assert unmeasured_delta['status'] == 'unmeasured'
|
|
568
568
|
assert unmeasured_delta['passed'] is None
|
|
569
|
+
assert unmeasured_delta['diagnostic']['after_screenshot_present'] is True
|
|
570
|
+
assert 'After screenshot artifact is present' in unmeasured_delta['reason']
|
|
571
|
+
|
|
572
|
+
canvas_payload = {
|
|
573
|
+
'bodyTextLength': 7,
|
|
574
|
+
'visibleTextSample': 'Luge',
|
|
575
|
+
'interactiveElements': 1,
|
|
576
|
+
'visibleInteractiveElements': 1,
|
|
577
|
+
'pathname': '/games/luge-run',
|
|
578
|
+
'title': 'Luge Run',
|
|
579
|
+
'headings': [],
|
|
580
|
+
'buttons': [],
|
|
581
|
+
'links': [],
|
|
582
|
+
'canvasCount': 1,
|
|
583
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': '', 'area': 420000}],
|
|
584
|
+
}
|
|
585
|
+
canvas_quality = namespace['evaluate_capture_quality']({
|
|
586
|
+
'ok': True,
|
|
587
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
588
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
589
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps(canvas_payload)],
|
|
590
|
+
}, '/games/luge-run', 'visual')
|
|
591
|
+
assert canvas_quality['valid'] is True, canvas_quality
|
|
592
|
+
assert canvas_quality['details']['canvas_capture_ready'] is True
|
|
593
|
+
assert canvas_quality['details']['body_text_ready'] is True
|
|
594
|
+
assert 'blank/near-blank' not in canvas_quality['reason']
|
|
595
|
+
|
|
596
|
+
playability_payload = {
|
|
597
|
+
**canvas_payload,
|
|
598
|
+
'interactiveElements': 0,
|
|
599
|
+
'visibleInteractiveElements': 0,
|
|
600
|
+
'largeVisibleElements': [],
|
|
601
|
+
}
|
|
602
|
+
playability_evidence = {
|
|
603
|
+
'input_events': [{'type': 'pointerdown'}],
|
|
604
|
+
'state_delta': {'changed': True, 'changed_keys': ['distance']},
|
|
605
|
+
'canvas_delta': {'changed_pixels': 18000},
|
|
606
|
+
'time_delta_ms': 1300,
|
|
607
|
+
}
|
|
608
|
+
playable_quality = namespace['evaluate_capture_quality']({
|
|
609
|
+
'ok': True,
|
|
610
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
611
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
612
|
+
'console': [
|
|
613
|
+
'RIDDLE_PROOF_STATE:' + json.dumps(playability_payload),
|
|
614
|
+
'RIDDLE_PROOF_EVIDENCE:' + json.dumps({'playability': playability_evidence}),
|
|
615
|
+
],
|
|
616
|
+
}, '/games/luge-run', 'playable')
|
|
617
|
+
assert playable_quality['valid'] is True, playable_quality
|
|
618
|
+
assert playable_quality['details']['playability_ready'] is True
|
|
619
|
+
assert playable_quality['details']['interactive_ready'] is True
|
|
569
620
|
|
|
570
|
-
return {
|
|
621
|
+
return {
|
|
622
|
+
'ok': True,
|
|
623
|
+
'telemetry_valid': quality['valid'],
|
|
624
|
+
'weak_delta_passed': weak_delta['passed'],
|
|
625
|
+
'canvas_valid': canvas_quality['valid'],
|
|
626
|
+
'playable_valid': playable_quality['valid'],
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def run_recon_quality_accepts_canvas_first_routes():
|
|
631
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-recon-canvas-quality-'))
|
|
632
|
+
state_path = tempdir / 'state.json'
|
|
633
|
+
try:
|
|
634
|
+
write_state(state_path, {
|
|
635
|
+
'after_worktree': str(tempdir),
|
|
636
|
+
'before_worktree': str(tempdir),
|
|
637
|
+
})
|
|
638
|
+
with temporary_env(RIDDLE_PROOF_STATE_FILE=str(state_path)):
|
|
639
|
+
sys.modules.pop('util', None)
|
|
640
|
+
source = RECON_PATH.read_text()
|
|
641
|
+
helpers_source = source.split('\ndef clean_next_cache', 1)[0]
|
|
642
|
+
namespace = {'__file__': str(RECON_PATH)}
|
|
643
|
+
exec(compile(helpers_source, str(RECON_PATH), 'exec'), namespace)
|
|
644
|
+
canvas_payload = {
|
|
645
|
+
'bodyTextLength': 4,
|
|
646
|
+
'visibleTextSample': 'Game',
|
|
647
|
+
'interactiveElements': 1,
|
|
648
|
+
'visibleInteractiveElements': 1,
|
|
649
|
+
'pathname': '/games/luge-run',
|
|
650
|
+
'title': 'Luge Run',
|
|
651
|
+
'headings': [],
|
|
652
|
+
'buttons': [],
|
|
653
|
+
'links': [],
|
|
654
|
+
'canvasCount': 1,
|
|
655
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': '', 'area': 420000}],
|
|
656
|
+
}
|
|
657
|
+
quality = namespace['evaluate_capture_quality']({
|
|
658
|
+
'ok': True,
|
|
659
|
+
'screenshots': [{'name': 'before.png', 'url': 'https://cdn.example.com/before.png'}],
|
|
660
|
+
'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/before.png'}],
|
|
661
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps(canvas_payload)],
|
|
662
|
+
}, '/games/luge-run')
|
|
663
|
+
assert quality['valid'] is True, quality
|
|
664
|
+
assert quality['details']['canvas_capture_ready'] is True
|
|
665
|
+
assert 'blank/near-blank' not in quality['reason']
|
|
666
|
+
return {'ok': True, 'valid': quality['valid']}
|
|
667
|
+
finally:
|
|
668
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
571
669
|
|
|
572
670
|
|
|
573
671
|
def load_util_with_fake(fake: FakeRiddle):
|
|
@@ -1992,6 +2090,7 @@ if __name__ == '__main__':
|
|
|
1992
2090
|
'implement_records_detection_when_changes_missing': run_implement_records_detection_when_changes_missing(),
|
|
1993
2091
|
'implement_ignores_tool_noise_when_detecting_changes': run_implement_ignores_tool_noise_when_detecting_changes(),
|
|
1994
2092
|
'verify_quality_ignores_proof_telemetry_console_text': run_verify_quality_ignores_proof_telemetry_console_text(),
|
|
2093
|
+
'recon_quality_accepts_canvas_first_routes': run_recon_quality_accepts_canvas_first_routes(),
|
|
1995
2094
|
'recon_then_author_request': run_recon_then_author_request(),
|
|
1996
2095
|
'recon_preserves_query_route': run_recon_preserves_query_route(),
|
|
1997
2096
|
'recon_route_literal_preference': run_recon_prefers_route_literals_over_import_paths(),
|