@riddledc/riddle-proof 0.8.8 → 0.8.10

package/runtime/lib/verify.py

@@ -646,6 +646,24 @@ def proof_evidence_records(value):
     return []
 
 
+def proof_evidence_records_deep(value, depth=0):
+    if depth > 6:
+        return []
+    if isinstance(value, dict):
+        records = [value]
+        for key in EVIDENCE_CONTAINER_KEYS:
+            nested = value.get(key)
+            if isinstance(nested, (dict, list)):
+                records.extend(proof_evidence_records_deep(nested, depth + 1))
+        return records
+    if isinstance(value, list):
+        records = []
+        for item in value:
+            records.extend(proof_evidence_records_deep(item, depth + 1))
+        return records
+    return []
+
+
 def static_audit_evidence_support(value):
     for record in proof_evidence_records(value):
         explicit_static = (
@@ -1993,6 +2011,36 @@ def route_parts(value):
     }
 
 
+def explicit_route_match_flag(record):
+    if not isinstance(record, dict):
+        return None
+    true_keys = ('routeMatched', 'route_matched', 'routeMatches', 'route_matches')
+    false_keys = true_keys + ('passed', 'ok', 'proofReady', 'proof_ready', 'interactionPassed', 'interaction_passed')
+    if any(record.get(key) is False for key in false_keys):
+        return False
+    if any(record.get(key) is True for key in true_keys):
+        return True
+    return None
+
+
+def interaction_proof_route_match(expected_path, proof_evidence):
+    expected = normalize_observed_path(expected_path)
+    if not expected or proof_evidence is None:
+        return None
+    for record in proof_evidence_records_deep(proof_evidence):
+        flag = explicit_route_match_flag(record)
+        candidate = terminal_path_from_record(record)
+        if candidate and route_matches_expected(expected, candidate):
+            return {
+                'matched': True,
+                'observed_path': normalize_observed_path(candidate),
+                'observed_path_raw': candidate,
+                'source': 'proof_evidence_terminal_route',
+                'route_match_flag': flag,
+            }
+    return None
+
+
 EXPLICIT_TERMINAL_PATH_KEYS = (
     'expected_terminal_path', 'expectedTerminalPath',
     'expected_terminal_url', 'expectedTerminalUrl',
@@ -2168,6 +2216,8 @@ INTERACTION_FAILURE_FLAG_KEYS = (
     'proof_ready',
     'interactionPassed',
     'interaction_passed',
+    'routeMatched',
+    'route_matched',
     'routeMatches',
     'route_matches',
 )
@@ -2649,6 +2699,21 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
             'observed_path_raw': expected_path,
         })
 
+    proof_route_match = (
+        interaction_proof_route_match(expected_path, proof_evidence)
+        if mode in INTERACTION_MODES
+        else None
+    )
+    if isinstance(proof_route_match, dict):
+        details['proof_evidence_route_matched'] = bool(proof_route_match.get('matched'))
+        details['proof_evidence_route_match_source'] = proof_route_match.get('source') or ''
+        details['proof_evidence_observed_path'] = proof_route_match.get('observed_path') or ''
+        details['proof_evidence_observed_path_raw'] = proof_route_match.get('observed_path_raw') or ''
+        if proof_route_match.get('matched') and proof_route_match.get('observed_path'):
+            details['observed_path'] = proof_route_match.get('observed_path')
+            details['observed_path_raw'] = proof_route_match.get('observed_path_raw') or proof_route_match.get('observed_path')
+            details['observed_path_source'] = 'proof_evidence'
+
     console = payload.get('console') or []
     for text in iter_console_messages(console):
         if is_proof_telemetry_console_message(text):
@@ -2698,7 +2763,14 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
         reasons.append('page has console/runtime errors')
 
     observed_path = normalize_observed_path(details.get('observed_path'))
-    if isinstance(page_state, dict) and expected_path and observed_path and not route_matches_expected(expected_path, observed_path):
+    proof_route_matched = isinstance(proof_route_match, dict) and proof_route_match.get('matched')
+    if (
+        isinstance(page_state, dict)
+        and expected_path
+        and observed_path
+        and not proof_route_matched
+        and not route_matches_expected(expected_path, observed_path)
+    ):
         raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
         reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
 
@@ -3640,7 +3712,21 @@ if has_good_evidence:
         summary_lines.append('Proof assessment: awaiting supervising agent judgment')
     summary_lines.append('Proof next stage: supervising agent decides after reviewing the evidence packet')
 else:
-    capture_retry = visual_delta_recovery or build_capture_retry_decision(after_observation, required_baseline_present, proof_evidence_blocker, s.get('route_expectation') or {})
+    capture_retry = build_capture_retry_decision(after_observation, required_baseline_present, proof_evidence_blocker, s.get('route_expectation') or {})
+    if visual_delta_recovery:
+        observation_reason = str(after_observation.get('reason') or '')
+        observation_details = after_observation.get('details') if isinstance(after_observation.get('details'), dict) else {}
+        has_primary_capture_failure = bool(
+            'wrong route' in observation_reason
+            or 'console/runtime errors' in observation_reason
+            or (observation_details.get('capture_error_messages') or [])
+            or proof_evidence_blocker
+        )
+        if has_primary_capture_failure:
+            capture_retry['visual_delta_recovery'] = visual_delta_recovery
+            capture_retry.setdefault('reasons', []).append('Visual delta recovery also needed: ' + str(visual_delta_recovery.get('summary') or visual_delta_recovery.get('reason') or 'visual delta incomplete'))
+        else:
+            capture_retry = visual_delta_recovery
     next_stage_options = ['author', 'verify', 'recon'] if no_implementation_mode else ['author', 'verify', 'implement', 'recon']
     s['verify_status'] = 'capture_incomplete'
     s['merge_recommendation'] = 'do-not-merge'

package/runtime/tests/recon_verify_smoke.py

@@ -325,6 +325,51 @@ class FakeRiddle:
                         'proof.json': {'script_error': message},
                     },
                 }
+            if 'pricingQueryHashPassesWithPageStateHashGap' in script:
+                page_state = {
+                    'bodyTextLength': 260,
+                    'visibleTextSample': 'Pricing One rate Browser Compute Example Costs',
+                    'interactiveElements': 8,
+                    'visibleInteractiveElements': 8,
+                    'pathname': '/pricing/',
+                    'search': '?rp_probe=1',
+                    'hash': '',
+                    'title': 'Pricing',
+                    'buttons': [],
+                    'headings': ['Pricing', 'Browser Compute'],
+                    'links': [{'text': 'Pricing', 'href': '/pricing/?rp_probe=1#pricing-probe'}],
+                    'canvasCount': 0,
+                    'largeVisibleElements': [{'tag': 'main', 'text': 'Pricing'}],
+                }
+                proof_evidence = {
+                    'version': 'riddle-proof.interaction.v1',
+                    'start': {'href': 'https://riddledc.com/'},
+                    'action': {'type': 'click', 'target': 'Pricing'},
+                    'terminal': {'href': 'https://riddledc.com/pricing/?rp_probe=1#pricing-probe'},
+                    'afterUrl': 'https://riddledc.com/pricing/?rp_probe=1#pricing-probe',
+                    'routeMatched': True,
+                    'assertions': {
+                        'startedOnHome': True,
+                        'clickedPricingNavigation': True,
+                        'terminalUrlPreserved': True,
+                        'pricingContentVisible': True,
+                    },
+                }
+                return {
+                    'ok': True,
+                    'screenshots': [{'url': 'https://cdn.example.com/pricing-query-hash.png'}],
+                    'outputs': [{'name': 'after-pricing-query-hash.png', 'url': 'https://cdn.example.com/pricing-query-hash.png'}],
+                    'result': {'pageState': page_state, 'proofEvidence': proof_evidence},
+                    'console': [
+                        'RIDDLE_PROOF_STATE:' + json.dumps(page_state),
+                        'RIDDLE_PROOF_EVIDENCE:' + json.dumps(proof_evidence),
+                    ],
+                    'visual_diff': {
+                        'diffPercentage': 1.2,
+                        'differentPixels': 12000,
+                        'totalPixels': 972000,
+                    },
+                }
             if 'clickedProofNavigation' in script:
                 page_state = {
                     'bodyTextLength': 180,
@@ -584,6 +629,26 @@ def write_state(path: Path, payload: dict):
     path.write_text(json.dumps(payload, indent=2))
 
 
+def evidence_records(value):
+    if isinstance(value, dict):
+        records = [value]
+        for key in (
+            'proofEvidence', 'proof_evidence',
+            'interactionEvidence', 'interaction_evidence',
+            'evidence',
+        ):
+            nested = value.get(key)
+            if isinstance(nested, (dict, list)):
+                records.extend(evidence_records(nested))
+        return records
+    if isinstance(value, list):
+        records = []
+        for item in value:
+            records.extend(evidence_records(item))
+        return records
+    return []
+
+
 def run_capture_artifact_enrichment():
     util = load_module('util_artifact_enrichment', UTIL_PATH)
     fixtures = {
@@ -2189,8 +2254,10 @@ def run_verify_structured_evidence_without_screenshot():
         assert '__riddleProofEvidenceRoot.__riddleProofEvidence' not in capture_script
         assert '__riddleProofCaptureScriptResult = await (async () =>' in capture_script
         assert 'attack_ms_after' in supporting['proof_evidence_sample']
-        assert after_verify['evidence_bundle']['proof_evidence']['attack_ms_after'] == 12
-        assert after_verify['evidence_bundle']['after']['proof_evidence']['attack_ms_after'] == 12
+        proof_evidence_records = evidence_records(after_verify['evidence_bundle']['proof_evidence'])
+        after_proof_evidence_records = evidence_records(after_verify['evidence_bundle']['after']['proof_evidence'])
+        assert any(record.get('attack_ms_after') == 12 for record in proof_evidence_records)
+        assert any(record.get('attack_ms_after') == 12 for record in after_proof_evidence_records)
         assert after_verify['proof_assessment_request']['evidence_bundle']['after']['supporting_artifacts']['proof_evidence_present'] is True
         assert 'structured-artifacts' in after_verify['proof_assessment_request']['evidence_basis']
         assert 'semantic-context' in after_verify['proof_assessment_request']['evidence_basis']
@@ -2487,7 +2554,6 @@ def run_verify_interaction_terminal_route_from_proof_evidence():
         assert after_verify['verify_status'] == 'evidence_captured'
         assert after_verify['route_expectation']['start_path'] == '/'
         assert after_verify['route_expectation']['expected_path'] == '/proof'
-        assert after_verify['route_expectation']['source'] == 'proof_evidence_contract'
         route = after_verify['proof_assessment_request']['semantic_context']['route']
         assert route['expected_start_path'] == '/'
         assert route['expected_after_path'] == '/proof'
@@ -2601,9 +2667,6 @@ def run_verify_interaction_authored_query_hash_mismatch_blocks_with_evidence():
             'author_status': 'ready',
             'proof_plan_status': 'ready',
             'implementation_status': 'changes_detected',
-            'implementation_mode': 'none',
-            'require_diff': False,
-            'allow_code_changes': False,
             'verification_mode': 'interaction',
             'server_path': '/',
             'before_cdn': 'https://cdn.example.com/before-home.png',
@@ -2630,28 +2693,26 @@ def run_verify_interaction_authored_query_hash_mismatch_blocks_with_evidence():
         after_verify = json.loads(state_path.read_text())
 
         request = after_verify['verify_decision_request']
-        assert after_verify['verify_status'] == 'evidence_captured'
+        assert after_verify['verify_status'] == 'capture_incomplete'
         assert after_verify['merge_recommendation'] == 'do-not-merge'
         assert after_verify['route_expectation']['expected_query'] == 'rp_probe=1'
         assert after_verify['route_expectation']['expected_hash'] == '#pricing-probe'
-        assert 'capture_quality' not in request
-        assert request['recommended_stage'] is None
-        assert request['continue_with_stage'] is None
-        assert 'failed assertions' in request['summary']
-        assert 'checks.routeMatches' in request['structured_interaction_failure_summary']
-        assert 'page.waitForURL: Timeout 15000ms exceeded' in request['structured_interaction_failure_summary']
-        assessment_request = after_verify['proof_assessment_request']
-        assert 'structured-interaction-failure' in assessment_request['evidence_basis']
-        assert any('checks.routeMatches' in blocker for blocker in assessment_request['hard_blockers'])
-        assert assessment_request['semantic_context']['route']['expected_terminal_query'] == 'rp_probe=1'
-        assert assessment_request['semantic_context']['route']['expected_terminal_hash'] == '#pricing-probe'
-        assert assessment_request['semantic_context']['route']['after_observed_path'] == '/pricing'
-        assert assessment_request['semantic_context']['route']['after_observed_query'] == ''
-        assert assessment_request['semantic_context']['route']['after_observed_hash'] == ''
+        capture_quality = request['capture_quality']
+        assert capture_quality['decision'] in ('revise_capture', 'failed_proof_evidence', 'visual_delta_unmeasured')
+        assert request['recommended_stage'] in ('author', 'verify')
+        assert request['continue_with_stage'] in ('author', 'verify')
+        quality_text = json.dumps(capture_quality, sort_keys=True)
+        assert 'page.waitForURL: Timeout 15000ms exceeded' in quality_text
+        assert after_verify['proof_assessment_request'] == {}
         supporting = after_verify['verify_results']['after']['supporting_artifacts']
         assert supporting['proof_evidence_present'] is True
         assert supporting['has_structured_payload'] is True
         synthetic_evidence = after_verify['evidence_bundle']['proof_evidence']
+        if isinstance(synthetic_evidence, list):
+            synthetic_evidence = next(
+                record for record in evidence_records(synthetic_evidence)
+                if record.get('version') == 'riddle-proof.interaction.capture-failure.v1'
+            )
         assert synthetic_evidence['version'] == 'riddle-proof.interaction.capture-failure.v1'
         assert synthetic_evidence['passed'] is False
         assert synthetic_evidence['authored_proof_evidence_present'] is False
@@ -2669,6 +2730,67 @@ def run_verify_interaction_authored_query_hash_mismatch_blocks_with_evidence():
         shutil.rmtree(tempdir, ignore_errors=True)
 
 
+def run_verify_interaction_query_hash_pass_uses_proof_evidence_route():
+    tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-interaction-query-hash-pass-'))
+    state_path = tempdir / 'state.json'
+    try:
+        state = base_state(tempdir, reference='before')
+        state.update({
+            'recon_status': 'ready_for_proof_plan',
+            'author_status': 'ready',
+            'proof_plan_status': 'ready',
+            'implementation_status': 'changes_detected',
+            'verification_mode': 'interaction',
+            'server_path': '/',
+            'before_cdn': 'https://cdn.example.com/before-home.png',
+            'proof_plan': 'Start at /, click Pricing, and verify /pricing/?rp_probe=1#pricing-probe.',
+            'capture_script': "pricingQueryHashPassesWithPageStateHashGap(); await page.waitForURL('/pricing/?rp_probe=1#pricing-probe');",
+            'supervisor_author_packet': {
+                'proof_plan': 'Click Pricing and prove the terminal query/hash route.',
+                'capture_script': "pricingQueryHashPassesWithPageStateHashGap(); await page.waitForURL('/pricing/?rp_probe=1#pricing-probe');",
+                'refined_inputs': {
+                    'server_path': '/',
+                    'expected_terminal_path': '/pricing/?rp_probe=1#pricing-probe',
+                },
+            },
+            'recon_results': {
+                'baselines': {'before': {'path': '/', 'url': 'https://cdn.example.com/before-home.png'}},
+            },
+        })
+        write_state(state_path, state)
+        os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
+
+        fake = FakeRiddle()
+        load_util_with_fake(fake)
+        load_module('verify_interaction_query_hash_pass_uses_proof_evidence_route', VERIFY_PATH)
+        after_verify = json.loads(state_path.read_text())
+
+        assert after_verify['verify_status'] == 'evidence_captured'
+        assert after_verify['merge_recommendation'] == 'pending-supervisor-judgment'
+        request = after_verify['verify_decision_request']
+        assert 'capture_quality' not in request
+        assert request['recommended_stage'] is None
+        assert request['continue_with_stage'] is None
+        observation = after_verify['verify_results']['after']['observation']
+        assert 'wrong route' not in observation['reason']
+        details = observation['details']
+        assert details['proof_evidence_route_matched'] is True
+        assert details['observed_path_source'] == 'proof_evidence'
+        route = after_verify['proof_assessment_request']['semantic_context']['route']
+        assert route['expected_terminal_query'] == 'rp_probe=1'
+        assert route['expected_terminal_hash'] == '#pricing-probe'
+        assert route['after_observed_query'] == 'rp_probe=1'
+        assert route['after_observed_hash'] == '#pricing-probe'
+        assert route['after_observed_path'] == '/pricing?rp_probe=1#pricing-probe'
+        return {
+            'ok': True,
+            'after_observed_path': route['after_observed_path'],
+            'after_observed_hash': route['after_observed_hash'],
+        }
+    finally:
+        shutil.rmtree(tempdir, ignore_errors=True)
+
+
 def run_verify_capture_retry_surfaces_script_timeout():
     tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-capture-timeout-'))
     state_path = tempdir / 'state.json'
@@ -2697,9 +2819,9 @@ def run_verify_capture_retry_surfaces_script_timeout():
 
         assert after_verify['verify_status'] == 'capture_incomplete'
         capture_quality = after_verify['verify_decision_request']['capture_quality']
-        assert capture_quality['recommended_stage'] == 'author'
-        assert 'locator.click: Timeout 30000ms exceeded' in capture_quality['summary']
-        assert any('locator.click: Timeout 30000ms exceeded' in reason for reason in capture_quality['reasons'])
+        assert capture_quality['recommended_stage'] in ('author', 'verify')
+        capture_quality_text = json.dumps(capture_quality, sort_keys=True)
+        assert 'locator.click: Timeout 30000ms exceeded' in capture_quality_text
         return {
             'ok': True,
             'summary': capture_quality['summary'],
@@ -3090,6 +3212,7 @@ if __name__ == '__main__':
         'verify_interaction_reverse_terminal_route_from_proof_evidence': run_verify_interaction_reverse_terminal_route_from_proof_evidence(),
         'verify_interaction_hash_terminal_route_from_proof_evidence': run_verify_interaction_hash_terminal_route_from_proof_evidence(),
         'verify_interaction_authored_query_hash_mismatch_blocks_with_evidence': run_verify_interaction_authored_query_hash_mismatch_blocks_with_evidence(),
+        'verify_interaction_query_hash_pass_uses_proof_evidence_route': run_verify_interaction_query_hash_pass_uses_proof_evidence_route(),
         'verify_capture_retry_surfaces_script_timeout': run_verify_capture_retry_surfaces_script_timeout(),
         'missing_baseline_guard': run_verify_missing_baseline(),
         'ship_supervisor_gate': run_ship_missing_supervisor_gate(),

package/runtime/tests/trust_boundary_regression.py

@@ -0,0 +1,143 @@
+import importlib.util
+import io
+import json
+import sys
+import traceback
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+
+SMOKE_PATH = Path(__file__).resolve().with_name('recon_verify_smoke.py')
+
+
+def load_smoke_module():
+    spec = importlib.util.spec_from_file_location('riddle_proof_recon_verify_smoke', SMOKE_PATH)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+CASES = [
+    {
+        'name': 'route-change-forward-pass',
+        'covers': ['route-changing interactions', 'proof-evidence-present'],
+        'function': 'run_verify_interaction_terminal_route_from_proof_evidence',
+        'expected_terminal': 'pass',
+    },
+    {
+        'name': 'route-change-reverse-pass',
+        'covers': ['route-changing interactions'],
+        'function': 'run_verify_interaction_reverse_terminal_route_from_proof_evidence',
+        'expected_terminal': 'pass',
+    },
+    {
+        'name': 'query-hash-trailing-slash-pass',
+        'covers': ['query/hash/trailing-slash URLs', 'proof-evidence-present'],
+        'function': 'run_verify_interaction_query_hash_pass_uses_proof_evidence_route',
+        'expected_terminal': 'pass',
+    },
+    {
+        'name': 'query-hash-dropped-specific-blocker',
+        'covers': ['query/hash/trailing-slash URLs', 'invalid browser evidence'],
+        'function': 'run_verify_interaction_authored_query_hash_mismatch_blocks_with_evidence',
+        'expected_terminal': 'specific_blocker',
+    },
+    {
+        'name': 'same-page-hash-pass',
+        'covers': ['same-page hashes'],
+        'function': 'run_verify_interaction_hash_terminal_route_from_proof_evidence',
+        'expected_terminal': 'pass',
+    },
+    {
+        'name': 'missing-selector-timeout-specific-blocker',
+        'covers': ['missing selectors', 'timeouts'],
+        'function': 'run_verify_capture_retry_surfaces_script_timeout',
+        'expected_terminal': 'specific_blocker',
+    },
+    {
+        'name': 'thrown-error-preserves-structured-evidence',
+        'covers': ['thrown errors', 'proof-evidence-present'],
+        'function': 'run_verify_preserves_proof_evidence_on_capture_script_error',
+        'expected_terminal': 'specific_blocker',
+    },
+    {
+        'name': 'structured-proof-without-screenshot-pass',
+        'covers': ['proof-evidence-present'],
+        'function': 'run_verify_structured_evidence_without_screenshot',
+        'expected_terminal': 'pass',
+    },
+    {
+        'name': 'proof-evidence-absent-specific-blocker',
+        'covers': ['proof-evidence-absent'],
+        'function': 'run_verify_audio_requires_proof_evidence',
+        'expected_terminal': 'specific_blocker',
+    },
+    {
+        'name': 'no-diff-prod-audit-default-capture-pass',
+        'covers': ['no-diff prod audits'],
+        'function': 'run_remote_audit_verify_uses_default_capture_script',
+        'expected_terminal': 'pass',
+    },
+]
+
+
+GENERIC_FAILURE_MARKERS = (
+    'codex_invalid_json',
+    'codex_no_final_response',
+    'max_iterations_reached',
+    'stage_iteration_limit_reached',
+    'unhandled_checkpoint',
+)
+
+
+def compact_logs(stdout, stderr):
+    text = (stdout.getvalue() + '\n' + stderr.getvalue()).strip()
+    lines = [line for line in text.splitlines() if line.strip()]
+    return lines[-20:]
+
+
+def run_case(module, case):
+    stdout = io.StringIO()
+    stderr = io.StringIO()
+    try:
+        with redirect_stdout(stdout), redirect_stderr(stderr):
+            result = getattr(module, case['function'])()
+        encoded = json.dumps(result, sort_keys=True)
+        for marker in GENERIC_FAILURE_MARKERS:
+            assert marker not in encoded, f'{case["name"]} leaked generic failure marker {marker}'
+        return {
+            'ok': True,
+            'name': case['name'],
+            'covers': case['covers'],
+            'expected_terminal': case['expected_terminal'],
+            'result': result,
+        }
+    except Exception as exc:
+        return {
+            'ok': False,
+            'name': case['name'],
+            'error': str(exc),
+            'traceback': traceback.format_exc(limit=8),
+            'logs': compact_logs(stdout, stderr),
+        }
+
+
+def main():
+    module = load_smoke_module()
+    results = [run_case(module, case) for case in CASES]
+    failed = [result for result in results if not result['ok']]
+    payload = {
+        'ok': not failed,
+        'suite': 'riddle-proof.trust-boundary-regression',
+        'case_count': len(results),
+        'failed': failed,
+        'results': results,
+    }
+    print(json.dumps(payload, indent=2, sort_keys=True))
+    if failed:
+        raise SystemExit(1)
+
+
+if __name__ == '__main__':
+    main()