@riddledc/riddle-proof 0.5.55 → 0.5.57

package/runtime/lib/verify.py

@@ -7,8 +7,9 @@ while good captures produce a structured evidence packet that the supervising ag
 must assess before the wrapper routes back into author/implement/recon work or ship.
 """
 
-import json, os, sys, time
+import json, os, struct, sys, time, zlib
 from urllib.parse import parse_qsl, urlparse
+from urllib.request import Request, urlopen
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from util import (
     append_capture_diagnostic,
@@ -935,6 +936,204 @@ def visual_delta_baseline_candidate(state, results):
     return {'label': '', 'url': ''}
 
 
+def image_artifact_url(value):
+    text = str(value or '').strip()
+    if not text:
+        return False
+    parsed = urlparse(text)
+    if parsed.scheme != 'https':
+        return False
+    path = parsed.path.lower()
+    return path.endswith(IMAGE_EXTENSIONS)
+
+
+def fetch_url_bytes(url, timeout=20, max_bytes=25 * 1024 * 1024):
+    request = Request(
+        url,
+        headers={
+            'User-Agent': 'riddle-proof-visual-delta/1.0',
+            'Accept': 'image/png,image/jpeg,image/webp,image/*;q=0.8,*/*;q=0.1',
+        },
+    )
+    with urlopen(request, timeout=timeout) as response:
+        content_length = response.headers.get('Content-Length')
+        if content_length:
+            try:
+                if int(content_length) > max_bytes:
+                    raise ValueError('image artifact exceeds max fetch size')
+            except ValueError:
+                raise
+            except Exception:
+                pass
+        data = response.read(max_bytes + 1)
+    if len(data) > max_bytes:
+        raise ValueError('image artifact exceeds max fetch size')
+    return data
+
+
+def _png_paeth(a, b, c):
+    p = a + b - c
+    pa = abs(p - a)
+    pb = abs(p - b)
+    pc = abs(p - c)
+    if pa <= pb and pa <= pc:
+        return a
+    if pb <= pc:
+        return b
+    return c
+
+
+def decode_png_rgba(data):
+    if not isinstance(data, (bytes, bytearray)) or not data.startswith(b'\x89PNG\r\n\x1a\n'):
+        raise ValueError('unsupported image format; expected PNG artifact')
+    offset = 8
+    width = height = bit_depth = color_type = interlace = None
+    idat = []
+    while offset + 8 <= len(data):
+        length = struct.unpack('>I', data[offset:offset + 4])[0]
+        chunk_type = data[offset + 4:offset + 8]
+        chunk_data = data[offset + 8:offset + 8 + length]
+        offset += 12 + length
+        if chunk_type == b'IHDR':
+            width, height, bit_depth, color_type, _compression, _filter, interlace = struct.unpack('>IIBBBBB', chunk_data)
+        elif chunk_type == b'IDAT':
+            idat.append(chunk_data)
+        elif chunk_type == b'IEND':
+            break
+    if not width or not height or bit_depth != 8 or interlace != 0:
+        raise ValueError('unsupported PNG artifact; requires 8-bit non-interlaced PNG')
+    channels_by_type = {0: 1, 2: 3, 4: 2, 6: 4}
+    channels = channels_by_type.get(color_type)
+    if not channels:
+        raise ValueError('unsupported PNG color type')
+    raw = zlib.decompress(b''.join(idat))
+    stride = width * channels
+    rows = []
+    pos = 0
+    previous = bytearray(stride)
+    for _row in range(height):
+        if pos >= len(raw):
+            raise ValueError('truncated PNG data')
+        filter_type = raw[pos]
+        pos += 1
+        scanline = bytearray(raw[pos:pos + stride])
+        pos += stride
+        for index in range(stride):
+            left = scanline[index - channels] if index >= channels else 0
+            up = previous[index]
+            up_left = previous[index - channels] if index >= channels else 0
+            if filter_type == 1:
+                scanline[index] = (scanline[index] + left) & 0xff
+            elif filter_type == 2:
+                scanline[index] = (scanline[index] + up) & 0xff
+            elif filter_type == 3:
+                scanline[index] = (scanline[index] + ((left + up) // 2)) & 0xff
+            elif filter_type == 4:
+                scanline[index] = (scanline[index] + _png_paeth(left, up, up_left)) & 0xff
+            elif filter_type != 0:
+                raise ValueError('unsupported PNG filter')
+        rows.append(bytes(scanline))
+        previous = scanline
+    rgba = bytearray(width * height * 4)
+    out = 0
+    for row in rows:
+        for index in range(0, len(row), channels):
+            if color_type == 0:
+                gray = row[index]
+                rgba[out:out + 4] = bytes((gray, gray, gray, 255))
+            elif color_type == 2:
+                rgba[out:out + 4] = bytes((row[index], row[index + 1], row[index + 2], 255))
+            elif color_type == 4:
+                gray = row[index]
+                rgba[out:out + 4] = bytes((gray, gray, gray, row[index + 1]))
+            else:
+                rgba[out:out + 4] = bytes((row[index], row[index + 1], row[index + 2], row[index + 3]))
+            out += 4
+    return {'width': width, 'height': height, 'rgba': bytes(rgba)}
+
+
+def compare_rgba_images(before_image, after_image, threshold=10):
+    before_width = int(before_image.get('width') or 0)
+    before_height = int(before_image.get('height') or 0)
+    after_width = int(after_image.get('width') or 0)
+    after_height = int(after_image.get('height') or 0)
+    before_rgba = before_image.get('rgba') or b''
+    after_rgba = after_image.get('rgba') or b''
+    overlap_width = min(before_width, after_width)
+    overlap_height = min(before_height, after_height)
+    total_pixels = max(before_width * before_height, after_width * after_height)
+    changed_pixels = total_pixels - (overlap_width * overlap_height)
+    for y in range(overlap_height):
+        before_row = y * before_width * 4
+        after_row = y * after_width * 4
+        for x in range(overlap_width):
+            b = before_row + x * 4
+            a = after_row + x * 4
+            if max(
+                abs(before_rgba[b] - after_rgba[a]),
+                abs(before_rgba[b + 1] - after_rgba[a + 1]),
+                abs(before_rgba[b + 2] - after_rgba[a + 2]),
+                abs(before_rgba[b + 3] - after_rgba[a + 3]),
+            ) > threshold:
+                changed_pixels += 1
+    change_percent = (changed_pixels / total_pixels) * 100 if total_pixels else None
+    return {
+        'changed_pixels': changed_pixels,
+        'total_pixels': total_pixels,
+        'change_percent': change_percent,
+        'before_width': before_width,
+        'before_height': before_height,
+        'after_width': after_width,
+        'after_height': after_height,
+    }
+
+
+def measure_visual_delta_from_image_artifacts(before_url, after_url):
+    if not (image_artifact_url(before_url) and image_artifact_url(after_url)):
+        return {
+            'status': 'skipped',
+            'reason': 'before/after URLs are not direct HTTPS image artifacts',
+        }
+    try:
+        before_image = decode_png_rgba(fetch_url_bytes(before_url))
+        after_image = decode_png_rgba(fetch_url_bytes(after_url))
+        comparison = compare_rgba_images(before_image, after_image)
+    except Exception as exc:
+        return {
+            'status': 'error',
+            'error': type(exc).__name__ + ': ' + str(exc)[:300],
+        }
+    changed_pixels = comparison.get('changed_pixels')
+    total_pixels = comparison.get('total_pixels')
+    percent = comparison.get('change_percent')
+    percent_pass = percent is not None and percent >= MIN_VISUAL_DELTA_PERCENT
+    pixel_pass = changed_pixels is not None and changed_pixels >= MIN_VISUAL_CHANGED_PIXELS
+    passed = bool(percent_pass or pixel_pass)
+    return {
+        'status': 'measured',
+        'passed': passed,
+        'change_percent': round(percent, 4) if percent is not None else None,
+        'changed_pixels': int(changed_pixels) if changed_pixels is not None else None,
+        'total_pixels': int(total_pixels) if total_pixels is not None else None,
+        'min_change_percent': MIN_VISUAL_DELTA_PERCENT,
+        'min_changed_pixels': MIN_VISUAL_CHANGED_PIXELS,
+        'source': 'riddle_artifact_image_diff',
+        'reason': (
+            'Measured visual delta from before/after screenshot artifacts clears the legibility threshold.'
+            if passed else
+            'Measured visual delta from before/after screenshot artifacts is below the legibility threshold.'
+        ),
+        'comparison': {
+            'before_url': before_url,
+            'after_url': after_url,
+            'before_width': comparison.get('before_width'),
+            'before_height': comparison.get('before_height'),
+            'after_width': comparison.get('after_width'),
+            'after_height': comparison.get('after_height'),
+        },
+    }
+
+
 def add_visual_delta_diagnostic(visual_delta, key, value):
     updated = dict(visual_delta or {})
     diagnostic = dict(updated.get('diagnostic') or {})
@@ -970,6 +1169,21 @@ def measure_visual_delta_against_baseline(state, results, after_payload, current
             },
         )
 
+    artifact_delta = measure_visual_delta_from_image_artifacts(before_url, after_url)
+    append_capture_diagnostic(state, 'visual_delta', 'riddle_artifact_image_diff', {
+        'baseline_label': baseline.get('label') or '',
+        'before_url': before_url,
+        'after_url': after_url,
+    }, artifact_delta)
+    if isinstance(artifact_delta, dict) and artifact_delta.get('status') == 'measured':
+        artifact_delta['comparison']['baseline_label'] = baseline.get('label') or ''
+        return artifact_delta
+    current_visual_delta = add_visual_delta_diagnostic(
+        current_visual_delta,
+        'artifact_image_diff',
+        artifact_delta,
+    )
+
     args = {
         'url_before': before_url,
         'url_after': after_url,
@@ -1497,6 +1711,39 @@ def build_capture_retry_decision(after_observation, required_baseline_present, p
     }
 
 
+def build_visual_delta_recovery_decision(verification_mode, visual_delta, visual_delta_blocker=''):
+    if not visual_delta_applies(verification_mode):
+        return None
+    if visual_delta_passes_ship_gate(visual_delta):
+        return None
+    if not isinstance(visual_delta, dict):
+        status = 'missing'
+    else:
+        status = str(visual_delta.get('status') or 'missing')
+    if status == 'measured':
+        return None
+    reason = visual_delta_blocker or visual_delta_blocker_for_mode(verification_mode, visual_delta)
+    diagnostic = visual_delta.get('diagnostic') if isinstance(visual_delta, dict) else {}
+    reasons = [
+        reason or 'Required visual_delta evidence is incomplete.',
+        'Stay in verify so the same run can retry capture/comparison recovery before proof review.',
+    ]
+    fallback = diagnostic.get('visual_diff_fallback') if isinstance(diagnostic, dict) else None
+    artifact_diff = diagnostic.get('artifact_image_diff') if isinstance(diagnostic, dict) else None
+    if isinstance(artifact_diff, dict) and artifact_diff.get('error'):
+        reasons.append('Artifact image diff failed: ' + str(artifact_diff.get('error'))[:300])
+    if isinstance(fallback, dict) and fallback.get('error'):
+        reasons.append('Riddle visual_diff fallback failed: ' + str(fallback.get('error'))[:300])
+    return {
+        'decision': 'revise_capture',
+        'summary': reason or 'Verify needs measured visual_delta evidence before proof review.',
+        'recommended_stage': 'verify',
+        'continue_with_stage': 'verify',
+        'reasons': reasons,
+        'visual_delta_status': status,
+    }
+
+
 def compact_semantic_list(value, limit):
     if not isinstance(value, list):
         return []
@@ -2006,7 +2253,20 @@ if proof_evidence_required_for_mode(s.get('verification_mode')):
     if proof_evidence_blocker:
         summary_lines.append('Structured proof evidence gate: ' + proof_evidence_blocker)
 
-has_good_evidence = required_baseline_present and after_observation.get('valid') and not proof_evidence_blocker
+visual_delta_recovery = build_visual_delta_recovery_decision(
+    s.get('verification_mode'),
+    visual_delta,
+    visual_delta_blocker,
+)
+if visual_delta_recovery:
+    summary_lines.append('Visual delta recovery: ' + visual_delta_recovery['summary'])
+
+has_good_evidence = (
+    required_baseline_present
+    and after_observation.get('valid')
+    and not proof_evidence_blocker
+    and not visual_delta_recovery
+)
 
 if has_good_evidence:
     s['capture_hint_saved'] = record_successful_capture_hint(
@@ -2044,7 +2304,7 @@ if has_good_evidence:
     summary_lines.append('Proof assessment: awaiting supervising agent judgment')
     summary_lines.append('Proof next stage: supervising agent decides after reviewing the evidence packet')
 else:
-    capture_retry = build_capture_retry_decision(after_observation, required_baseline_present, proof_evidence_blocker)
+    capture_retry = visual_delta_recovery or build_capture_retry_decision(after_observation, required_baseline_present, proof_evidence_blocker)
     s['verify_status'] = 'capture_incomplete'
     s['merge_recommendation'] = 'do-not-merge'
     s['proof_assessment'] = {}
@@ -2061,8 +2321,9 @@ else:
         'continue_with_stage': capture_retry.get('continue_with_stage') or 'author',
         'fields_agent_may_update': ['capture_script', 'server_path', 'wait_for_selector', 'proof_plan'],
         'instructions': [
-            'The after-proof is missing or low quality, so return to author when the capture plan itself needs revision.',
-            'Adjust capture_script, server_path, wait_for_selector, and/or proof_plan before rerunning verify.',
+            'The after-proof evidence packet is incomplete, so use the recommended stage before proof review.',
+            'Adjust capture_script, server_path, wait_for_selector, and/or proof_plan only when the recommended stage is author.',
+            'If recommended_stage=verify, rerun verify so capture/comparison recovery can continue without changing proof authorship.',
             'If the baseline itself is wrong, return to recon instead of forcing verify to rediscover context.',
         ],
     }

package/runtime/tests/recon_verify_smoke.py

@@ -3,8 +3,10 @@ import json
 import os
 import shutil
 import subprocess as sp
+import struct
 import sys
 import tempfile
+import zlib
 from contextlib import contextmanager
 from pathlib import Path
 
@@ -35,6 +37,28 @@ def state_console(payload):
     }
 
 
+def png_rgba(width, height, pixels):
+    raw_rows = []
+    for y in range(height):
+        row = bytearray([0])
+        for x in range(width):
+            row.extend(pixels[(y * width + x) * 4:(y * width + x + 1) * 4])
+        raw_rows.append(bytes(row))
+    def chunk(kind, data):
+        return (
+            struct.pack('>I', len(data))
+            + kind
+            + data
+            + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
+        )
+    return (
+        b'\x89PNG\r\n\x1a\n'
+        + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 6, 0, 0, 0))
+        + chunk(b'IDAT', zlib.compress(b''.join(raw_rows)))
+        + chunk(b'IEND', b'')
+    )
+
+
 def load_module(name: str, path: Path):
     sys.modules.pop(name, None)
     spec = importlib.util.spec_from_file_location(name, path)
@@ -204,12 +228,13 @@ class FakeRiddle:
                     ],
                 }
             if 'after-proof' in script:
-                outputs = [{'name': 'after.png', 'url': 'https://cdn.example.com/after.png'}]
+                after_url = 'https://cdn.example.com/after-artifact' if 'noVisualDelta' in script else 'https://cdn.example.com/after.png'
+                outputs = [{'name': 'after.png', 'url': after_url}]
                 if 'proof-session' in script:
                     outputs.append({'name': 'proof-session.json', 'url': 'https://cdn.example.com/proof-session.json'})
-                return {
+                payload = {
                     'ok': True,
-                    'screenshots': [{'url': 'https://cdn.example.com/after.png'}],
+                    'screenshots': [{'url': after_url}],
                     'outputs': outputs,
                     'console': state_console({
                         'bodyTextLength': 180,
@@ -225,6 +250,13 @@ class FakeRiddle:
                         'largeVisibleElements': [{'tag': 'button', 'text': 'Buy Now'}],
                     }),
                 }
+                if 'noVisualDelta' not in script:
+                    payload['visual_diff'] = {
+                        'diffPercentage': 1.2,
+                        'differentPixels': 12000,
+                        'totalPixels': 972000,
+                    }
+                return payload
             if 'prod.example.com/pricing' in script:
                 return {
                     'ok': True,
@@ -585,6 +617,32 @@ def run_verify_quality_ignores_proof_telemetry_console_text():
     assert visual_diff_delta['change_percent'] == 1.45
     assert visual_diff_delta['changed_pixels'] == 14094
 
+    before_png = png_rgba(2, 1, bytes([
+        0, 0, 0, 255,
+        0, 0, 0, 255,
+    ]))
+    after_png = png_rgba(2, 1, bytes([
+        0, 0, 0, 255,
+        255, 255, 255, 255,
+    ]))
+
+    def fake_fetch_url_bytes(url, timeout=20, max_bytes=25 * 1024 * 1024):
+        if url.endswith('before.png'):
+            return before_png
+        if url.endswith('after.png'):
+            return after_png
+        raise AssertionError(f'unexpected image fetch: {url}')
+
+    namespace['fetch_url_bytes'] = fake_fetch_url_bytes
+    artifact_image_delta = namespace['measure_visual_delta_from_image_artifacts'](
+        'https://riddle-screenshots.example/before.png',
+        'https://riddle-screenshots.example/after.png',
+    )
+    assert artifact_image_delta['status'] == 'measured'
+    assert artifact_image_delta['source'] == 'riddle_artifact_image_diff'
+    assert artifact_image_delta['changed_pixels'] == 1
+    assert artifact_image_delta['change_percent'] == 50
+
     fallback_calls = []
 
     def fake_invoke_retry(tool, args, retries=3, timeout=180):
@@ -601,6 +659,7 @@ def run_verify_quality_ignores_proof_telemetry_console_text():
 
     namespace['invoke_retry'] = fake_invoke_retry
     namespace['append_capture_diagnostic'] = lambda *args, **kwargs: None
+    namespace['fetch_url_bytes'] = lambda *args, **kwargs: (_ for _ in ()).throw(ValueError('image fetch unavailable'))
     fallback_delta = namespace['measure_visual_delta_against_baseline'](
         {'verification_mode': 'visual', 'requested_reference': 'before', 'before_cdn': 'https://cdn.example.com/before.png'},
         {'baseline': {'before': {'url': 'https://cdn.example.com/before.png'}}},
@@ -1433,8 +1492,9 @@ def run_verify_requests_supervisor_assessment():
         assert after_verify['proof_assessment_source'] is None
         assert after_verify['proof_assessment_request']['status'] == 'needs_supervising_agent_assessment'
         visual_delta = after_verify['proof_assessment_request']['visual_delta']
-        assert visual_delta['status'] == 'unmeasured'
-        assert visual_delta['passed'] is None
+        assert visual_delta['status'] == 'measured'
+        assert visual_delta['passed'] is True
+        assert visual_delta['changed_pixels'] == 12000
         semantic_context = after_verify['proof_assessment_request']['semantic_context']
         assert semantic_context['route']['expected_path'] == '/pricing'
         assert semantic_context['route']['after_observed_path'] == '/pricing'
@@ -1450,14 +1510,15 @@ def run_verify_requests_supervisor_assessment():
         assert artifact_production['image_output_count'] >= 1
         assert artifact_production['proof_evidence_present'] is False
         artifact_usage = after_verify['proof_assessment_request']['artifact_usage']
-        assert artifact_usage['missing_required_signals'] == ['visual_delta']
+        assert artifact_usage['missing_required_signals'] == []
         assert 'after-capture' in artifact_usage['supervisor_review_signals']
         assert 'baseline_context' in artifact_usage['required_signals']
         assert 'route_semantics' in artifact_usage['available_signals']
-        assert 'visual_delta.status=unmeasured' in after_verify['proof_assessment_request']['hard_blockers'][0]
+        assert 'visual_delta' in artifact_usage['available_signals']
+        assert after_verify['proof_assessment_request']['hard_blockers'] == []
         assert after_verify['proof_assessment_request']['evidence_bundle']['artifact_contract']['required']['screenshot'] is True
         assert after_verify['proof_assessment_request']['evidence_bundle']['artifact_production']['image_output_count'] >= 1
-        assert after_verify['proof_assessment_request']['evidence_bundle']['artifact_usage']['missing_required_signals'] == ['visual_delta']
+        assert after_verify['proof_assessment_request']['evidence_bundle']['artifact_usage']['missing_required_signals'] == []
         assert 'capture success is not proof' in '\n'.join(after_verify['proof_assessment_request']['instructions'])
         assert after_verify['verify_decision_request']['continue_with_stage'] is None
         assert after_verify['verify_results']['baseline']['before']['source'] == 'recon'
@@ -1501,6 +1562,55 @@ def run_verify_requests_supervisor_assessment():
         shutil.rmtree(tempdir, ignore_errors=True)
 
 
+def run_verify_routes_unmeasured_visual_delta_to_recovery():
+    tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-visual-recovery-'))
+    state_path = tempdir / 'state.json'
+    try:
+        state = base_state(tempdir, reference='before')
+        state.update({
+            'repo': 'example/repo',
+            'recon_status': 'ready_for_proof_plan',
+            'author_status': 'ready',
+            'proof_plan_status': 'ready',
+            'implementation_status': 'changes_detected',
+            'before_cdn': 'https://cdn.example.com/before-artifact',
+            'proof_plan': 'Use the recon-confirmed /pricing route and capture the CTA state once it stabilizes.',
+            'capture_script': "noVisualDelta(); await page.waitForSelector('[data-testid=pricing-cta]'); await saveScreenshot('after-proof');",
+            'wait_for_selector': '[data-testid=pricing-cta]',
+            'recon_results': {
+                'baselines': {
+                    'before': {'path': '/pricing', 'url': 'https://cdn.example.com/before-artifact'},
+                },
+            },
+        })
+        write_state(state_path, state)
+        os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
+
+        fake = FakeRiddle()
+        load_util_with_fake(fake)
+        load_module('verify_unmeasured_visual_delta_recovery', VERIFY_PATH)
+        after_verify = json.loads(state_path.read_text())
+
+        assert after_verify['verify_status'] == 'capture_incomplete'
+        assert after_verify['proof_assessment_request'] == {}
+        decision_request = after_verify['verify_decision_request']
+        assert decision_request['capture_quality']['decision'] == 'revise_capture'
+        assert decision_request['recommended_stage'] == 'verify'
+        assert decision_request['continue_with_stage'] == 'verify'
+        visual_delta = after_verify['evidence_bundle']['after']['visual_delta']
+        assert visual_delta['status'] == 'unmeasured'
+        assert visual_delta['diagnostic']['visual_diff_fallback']['status'] == 'error'
+        assert 'Visual delta recovery' in after_verify['verify_summary']
+
+        return {
+            'ok': True,
+            'verify_status': after_verify['verify_status'],
+            'continue_with_stage': decision_request['continue_with_stage'],
+        }
+    finally:
+        shutil.rmtree(tempdir, ignore_errors=True)
+
+
 def run_verify_structured_evidence_without_screenshot():
     tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-verify-structured-'))
     state_path = tempdir / 'state.json'
@@ -2142,6 +2252,7 @@ if __name__ == '__main__':
         'capture_hint_rejects_route_specific_mode_only_match': run_capture_hint_rejects_route_specific_mode_only_match(),
         'author_applies_supervisor_packet': run_author_applies_supervisor_packet(),
         'verify_requests_supervisor_assessment': run_verify_requests_supervisor_assessment(),
+        'verify_routes_unmeasured_visual_delta_to_recovery': run_verify_routes_unmeasured_visual_delta_to_recovery(),
         'verify_structured_evidence_without_screenshot': run_verify_structured_evidence_without_screenshot(),
         'verify_audio_requires_proof_evidence': run_verify_audio_requires_proof_evidence(),
         'verify_audio_rejects_failed_nested_proof_evidence': run_verify_audio_rejects_failed_nested_proof_evidence(),