@riddledc/riddle-proof 0.8.30 → 0.8.32

package/runtime/lib/verify.py

@@ -310,12 +310,13 @@ def payload_has_capture_artifacts(payload):
 def capture_payload_error(payload):
     if not isinstance(payload, dict):
         return ''
-    if payload.get('ok') is False and not payload_has_capture_artifacts(payload):
-        for key in ('error', 'stderr', 'stdout'):
+    if payload.get('ok') is False:
+        for key in ('error', 'script_error', 'stderr', 'stdout'):
             value = payload.get(key)
             if value:
                 return str(value).strip()
-        return 'capture tool returned ok=false without artifacts'
+        if not payload_has_capture_artifacts(payload):
+            return 'capture tool returned ok=false without artifacts'
     return ''
 
 

package/runtime/pipelines/riddle-proof-ship.lobster

@@ -12,7 +12,17 @@ steps:
       import json, os
       state_file = os.environ.get('RIDDLE_PROOF_STATE_FILE', '/tmp/riddle-proof-state.json')
       s = json.load(open(state_file))
-      if not s.get('after_cdn'):
+      after = ((s.get('evidence_bundle') or {}).get('after') or {})
+      observation = after.get('observation') or {}
+      supporting = after.get('supporting_artifacts') or {}
+      has_structured_after = bool(
+          observation.get('valid') and (
+              supporting.get('has_structured_payload') or
+              supporting.get('proof_evidence_present') or
+              observation.get('telemetry_ready')
+          )
+      )
+      if not s.get('after_cdn') and not has_structured_after:
           raise SystemExit('No after evidence. Run verify first.')
       print('SHIP')
       print('  Proof goal: ' + s.get('change_request', ''))

package/runtime/tests/recon_verify_smoke.py

@@ -89,6 +89,138 @@ class FakeRiddle:
                 'id': f'pv-{label}',
                 'preview_url': f'https://preview.example.com/{label}/',
             }
+        if tool == 'riddle_server_preview':
+            script = args.get('script', '')
+            target_path = args.get('path') or '/'
+            path_only, _, query = str(target_path).partition('?')
+            search = '?' + query if query else ''
+            delegated_markers = [
+                'after-proof',
+                'audioNoProof',
+                'audioFailedProof',
+                'throwAfterProofEvidence',
+                'attack_ms_after',
+                'window.__riddleProofEvidence',
+                'globalThis.__riddleProofEvidence',
+                'clickedSkipHashNavigation',
+                'pricingQueryHashDropsTerminal',
+                'pricingQueryHashStructuredNegativeControl',
+                'pricingQueryHashPassesWithPageStateHashGap',
+                'clickedProofNavigation',
+                'clickedHomeNavigation',
+                'skipLinkTimeout',
+                'interactionThrownAfterFailedEvidence',
+                'interactionThrownError',
+            ]
+            if any(marker in script for marker in delegated_markers):
+                return self.invoke_retry('riddle_script', {'script': script}, retries=retries, timeout=timeout)
+            if path_only == '/wrong' or '/wrong' in script:
+                return {
+                    'ok': True,
+                    'runner': 'local-server-preview',
+                    'target_url': 'http://127.0.0.1:3000/wrong',
+                    'screenshots': [{'url': 'https://cdn.example.com/wrong.png'}],
+                    'outputs': [{'name': 'wrong.png', 'url': 'https://cdn.example.com/wrong.png'}],
+                    'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":5,"interactiveElements":0,"pathname":"/wrong","title":"Wrong"}'],
+                }
+            if '/games/drum-sequencer' in path_only:
+                page_state = {
+                    'bodyTextLength': 240,
+                    'visibleTextSample': 'Neon Step Sequencer Monkberry Moon Delight Mix Board Play All',
+                    'interactiveElements': 8,
+                    'visibleInteractiveElements': 8,
+                    'pathname': '/games/drum-sequencer',
+                    'search': search,
+                    'title': 'Neon Step Sequencer',
+                    'buttons': ['Play All', 'Shuffle'],
+                    'headings': ['Neon Step Sequencer'],
+                    'links': [],
+                    'canvasCount': 1,
+                    'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
+                }
+                return {
+                    'ok': True,
+                    'runner': 'local-server-preview',
+                    'target_url': 'http://127.0.0.1:3000' + str(target_path),
+                    'screenshots': [{'url': 'https://cdn.example.com/sequencer-before.png'}],
+                    'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/sequencer-before.png'}],
+                    'console': state_console(page_state),
+                }
+            if '/games/tic-tac-toe' in path_only:
+                return {
+                    'ok': True,
+                    'runner': 'local-server-preview',
+                    'target_url': 'http://127.0.0.1:3000' + str(target_path),
+                    'screenshots': [{'url': 'https://cdn.example.com/tictactoe-before.png'}],
+                    'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/tictactoe-before.png'}],
+                    'console': state_console({
+                        'bodyTextLength': 220,
+                        'visibleTextSample': 'LilArcade Tic Tac Toe Player X Reset Game',
+                        'interactiveElements': 5,
+                        'visibleInteractiveElements': 5,
+                        'pathname': '/games/tic-tac-toe',
+                        'title': 'TicTacToe',
+                        'buttons': ['Reset Game'],
+                        'headings': ['Tic Tac Toe'],
+                        'links': [],
+                        'canvasCount': 0,
+                        'largeVisibleElements': [{'tag': 'button', 'text': 'Reset Game'}],
+                    }),
+                }
+            if 'after-proof' in script:
+                after_url = 'https://cdn.example.com/after-artifact' if 'noVisualDelta' in script else 'https://cdn.example.com/after.png'
+                outputs = [{'name': 'after.png', 'url': after_url}]
+                if 'proof-session' in script:
+                    outputs.append({'name': 'proof-session.json', 'url': 'https://cdn.example.com/proof-session.json'})
+                payload = {
+                    'ok': True,
+                    'runner': 'local-server-preview',
+                    'target_url': 'http://127.0.0.1:3000' + str(target_path),
+                    'screenshots': [{'url': after_url}],
+                    'outputs': outputs,
+                    'console': state_console({
+                        'bodyTextLength': 180,
+                        'visibleTextSample': 'Pricing CTA Buy Now',
+                        'interactiveElements': 4,
+                        'visibleInteractiveElements': 4,
+                        'pathname': path_only or '/pricing',
+                        'search': search,
+                        'title': 'After',
+                        'buttons': ['Buy Now'],
+                        'headings': ['Pricing'],
+                        'links': [],
+                        'canvasCount': 0,
+                        'largeVisibleElements': [{'tag': 'button', 'text': 'Buy Now'}],
+                    }),
+                }
+                if 'noVisualDelta' not in script:
+                    payload['visual_diff'] = {
+                        'diffPercentage': 1.2,
+                        'differentPixels': 12000,
+                        'totalPixels': 972000,
+                    }
+                return payload
+            return {
+                'ok': True,
+                'runner': 'local-server-preview',
+                'target_url': 'http://127.0.0.1:3000' + str(target_path),
+                'screenshots': [{'url': 'https://cdn.example.com/home-before.png'}],
+                'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/home-before.png'}],
+                'console': state_console({
+                    'bodyTextLength': 180,
+                    'visibleTextSample': 'Riddle Proof homepage hero Start Free',
+                    'interactiveElements': 4,
+                    'visibleInteractiveElements': 4,
+                    'pathname': path_only or '/',
+                    'search': search,
+                    'title': 'Riddle',
+                    'buttons': ['Start Free'],
+                    'headings': ['Riddle Proof'],
+                    'links': [],
+                    'canvasCount': 0,
+                    'largeVisibleElements': [{'tag': 'button', 'text': 'Start Free'}],
+                }),
+            }
         if tool == 'riddle_script':
             script = args.get('script', '')
             if 'preview.example.com' in script and '/wrong' in script:

package/runtime/tests/ship_artifact_publication.py

@@ -0,0 +1,185 @@
+import json
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SHIP = ROOT / "runtime" / "lib" / "ship.py"
+
+
+def run(args, cwd, env=None):
+    result = subprocess.run(args, cwd=cwd, env=env, capture_output=True, text=True, timeout=120)
+    if result.returncode != 0:
+        raise AssertionError(
+            f"{' '.join(args)} failed\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+        )
+    return result
+
+
+def write_fake_gh(path):
+    path.write_text(
+        """#!/usr/bin/env python3
+import json
+import os
+import sys
+
+args = sys.argv[1:]
+if args[:3] == ["repo", "view", "--json"]:
+    print("example/test-repo")
+    raise SystemExit(0)
+if args[:2] == ["pr", "list"]:
+    print("")
+    raise SystemExit(0)
+if args[:2] == ["pr", "create"]:
+    print("https://github.com/example/test-repo/pull/321")
+    raise SystemExit(0)
+if args[:2] == ["pr", "comment"]:
+    body = ""
+    if "--body" in args:
+        body = args[args.index("--body") + 1]
+    with open(os.environ["FAKE_GH_COMMENT_BODY"], "w") as f:
+        f.write(body)
+    print("https://github.com/example/test-repo/pull/321#issuecomment-999")
+    raise SystemExit(0)
+if args[:2] == ["pr", "checks"]:
+    print("[]")
+    raise SystemExit(0)
+if args[:2] == ["pr", "ready"]:
+    raise SystemExit(0)
+if args[:2] == ["pr", "edit"]:
+    raise SystemExit(0)
+print("unknown gh command: " + " ".join(args), file=sys.stderr)
+raise SystemExit(1)
+""",
+        encoding="utf-8",
+    )
+    path.chmod(0o755)
+
+
+def main():
+    with tempfile.TemporaryDirectory(prefix="riddle-proof-ship-artifacts-") as tmp:
+        root = Path(tmp)
+        origin = root / "origin.git"
+        repo = root / "repo"
+        artifacts = root / "artifacts"
+        bin_dir = root / "bin"
+        state_path = root / "state.json"
+        comment_body_path = root / "comment.md"
+        artifacts.mkdir()
+        bin_dir.mkdir()
+
+        run(["git", "init", "--bare", str(origin)], cwd=root)
+        run(["git", "init", str(repo)], cwd=root)
+        run(["git", "config", "user.name", "Test User"], cwd=repo)
+        run(["git", "config", "user.email", "test@example.com"], cwd=repo)
+        run(["git", "remote", "add", "origin", str(origin)], cwd=repo)
+        run(["git", "checkout", "-b", "agent/proof-artifact-test"], cwd=repo)
+        (repo / "README.md").write_text("initial\n", encoding="utf-8")
+        run(["git", "add", "README.md"], cwd=repo)
+        run(["git", "commit", "-m", "Initial"], cwd=repo)
+        (repo / "README.md").write_text("changed\n", encoding="utf-8")
+
+        # Tiny valid PNG header/body is enough for GitHub Markdown image embedding.
+        screenshot = artifacts / "after-proof.png"
+        screenshot.write_bytes(
+            bytes.fromhex(
+                "89504e470d0a1a0a0000000d4948445200000001000000010802000000907753de"
+                "0000000c49444154789c63606060000000040001f61738550000000049454e44ae426082"
+            )
+        )
+        proof_json = artifacts / "proof.json"
+        proof_json.write_text(
+            json.dumps(
+                {
+                    "version": "riddle-proof.test.v1",
+                    "assertions": [{"name": "proof image published", "passed": True}],
+                },
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+
+        write_fake_gh(bin_dir / "gh")
+        state = {
+            "repo_dir": str(repo),
+            "branch": "agent/proof-artifact-test",
+            "target_branch": "agent/proof-artifact-test",
+            "run_id": "rp_ship_artifact_test",
+            "change_request": "Prove PR artifact publication.",
+            "commit_message": "Test proof artifact publication",
+            "success_criteria": "The PR proof comment embeds a GitHub-hosted image.",
+            "verification_mode": "proof",
+            "requested_reference": "none",
+            "reference": "none",
+            "verify_status": "evidence_captured",
+            "after_cdn": screenshot.as_uri(),
+            "assertion_status": "passed",
+            "proof_summary": "All assertions passed.",
+            "proof_assessment_source": "supervising_agent",
+            "proof_assessment": {
+                "source": "supervising_agent",
+                "decision": "ready_to_ship",
+                "summary": "Evidence is strong enough to ship.",
+            },
+            "evidence_bundle": {
+                "verification_mode": "proof",
+                "after": {
+                    "observation": {"valid": True, "reason": "ok", "telemetry_ready": True},
+                    "supporting_artifacts": {
+                        "has_structured_payload": True,
+                        "proof_evidence_present": True,
+                        "image_outputs": [{"name": "after-proof.png", "url": screenshot.as_uri()}],
+                        "data_outputs": [{"name": "proof.json", "url": proof_json.as_uri()}],
+                    },
+                },
+            },
+            "verify_results": {
+                "after": {
+                    "raw": {
+                        "outputs": [
+                            {"name": "after-proof.png", "url": screenshot.as_uri(), "path": str(screenshot)},
+                            {"name": "proof.json", "url": proof_json.as_uri(), "path": str(proof_json)},
+                        ],
+                    },
+                },
+            },
+        }
+        state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
+        env = {
+            **os.environ,
+            "PATH": str(bin_dir) + os.pathsep + os.environ.get("PATH", ""),
+            "RIDDLE_PROOF_STATE_FILE": str(state_path),
+            "FAKE_GH_COMMENT_BODY": str(comment_body_path),
+            "DISCORD_BOT_TOKEN": "",
+            "OPENCLAW_HOME": str(root / "openclaw-home"),
+        }
+
+        run(["python3", str(SHIP)], cwd=repo, env=env)
+
+        updated = json.loads(state_path.read_text(encoding="utf-8"))
+        publication = updated.get("proof_artifact_publication") or {}
+        assert publication.get("ok") is True, "proof artifact publication should be recorded"
+        assert publication.get("artifacts"), "published artifact list should be recorded"
+        assert updated.get("ship_report", {}).get("after_artifact_url", "").startswith(
+            "https://raw.githubusercontent.com/example/test-repo/"
+        ), "ship report should expose a public after artifact URL"
+
+        comment = comment_body_path.read_text(encoding="utf-8")
+        assert "file://" not in comment, "PR proof comment must not expose local file URLs"
+        assert "![after](https://raw.githubusercontent.com/example/test-repo/" in comment, (
+            "PR proof comment should embed the GitHub-hosted after screenshot"
+        )
+        assert "[proof.json](https://github.com/example/test-repo/blob/" in comment, (
+            "PR proof comment should link the structured proof JSON"
+        )
+        assert "Proof artifacts:" in comment, "PR proof comment should link the artifact bundle"
+
+        artifact_branch = publication.get("branch")
+        refs = run(["git", f"--git-dir={origin}", "show-ref", f"refs/heads/{artifact_branch}"], cwd=root)
+        assert artifact_branch in refs.stdout, "artifact branch should be pushed to origin"
+
+
+if __name__ == "__main__":
+    main()