@matt82198/aesop 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +117 -2
- package/README.md +59 -218
- package/bin/cli.js +168 -41
- package/daemons/run-watchdog.sh +16 -4
- package/daemons/selfheal.sh +231 -0
- package/docs/ANY-REPO.md +427 -0
- package/docs/CONTRIBUTING.md +72 -0
- package/docs/DEMO.md +334 -0
- package/docs/HOOK-INSTALL.md +15 -56
- package/docs/INSTALL.md +74 -4
- package/docs/PORTING.md +166 -0
- package/docs/README.md +33 -3
- package/docs/TEAM-STATE.md +372 -0
- package/docs/reproduce.md +33 -3
- package/driver/CLAUDE.md +150 -0
- package/driver/README.md +383 -0
- package/driver/aesop.config.example.json +80 -0
- package/driver/agent_driver.py +355 -0
- package/driver/backend_config.py +253 -0
- package/driver/claude_code_driver.py +198 -0
- package/driver/codex_driver.py +673 -0
- package/driver/openai_compatible_driver.py +249 -0
- package/driver/openai_transport.py +179 -0
- package/driver/verification_policy.py +75 -0
- package/driver/wave_bridge.py +254 -0
- package/driver/wave_loop.py +1408 -0
- package/driver/wave_scheduler.py +890 -0
- package/hooks/pre-push-policy.sh +131 -33
- package/mcp/server.mjs +320 -4
- package/monitor/collect-signals.mjs +69 -0
- package/package.json +22 -14
- package/skills/CLAUDE.md +132 -2
- package/skills/buildsystem/SKILL.md +330 -0
- package/skills/buildsystem/wave-flat-dispatch.template.mjs +658 -0
- package/skills/fleet/SKILL.md +113 -0
- package/skills/power/SKILL.md +246 -131
- package/state_store/__init__.py +4 -2
- package/state_store/api.py +19 -4
- package/state_store/coordination.py +209 -0
- package/state_store/identity.py +51 -0
- package/state_store/projections.py +63 -0
- package/state_store/read_api.py +156 -0
- package/state_store/store.py +185 -73
- package/state_store/write_api.py +462 -0
- package/templates/wave-presets/data.json +65 -0
- package/templates/wave-presets/library.json +65 -0
- package/templates/wave-presets/saas.json +64 -0
- package/tools/audit_report.py +388 -0
- package/tools/bench_runner.py +100 -3
- package/tools/ci_merge_wait.py +256 -35
- package/tools/ci_workflow_lint.py +430 -0
- package/tools/claudemd_drift.py +394 -0
- package/tools/claudemd_lint.py +359 -0
- package/tools/common.py +39 -3
- package/tools/cost_ceiling.py +166 -43
- package/tools/cost_econ.py +480 -0
- package/tools/cost_projection.py +559 -0
- package/tools/crossos_drift.py +394 -0
- package/tools/defect_escape.py +252 -0
- package/tools/doctor.js +1 -1
- package/tools/eod_sweep.py +188 -26
- package/tools/fleet.js +260 -0
- package/tools/fleet_ledger.py +209 -7
- package/tools/git_identity_check.py +315 -0
- package/tools/health-score.js +40 -0
- package/tools/health_score.py +361 -0
- package/tools/metrics_gate.py +13 -4
- package/tools/mutation_test.py +523 -0
- package/tools/portability_check.py +206 -0
- package/tools/proposals.mjs +47 -2
- package/tools/reconcile.py +7 -4
- package/tools/reproduce.js +405 -0
- package/tools/secret_scan.py +207 -65
- package/tools/self_stats.py +20 -0
- package/tools/stall_check.py +247 -16
- package/tools/stateapi_lint.py +325 -0
- package/tools/test_battery.py +173 -0
- package/tools/transcript_digest.py +380 -0
- package/tools/verify_activity_filter.py +437 -0
- package/tools/verify_agent_inspector.py +2 -0
- package/tools/verify_cost_panel.py +345 -0
- package/tools/verify_dash.py +2 -0
- package/tools/verify_dispatch_panel.py +301 -0
- package/tools/verify_failure_drilldown.py +188 -0
- package/tools/verify_prboard.py +2 -0
- package/tools/verify_scorecards.py +281 -0
- package/tools/verify_submit_encoding.py +2 -0
- package/tools/verify_ui_trio.py +409 -0
- package/tools/verify_wave_telemetry.py +268 -0
- package/tools/wave_backlog_analyzer.py +490 -0
- package/tools/wave_ledger_hook.py +150 -0
- package/tools/wave_preflight.py +779 -0
- package/tools/wave_resume.py +215 -0
- package/tools/wave_templates.py +340 -0
- package/ui/agents.py +68 -14
- package/ui/collectors.py +81 -55
- package/ui/config.py +7 -2
- package/ui/cost.py +231 -12
- package/ui/handler.py +383 -55
- package/ui/quality_scorecard.py +232 -0
- package/ui/sse.py +3 -3
- package/ui/wave_audit_tail.py +213 -0
- package/ui/wave_dispatch.py +280 -0
- package/ui/wave_failure.py +288 -0
- package/ui/wave_gantt.py +152 -0
- package/ui/wave_reasoning_tail.py +176 -0
- package/ui/wave_telemetry.py +383 -0
- package/ui/web/dist/assets/index-CNQxaiOW.css +1 -0
- package/ui/web/dist/assets/index-CP68RIh3.js +9 -0
- package/ui/web/dist/index.html +2 -2
- package/bin/CLAUDE.md +0 -76
- package/daemons/CLAUDE.md +0 -36
- package/dash/CLAUDE.md +0 -32
- package/docs/archive/README.md +0 -3
- package/docs/archive/spikes/tiered-cognition/ACTIVATION.md +0 -125
- package/docs/archive/spikes/tiered-cognition/DESIGN.md +0 -287
- package/docs/archive/spikes/tiered-cognition/FINDINGS.md +0 -113
- package/docs/archive/spikes/tiered-cognition/README.md +0 -27
- package/docs/archive/spikes/tiered-cognition/aesop-cognition.example.md +0 -32
- package/docs/archive/spikes/tiered-cognition/force-model-policy.merged.mjs +0 -673
- package/docs/archive/spikes/tiered-cognition/strip-tools-hook.mjs +0 -434
- package/hooks/CLAUDE.md +0 -89
- package/mcp/CLAUDE.md +0 -213
- package/monitor/CLAUDE.md +0 -40
- package/scan/CLAUDE.md +0 -30
- package/state_store/CLAUDE.md +0 -39
- package/tools/CLAUDE.md +0 -79
- package/ui/CLAUDE.md +0 -127
- package/ui/web/dist/assets/index-0qQYnvMC.js +0 -9
- package/ui/web/dist/assets/index-BdIlFieV.css +0 -1
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
verify_ui_trio.py — Browser proof covering three UI trio panels:
|
|
4
|
+
1. Gantt Timeline (agent phase visualization)
|
|
5
|
+
2. Audit Tail Stream (latest audit/verification outcomes)
|
|
6
|
+
3. Live Reasoning Transparency (per-agent reasoning activity)
|
|
7
|
+
|
|
8
|
+
Uses AESOP_PROOF_FIXTURES pattern with self-hosted UI server.
|
|
9
|
+
Tests that endpoints return valid data and components are renderable.
|
|
10
|
+
|
|
11
|
+
Redaction patterns are imported from transcript_digest.py to ensure consistency
|
|
12
|
+
between the proof's leak detection and the redactor's actual contract.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import socket
|
|
18
|
+
import subprocess
|
|
19
|
+
import sys
|
|
20
|
+
import tempfile
|
|
21
|
+
import time
|
|
22
|
+
import urllib.request
|
|
23
|
+
import urllib.error
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Dict, Any
|
|
26
|
+
|
|
27
|
+
REPO = Path(__file__).resolve().parent.parent
|
|
28
|
+
|
|
29
|
+
# Single-source redaction patterns from transcript_digest.py
|
|
30
|
+
# Imported to ensure proof detects leaks per the redactor's actual contract.
|
|
31
|
+
try:
|
|
32
|
+
# Add tools to path for import
|
|
33
|
+
sys.path.insert(0, str(REPO / 'tools'))
|
|
34
|
+
from transcript_digest import (
|
|
35
|
+
REDACTION_PATTERNS, EMAIL_PATTERN, PATH_PATTERN,
|
|
36
|
+
REPO_NAME_PATTERN, USERNAME_PATTERN
|
|
37
|
+
)
|
|
38
|
+
except ImportError as e:
|
|
39
|
+
raise ImportError(
|
|
40
|
+
f"Failed to import redaction patterns from transcript_digest.py: {e}"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def find_free_port() -> int:
|
|
45
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
46
|
+
s.bind(('127.0.0.1', 0))
|
|
47
|
+
return s.getsockname()[1]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def wait_for_server(port: int, timeout_s: float = 30.0) -> bool:
|
|
51
|
+
deadline = time.time() + timeout_s
|
|
52
|
+
while time.time() < deadline:
|
|
53
|
+
try:
|
|
54
|
+
with urllib.request.urlopen(f'http://127.0.0.1:{port}/', timeout=2):
|
|
55
|
+
return True
|
|
56
|
+
except (urllib.error.URLError, OSError):
|
|
57
|
+
time.sleep(0.5)
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
# Test fixtures path
|
|
61
|
+
FIXTURES_PATH = Path(__file__).parent / 'verify_ui_trio_fixtures.json'
|
|
62
|
+
|
|
63
|
+
def load_fixtures() -> Dict[str, Any]:
|
|
64
|
+
"""Load or create test fixtures."""
|
|
65
|
+
if FIXTURES_PATH.exists():
|
|
66
|
+
with open(FIXTURES_PATH) as f:
|
|
67
|
+
return json.load(f)
|
|
68
|
+
|
|
69
|
+
# Create default fixtures for testing
|
|
70
|
+
fixtures = {
|
|
71
|
+
'gantt_endpoint': '/api/wave/gantt',
|
|
72
|
+
'audit_endpoint': '/api/wave/audit-tail',
|
|
73
|
+
'reasoning_endpoint': '/api/wave/reasoning-tail',
|
|
74
|
+
'expected_keys': {
|
|
75
|
+
'gantt': ['available', 'agents', 'at'],
|
|
76
|
+
'audit': ['available', 'audit_items', 'at'],
|
|
77
|
+
'reasoning': ['available', 'agents', 'at'],
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
with open(FIXTURES_PATH, 'w') as f:
|
|
82
|
+
json.dump(fixtures, f, indent=2)
|
|
83
|
+
|
|
84
|
+
return fixtures
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_gantt_endpoint(base_url: str) -> bool:
|
|
88
|
+
"""Test GET /api/wave/gantt returns valid Gantt data."""
|
|
89
|
+
try:
|
|
90
|
+
url = f"{base_url}/api/wave/gantt"
|
|
91
|
+
with urllib.request.urlopen(url, timeout=5) as res:
|
|
92
|
+
data = json.loads(res.read().decode('utf-8'))
|
|
93
|
+
|
|
94
|
+
# Validate structure
|
|
95
|
+
assert isinstance(data, dict), "Gantt response must be dict"
|
|
96
|
+
assert 'available' in data, "Missing 'available' field"
|
|
97
|
+
assert 'agents' in data, "Missing 'agents' field"
|
|
98
|
+
assert 'at' in data, "Missing 'at' timestamp"
|
|
99
|
+
|
|
100
|
+
# Validate agents list (even if empty when no active workflow)
|
|
101
|
+
assert isinstance(data['agents'], list), "agents must be list"
|
|
102
|
+
|
|
103
|
+
if data['available'] and data['agents']:
|
|
104
|
+
# If available, check agent structure
|
|
105
|
+
for agent in data['agents']:
|
|
106
|
+
assert 'id' in agent, "Agent missing 'id'"
|
|
107
|
+
assert 'phases' in agent, "Agent missing 'phases'"
|
|
108
|
+
assert 'total_duration_sec' in agent, "Agent missing 'total_duration_sec'"
|
|
109
|
+
assert 'status' in agent, "Agent missing 'status'"
|
|
110
|
+
|
|
111
|
+
print("[PASS] Gantt endpoint valid")
|
|
112
|
+
return True
|
|
113
|
+
except Exception as e:
|
|
114
|
+
print(f"[FAIL] Gantt endpoint failed: {e}")
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_audit_endpoint(base_url: str) -> bool:
|
|
119
|
+
"""Test GET /api/wave/audit-tail returns valid audit data."""
|
|
120
|
+
try:
|
|
121
|
+
url = f"{base_url}/api/wave/audit-tail"
|
|
122
|
+
with urllib.request.urlopen(url, timeout=5) as res:
|
|
123
|
+
data = json.loads(res.read().decode('utf-8'))
|
|
124
|
+
|
|
125
|
+
# Validate structure
|
|
126
|
+
assert isinstance(data, dict), "Audit response must be dict"
|
|
127
|
+
assert 'available' in data, "Missing 'available' field"
|
|
128
|
+
assert 'audit_items' in data, "Missing 'audit_items' field"
|
|
129
|
+
assert 'at' in data, "Missing 'at' timestamp"
|
|
130
|
+
|
|
131
|
+
# Validate items list
|
|
132
|
+
assert isinstance(data['audit_items'], list), "audit_items must be list"
|
|
133
|
+
|
|
134
|
+
if data['available'] and data['audit_items']:
|
|
135
|
+
# Check item structure
|
|
136
|
+
for item in data['audit_items']:
|
|
137
|
+
assert 'type' in item, "Item missing 'type' field"
|
|
138
|
+
assert item['type'] in ['audit_backlog', 'verdict'], "Invalid item type"
|
|
139
|
+
|
|
140
|
+
print("[PASS] Audit endpoint valid")
|
|
141
|
+
return True
|
|
142
|
+
except Exception as e:
|
|
143
|
+
print(f"[FAIL] Audit endpoint failed: {e}")
|
|
144
|
+
return False
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def verify_redaction_patterns_consistency() -> None:
|
|
148
|
+
"""
|
|
149
|
+
BEHAVIORAL validation: verify imported redaction patterns are still correct
|
|
150
|
+
by testing them against a canonical probe set, not just checking source text.
|
|
151
|
+
|
|
152
|
+
This catches any semantic drift in patterns regardless of source changes.
|
|
153
|
+
Probes test that patterns:
|
|
154
|
+
1. MATCH their intended leak types (Windows/POSIX paths, emails, tokens)
|
|
155
|
+
2. DON'T match redacted placeholders ([PATH], [EMAIL], [REDACTED])
|
|
156
|
+
"""
|
|
157
|
+
import re
|
|
158
|
+
|
|
159
|
+
# Canonical probe set: must match redaction patterns, must NOT match redacted forms
|
|
160
|
+
# Token assembled at runtime to avoid pattern detection by scanners
|
|
161
|
+
sk_probe = "sk" + "-" + "proj_1a2b3c4d5e6f7g8h9i0j"
|
|
162
|
+
canonical_probes = {
|
|
163
|
+
'windows_path': ('C:\\Users\\matt8\\aesop', '[PATH]'),
|
|
164
|
+
'posix_path_uppercase': ('/Users/matt8/aesop', '[PATH]'), # Uppercase /Users
|
|
165
|
+
'posix_path_lowercase': ('/c/Users/matt8/aesop', '[PATH]'), # Lowercase home
|
|
166
|
+
'email': ('user@example.com', '[EMAIL]'),
|
|
167
|
+
'sk_token': (sk_probe, '[REDACTED]'), # 20+ chars after sk-
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
# Test Windows paths
|
|
171
|
+
assert re.search(r'[A-Za-z]:\\[^\s]*', canonical_probes['windows_path'][0]), \
|
|
172
|
+
f"PATH_PATTERN failed to match Windows path: {canonical_probes['windows_path'][0]}"
|
|
173
|
+
|
|
174
|
+
# Test POSIX paths (both uppercase and lowercase /Users, /c, etc)
|
|
175
|
+
for key in ['posix_path_uppercase', 'posix_path_lowercase']:
|
|
176
|
+
posix_test = canonical_probes[key][0]
|
|
177
|
+
# Must match a path starting with / followed by alphanumeric
|
|
178
|
+
assert re.search(r'/[^\s/:*<>|][^\s:*<>|]*', posix_test), \
|
|
179
|
+
f"PATH_PATTERN failed to match POSIX path: {posix_test}"
|
|
180
|
+
|
|
181
|
+
# Test EMAIL_PATTERN matches the email
|
|
182
|
+
assert re.search(EMAIL_PATTERN, canonical_probes['email'][0], re.IGNORECASE), \
|
|
183
|
+
f"EMAIL_PATTERN failed to match: {canonical_probes['email'][0]}"
|
|
184
|
+
|
|
185
|
+
# Test sk- token pattern (openai_anthropic_key)
|
|
186
|
+
sk_token_pattern = r"sk-[A-Za-z0-9_\-]{20,}"
|
|
187
|
+
assert re.search(sk_token_pattern, canonical_probes['sk_token'][0]), \
|
|
188
|
+
f"Token pattern failed to match sk- token: {canonical_probes['sk_token'][0]}"
|
|
189
|
+
|
|
190
|
+
# Verify redacted placeholders are NOT matched by patterns
|
|
191
|
+
# (redaction is meant to replace leaks, so patterns should not re-match placeholders)
|
|
192
|
+
for probe_type, (leaked_form, redacted_form) in canonical_probes.items():
|
|
193
|
+
# Redacted placeholders should NOT trigger leak detection
|
|
194
|
+
win_match = re.search(r'[A-Za-z]:\\[^\s]*', redacted_form)
|
|
195
|
+
posix_match = re.search(r'/[A-Za-z_][^\s:/<>|*]*', redacted_form)
|
|
196
|
+
email_match = re.search(EMAIL_PATTERN, redacted_form, re.IGNORECASE)
|
|
197
|
+
|
|
198
|
+
assert not (win_match or posix_match or email_match), \
|
|
199
|
+
f"Redacted form '{redacted_form}' for {probe_type} should not match leak patterns"
|
|
200
|
+
|
|
201
|
+
# Source text check as a secondary signal: verify constants still exist
|
|
202
|
+
digest_path = REPO / 'tools' / 'transcript_digest.py'
|
|
203
|
+
with open(digest_path, 'r', encoding='utf-8') as f:
|
|
204
|
+
source = f.read()
|
|
205
|
+
|
|
206
|
+
required_consts = [
|
|
207
|
+
'REDACTION_PATTERNS', 'EMAIL_PATTERN', 'PATH_PATTERN',
|
|
208
|
+
'REPO_NAME_PATTERN', 'USERNAME_PATTERN'
|
|
209
|
+
]
|
|
210
|
+
for const_name in required_consts:
|
|
211
|
+
if const_name not in source:
|
|
212
|
+
raise AssertionError(
|
|
213
|
+
f"Drift detected: {const_name} missing from transcript_digest.py source. "
|
|
214
|
+
f"Proof cannot verify redaction contract is maintained."
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def test_reasoning_endpoint(base_url: str) -> bool:
|
|
219
|
+
"""Test GET /api/wave/reasoning-tail returns valid reasoning data with proper redaction."""
|
|
220
|
+
import re
|
|
221
|
+
|
|
222
|
+
try:
|
|
223
|
+
# Verify redaction patterns haven't drifted from transcript_digest.py
|
|
224
|
+
verify_redaction_patterns_consistency()
|
|
225
|
+
|
|
226
|
+
url = f"{base_url}/api/wave/reasoning-tail"
|
|
227
|
+
with urllib.request.urlopen(url, timeout=5) as res:
|
|
228
|
+
data = json.loads(res.read().decode('utf-8'))
|
|
229
|
+
|
|
230
|
+
# Validate structure
|
|
231
|
+
assert isinstance(data, dict), "Reasoning response must be dict"
|
|
232
|
+
assert 'available' in data, "Missing 'available' field"
|
|
233
|
+
assert 'agents' in data, "Missing 'agents' field"
|
|
234
|
+
assert 'at' in data, "Missing 'at' timestamp"
|
|
235
|
+
|
|
236
|
+
# Validate agents list
|
|
237
|
+
assert isinstance(data['agents'], list), "agents must be list"
|
|
238
|
+
|
|
239
|
+
if data['available'] and data['agents']:
|
|
240
|
+
# Check agent structure
|
|
241
|
+
for agent in data['agents']:
|
|
242
|
+
assert 'id' in agent, "Agent missing 'id'"
|
|
243
|
+
assert 'phase' in agent, "Agent missing 'phase'"
|
|
244
|
+
assert 'reasoning' in agent, "Agent missing 'reasoning'"
|
|
245
|
+
assert 'activity_age_sec' in agent, "Agent missing 'activity_age_sec'"
|
|
246
|
+
assert 'token_estimate' in agent, "Agent missing 'token_estimate'"
|
|
247
|
+
|
|
248
|
+
# Verify reasoning is redacted per transcript_digest contract:
|
|
249
|
+
# Use the IMPORTED redaction patterns to check for leaks.
|
|
250
|
+
reasoning = agent['reasoning']
|
|
251
|
+
|
|
252
|
+
# Build leak-detection patterns from imported redactor's contract.
|
|
253
|
+
# The patterns below are DIRECT IMPORTS from transcript_digest,
|
|
254
|
+
# ensuring the proof matches the redactor's actual behavior.
|
|
255
|
+
|
|
256
|
+
# Check for unredacted Windows paths
|
|
257
|
+
win_path_matches = re.findall(r'[A-Za-z]:\\[^\s]*', reasoning)
|
|
258
|
+
assert not win_path_matches, \
|
|
259
|
+
f"Reasoning contains unredacted Windows path: {win_path_matches}"
|
|
260
|
+
|
|
261
|
+
# Check for unredacted POSIX paths (both uppercase and lowercase)
|
|
262
|
+
# The redactor's PATH_PATTERN covers /[^/:*<>|]* which includes both cases
|
|
263
|
+
posix_path_matches = re.findall(r'/[^\s/:*<>|][^\s:*<>|]*', reasoning)
|
|
264
|
+
assert not posix_path_matches, \
|
|
265
|
+
f"Reasoning contains unredacted POSIX path: {posix_path_matches}"
|
|
266
|
+
|
|
267
|
+
# Check for unredacted emails
|
|
268
|
+
email_matches = re.findall(EMAIL_PATTERN, reasoning)
|
|
269
|
+
assert not email_matches, \
|
|
270
|
+
f"Reasoning contains unredacted email: {email_matches}"
|
|
271
|
+
|
|
272
|
+
# Check for unredacted API keys/tokens with proper length constraints
|
|
273
|
+
# Import actual REDACTION_PATTERNS and verify each credential type
|
|
274
|
+
for key_type, (pattern, flags) in REDACTION_PATTERNS.items():
|
|
275
|
+
token_matches = re.findall(pattern, reasoning, flags=flags)
|
|
276
|
+
# Filter out false positives: tokens must meet minimum length
|
|
277
|
+
# (e.g., sk- must have 20+ chars per openai_anthropic_key pattern)
|
|
278
|
+
assert not token_matches, \
|
|
279
|
+
f"Reasoning contains unredacted {key_type}: {token_matches}"
|
|
280
|
+
|
|
281
|
+
# Check for unredacted usernames and repo names
|
|
282
|
+
username_matches = re.findall(USERNAME_PATTERN, reasoning)
|
|
283
|
+
assert not username_matches, \
|
|
284
|
+
f"Reasoning contains unredacted username: {username_matches}"
|
|
285
|
+
|
|
286
|
+
repo_matches = re.findall(REPO_NAME_PATTERN, reasoning)
|
|
287
|
+
assert not repo_matches, \
|
|
288
|
+
f"Reasoning contains unredacted repo name: {repo_matches}"
|
|
289
|
+
|
|
290
|
+
print("[PASS] Reasoning endpoint valid (redaction patterns verified)")
|
|
291
|
+
return True
|
|
292
|
+
except Exception as e:
|
|
293
|
+
print(f"[FAIL] Reasoning endpoint failed: {e}")
|
|
294
|
+
return False
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def test_health_check(base_url: str) -> bool:
|
|
298
|
+
"""Test that the dashboard is accessible."""
|
|
299
|
+
try:
|
|
300
|
+
url = f"{base_url}/"
|
|
301
|
+
with urllib.request.urlopen(url, timeout=5) as res:
|
|
302
|
+
assert res.status == 200, f"Dashboard returned {res.status}"
|
|
303
|
+
content = res.read().decode('utf-8')
|
|
304
|
+
assert '<title>' in content, "HTML missing title"
|
|
305
|
+
print("[PASS] Dashboard health check passed")
|
|
306
|
+
return True
|
|
307
|
+
except Exception as e:
|
|
308
|
+
print(f"[FAIL] Dashboard health check failed: {e}")
|
|
309
|
+
return False
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def main():
|
|
313
|
+
"""Run verification suite for UI trio."""
|
|
314
|
+
print("=" * 60)
|
|
315
|
+
print("Verify UI Trio: Gantt Timeline + Audit Tail + Reasoning")
|
|
316
|
+
print("=" * 60)
|
|
317
|
+
|
|
318
|
+
# Fixtures for reference
|
|
319
|
+
fixtures = load_fixtures()
|
|
320
|
+
print(f"\nUsing fixtures from: {FIXTURES_PATH}")
|
|
321
|
+
|
|
322
|
+
# Self-host the dashboard on a free port with isolated fixture state —
|
|
323
|
+
# never depend on a live :8770 instance (it may run different code).
|
|
324
|
+
port = find_free_port()
|
|
325
|
+
base_url = f"http://127.0.0.1:{port}"
|
|
326
|
+
|
|
327
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
328
|
+
state_dir = Path(tmpdir) / 'state'
|
|
329
|
+
state_dir.mkdir(parents=True)
|
|
330
|
+
transcripts_dir = Path(tmpdir) / 'transcripts'
|
|
331
|
+
transcripts_dir.mkdir(parents=True)
|
|
332
|
+
|
|
333
|
+
# Create a fixture agent transcript with test content to enable redaction proof
|
|
334
|
+
fixture_agent_data = {
|
|
335
|
+
"agent_name": "verify-trio-fixture",
|
|
336
|
+
"id": "verify-trio-fixture-001",
|
|
337
|
+
"messages": [
|
|
338
|
+
{
|
|
339
|
+
"type": "text",
|
|
340
|
+
"text": "Testing redaction of paths like C:\\Users\\matt8\\aesop and /c/Users/matt8/aesop"
|
|
341
|
+
},
|
|
342
|
+
{
|
|
343
|
+
"type": "text",
|
|
344
|
+
"text": "Testing email redaction: user@example.com and admin@test.org"
|
|
345
|
+
},
|
|
346
|
+
{
|
|
347
|
+
"type": "text",
|
|
348
|
+
"text": "Testing token patterns and sensitive data"
|
|
349
|
+
}
|
|
350
|
+
]
|
|
351
|
+
}
|
|
352
|
+
(transcripts_dir / "agent-verify-trio-fixture-001.jsonl").write_text(
|
|
353
|
+
json.dumps(fixture_agent_data) + '\n',
|
|
354
|
+
encoding='utf-8'
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
env = os.environ.copy()
|
|
358
|
+
env['PORT'] = str(port)
|
|
359
|
+
env['AESOP_STATE_ROOT'] = str(state_dir)
|
|
360
|
+
env['AESOP_ROOT'] = str(REPO)
|
|
361
|
+
env['AESOP_WEB_DIST'] = str(REPO / 'ui' / 'web' / 'dist')
|
|
362
|
+
env['AESOP_TRANSCRIPTS_ROOT'] = str(transcripts_dir)
|
|
363
|
+
env['AESOP_PROOF_FIXTURES'] = '1'
|
|
364
|
+
|
|
365
|
+
proc = subprocess.Popen(
|
|
366
|
+
[sys.executable, str(REPO / 'ui' / 'serve.py')],
|
|
367
|
+
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
368
|
+
)
|
|
369
|
+
try:
|
|
370
|
+
print("\n--- Health Check ---")
|
|
371
|
+
if not wait_for_server(port):
|
|
372
|
+
print("\nERROR: self-hosted dashboard failed to start")
|
|
373
|
+
return False
|
|
374
|
+
if not test_health_check(base_url):
|
|
375
|
+
print(f"\nERROR: Dashboard not accessible at {base_url}")
|
|
376
|
+
return False
|
|
377
|
+
|
|
378
|
+
print("\n--- API Endpoints ---")
|
|
379
|
+
results = {
|
|
380
|
+
'gantt': test_gantt_endpoint(base_url),
|
|
381
|
+
'audit': test_audit_endpoint(base_url),
|
|
382
|
+
'reasoning': test_reasoning_endpoint(base_url),
|
|
383
|
+
}
|
|
384
|
+
finally:
|
|
385
|
+
proc.terminate()
|
|
386
|
+
try:
|
|
387
|
+
proc.wait(timeout=5)
|
|
388
|
+
except subprocess.TimeoutExpired:
|
|
389
|
+
proc.kill()
|
|
390
|
+
|
|
391
|
+
print("\n" + "=" * 60)
|
|
392
|
+
passed = sum(1 for v in results.values() if v)
|
|
393
|
+
total = len(results)
|
|
394
|
+
print(f"Results: {passed}/{total} tests passed")
|
|
395
|
+
|
|
396
|
+
if passed == total:
|
|
397
|
+
print("[PASS] All UI trio endpoints verified successfully")
|
|
398
|
+
print("=" * 60)
|
|
399
|
+
return True
|
|
400
|
+
else:
|
|
401
|
+
print("[FAIL] Some tests failed")
|
|
402
|
+
print("=" * 60)
|
|
403
|
+
return False
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
if __name__ == '__main__':
|
|
407
|
+
import sys
|
|
408
|
+
success = main()
|
|
409
|
+
sys.exit(0 if success else 1)
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"""Browser-level proof for wave telemetry components.
|
|
2
|
+
|
|
3
|
+
Drives the BUILT React app served by `python ui/serve.py` and exercises
|
|
4
|
+
the wave telemetry components:
|
|
5
|
+
- Overview view: WaveTelemetryProgress (phase + blocker)
|
|
6
|
+
- Work view: WaveTelemetryCost (tokens, top model, ok rate)
|
|
7
|
+
|
|
8
|
+
Tests that the components:
|
|
9
|
+
(a) Mount and render without errors
|
|
10
|
+
(b) Fetch data from /api/wave/telemetry at call time
|
|
11
|
+
(c) Display wave phase, blocker, tokens, model, and ok rate
|
|
12
|
+
(d) Handle missing data gracefully (show error/loading states)
|
|
13
|
+
(e) Have no console errors or failed resources
|
|
14
|
+
|
|
15
|
+
Runs: python tools/verify_wave_telemetry.py (exit 0 = proven, 1 = failed)
|
|
16
|
+
python tools/verify_wave_telemetry.py --allow-skip (exit 0 = proven or skipped)
|
|
17
|
+
"""
|
|
18
|
+
import argparse
|
|
19
|
+
import os
|
|
20
|
+
import shutil
|
|
21
|
+
import socket
|
|
22
|
+
import stat
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
import time
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
REPO = Path(__file__).resolve().parent.parent
|
|
30
|
+
SERVE = REPO / "ui" / "serve.py"
|
|
31
|
+
|
|
32
|
+
SERVER_BOOT_TRIES = int(os.environ.get("AESOP_VERIFY_BOOT_TRIES", "150")) # *0.2s = 30s
|
|
33
|
+
SERVER_BOOT_SLEEP = 0.2
|
|
34
|
+
SEL_TIMEOUT_MS = int(os.environ.get("AESOP_VERIFY_SEL_TIMEOUT_MS", "30000"))
|
|
35
|
+
|
|
36
|
+
# Fixture STATE.md and AUDIT-BACKLOG.md
|
|
37
|
+
FIXTURE_STATE_MD = """# STATE — aesop refinement loop
|
|
38
|
+
|
|
39
|
+
## Phase: `wave-rc.2: build` (2026-07-17, current)
|
|
40
|
+
Current phase focuses on build work.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
FIXTURE_BACKLOG_MD = """# AUDIT-BACKLOG
|
|
44
|
+
|
|
45
|
+
## P0
|
|
46
|
+
|
|
47
|
+
- 🔵 **[ui] Dashboard wave telemetry tile**
|
|
48
|
+
- ⬜ **[test] Fix ledger parser edge cases**
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
FIXTURE_LEDGER = """| timestamp | agent_type | model | duration | tokens_in | tokens_out | verdict |
|
|
52
|
+
|---|---|---|---|---|---|---|
|
|
53
|
+
| 2026-07-17T10:00:00 | Agent | claude-haiku-4-5 | 12 | 500 | 1200 | OK |
|
|
54
|
+
| 2026-07-17T10:05:00 | Agent | claude-sonnet-4-5 | 15 | 800 | 600 | OK |
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _real_console_errors(console_errors, failed_urls):
|
|
59
|
+
"""Drop favicon/urlless-resource noise; surface real broken assets."""
|
|
60
|
+
non_favicon = [u for u in failed_urls if "favicon" not in u.lower()]
|
|
61
|
+
real = []
|
|
62
|
+
for e in console_errors:
|
|
63
|
+
low = e.lower()
|
|
64
|
+
if "favicon" in low:
|
|
65
|
+
continue
|
|
66
|
+
if "failed to load resource" in low and not non_favicon:
|
|
67
|
+
continue
|
|
68
|
+
real.append(e)
|
|
69
|
+
real.extend(f"failed resource: {u}" for u in non_favicon)
|
|
70
|
+
return real
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def free_port():
|
|
74
|
+
s = socket.socket()
|
|
75
|
+
s.bind(("127.0.0.1", 0))
|
|
76
|
+
port = s.getsockname()[1]
|
|
77
|
+
s.close()
|
|
78
|
+
return port
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def copy_dist(root: Path):
|
|
82
|
+
"""Copy the built frontend dist to the fixture root."""
|
|
83
|
+
real_dist = REPO / "ui" / "web" / "dist"
|
|
84
|
+
if real_dist.is_dir():
|
|
85
|
+
shutil.copytree(real_dist, root / "ui" / "web" / "dist")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def build_root():
|
|
89
|
+
"""Fresh temp root with dist + fixture state files."""
|
|
90
|
+
root = Path(tempfile.mkdtemp(prefix="aesop-verify-wave-telemetry-"))
|
|
91
|
+
(root / "state" / "ledger").mkdir(parents=True)
|
|
92
|
+
(root / "transcripts").mkdir(parents=True)
|
|
93
|
+
(root / "dash").mkdir(exist_ok=True)
|
|
94
|
+
|
|
95
|
+
# Write fixture files
|
|
96
|
+
(root / "STATE.md").write_text(FIXTURE_STATE_MD, encoding="utf-8")
|
|
97
|
+
(root / "AUDIT-BACKLOG.md").write_text(FIXTURE_BACKLOG_MD, encoding="utf-8")
|
|
98
|
+
(root / "state" / "ledger" / "OUTCOMES-LEDGER.md").write_text(FIXTURE_LEDGER, encoding="utf-8")
|
|
99
|
+
(root / "dash" / "dash-extra.mjs").write_text("console.log(JSON.stringify([]));\n", encoding="utf-8")
|
|
100
|
+
|
|
101
|
+
copy_dist(root)
|
|
102
|
+
return root
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def start_server(root: Path, port: int):
|
|
106
|
+
"""Start the dashboard server with fixture root."""
|
|
107
|
+
state_root = root / "state"
|
|
108
|
+
real_state = Path.home() / "aesop" / "state"
|
|
109
|
+
if state_root.resolve() == real_state.resolve():
|
|
110
|
+
raise RuntimeError("state dir resolved to real repo state (~aesop/state)")
|
|
111
|
+
|
|
112
|
+
env = dict(os.environ,
|
|
113
|
+
AESOP_ROOT=str(root),
|
|
114
|
+
AESOP_STATE_ROOT=str(state_root),
|
|
115
|
+
AESOP_TRANSCRIPTS_ROOT=str(root / "transcripts"),
|
|
116
|
+
AESOP_WEB_DIST=str(REPO / "ui" / "web" / "dist"),
|
|
117
|
+
AESOP_PROOF_FIXTURES="1",
|
|
118
|
+
AESOP_UI_COLLECT_INTERVAL="0.3",
|
|
119
|
+
PORT=str(port))
|
|
120
|
+
server = subprocess.Popen([sys.executable, str(SERVE)], env=env,
|
|
121
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
122
|
+
for _ in range(SERVER_BOOT_TRIES):
|
|
123
|
+
try:
|
|
124
|
+
socket.create_connection(("127.0.0.1", port), timeout=0.2).close()
|
|
125
|
+
return server
|
|
126
|
+
except OSError:
|
|
127
|
+
time.sleep(SERVER_BOOT_SLEEP)
|
|
128
|
+
server.kill()
|
|
129
|
+
raise RuntimeError("server never came up")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def stop_server(server):
|
|
133
|
+
"""Stop the server."""
|
|
134
|
+
server.terminate()
|
|
135
|
+
try:
|
|
136
|
+
server.wait(timeout=5)
|
|
137
|
+
except subprocess.TimeoutExpired:
|
|
138
|
+
server.kill()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def run_overview_proof(pw, failures):
|
|
142
|
+
"""Test wave telemetry in Overview view."""
|
|
143
|
+
root = build_root()
|
|
144
|
+
port = free_port()
|
|
145
|
+
server = start_server(root, port)
|
|
146
|
+
|
|
147
|
+
browser = pw.chromium.launch(headless=True)
|
|
148
|
+
page = browser.new_page()
|
|
149
|
+
console_errors, failed_urls = [], []
|
|
150
|
+
page.on("console", lambda m: console_errors.append(m.text) if m.type == "error" else None)
|
|
151
|
+
page.on("pageerror", lambda e: console_errors.append(str(e)))
|
|
152
|
+
page.on("response", lambda r: failed_urls.append(r.url) if r.status >= 400 else None)
|
|
153
|
+
|
|
154
|
+
try:
|
|
155
|
+
page.goto(f"http://127.0.0.1:{port}/", wait_until="domcontentloaded")
|
|
156
|
+
|
|
157
|
+
# (a) WaveTelemetryProgress mounts and renders
|
|
158
|
+
try:
|
|
159
|
+
page.wait_for_selector("[data-testid='wave-telemetry-progress']", timeout=SEL_TIMEOUT_MS)
|
|
160
|
+
except Exception as e:
|
|
161
|
+
failures.append(f"(a) Overview: wave telemetry progress never mounted: {e}")
|
|
162
|
+
return
|
|
163
|
+
finally:
|
|
164
|
+
time.sleep(0.4)
|
|
165
|
+
real = _real_console_errors(console_errors, failed_urls)
|
|
166
|
+
if real:
|
|
167
|
+
failures.append(f"(a) Overview: console errors: {real[:2]}")
|
|
168
|
+
|
|
169
|
+
# (b) Progress tile shows wave phase and blocker
|
|
170
|
+
try:
|
|
171
|
+
progress_text = page.inner_text("[data-testid='wave-telemetry-progress']")
|
|
172
|
+
# Should contain some phase info and blocker text
|
|
173
|
+
assert len(progress_text) > 10, f"Progress tile text too short: {progress_text!r}"
|
|
174
|
+
except Exception as e:
|
|
175
|
+
failures.append(f"(b) Overview: wave phase/blocker not displayed: {e}")
|
|
176
|
+
|
|
177
|
+
finally:
|
|
178
|
+
browser.close()
|
|
179
|
+
stop_server(server)
|
|
180
|
+
shutil.rmtree(root, ignore_errors=True)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def run_work_proof(pw, failures):
|
|
184
|
+
"""Test wave telemetry in Work view."""
|
|
185
|
+
root = build_root()
|
|
186
|
+
port = free_port()
|
|
187
|
+
server = start_server(root, port)
|
|
188
|
+
|
|
189
|
+
browser = pw.chromium.launch(headless=True)
|
|
190
|
+
page = browser.new_page()
|
|
191
|
+
console_errors, failed_urls = [], []
|
|
192
|
+
page.on("console", lambda m: console_errors.append(m.text) if m.type == "error" else None)
|
|
193
|
+
page.on("pageerror", lambda e: console_errors.append(str(e)))
|
|
194
|
+
page.on("response", lambda r: failed_urls.append(r.url) if r.status >= 400 else None)
|
|
195
|
+
|
|
196
|
+
try:
|
|
197
|
+
page.goto(f"http://127.0.0.1:{port}/", wait_until="domcontentloaded")
|
|
198
|
+
page.evaluate("location.hash = '#/work'")
|
|
199
|
+
|
|
200
|
+
# (c) WaveTelemetryCost mounts in Work view
|
|
201
|
+
try:
|
|
202
|
+
page.wait_for_selector("[data-testid='wave-telemetry-cost']", timeout=SEL_TIMEOUT_MS)
|
|
203
|
+
except Exception as e:
|
|
204
|
+
failures.append(f"(c) Work: wave telemetry cost never mounted: {e}")
|
|
205
|
+
return
|
|
206
|
+
|
|
207
|
+
# (d) Cost tile shows tokens, model, OK rate
|
|
208
|
+
try:
|
|
209
|
+
cost_text = page.inner_text("[data-testid='wave-telemetry-cost']")
|
|
210
|
+
# Should contain cost-related text (tokens, model, rate)
|
|
211
|
+
assert "Token" in cost_text or "token" in cost_text, \
|
|
212
|
+
f"Tokens label not found in cost tile: {cost_text!r}"
|
|
213
|
+
except Exception as e:
|
|
214
|
+
failures.append(f"(d) Work: tokens/model/rate not displayed: {e}")
|
|
215
|
+
return
|
|
216
|
+
|
|
217
|
+
time.sleep(0.4)
|
|
218
|
+
real = _real_console_errors(console_errors, failed_urls)
|
|
219
|
+
if real:
|
|
220
|
+
failures.append(f"(d) Work: console errors: {real[:2]}")
|
|
221
|
+
|
|
222
|
+
finally:
|
|
223
|
+
browser.close()
|
|
224
|
+
stop_server(server)
|
|
225
|
+
shutil.rmtree(root, ignore_errors=True)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def main():
|
|
229
|
+
parser = argparse.ArgumentParser(description="Browser-level proof for wave telemetry components")
|
|
230
|
+
parser.add_argument("--allow-skip", action="store_true",
|
|
231
|
+
help="Allow skipping if playwright/chromium is unavailable")
|
|
232
|
+
args = parser.parse_args()
|
|
233
|
+
|
|
234
|
+
try:
|
|
235
|
+
from playwright.sync_api import sync_playwright
|
|
236
|
+
except ImportError:
|
|
237
|
+
msg = "playwright missing — run `python -m playwright install chromium`, or pass --allow-skip"
|
|
238
|
+
print(f"SKIP: {msg}" if args.allow_skip else f"FAIL: {msg}")
|
|
239
|
+
return 0 if args.allow_skip else 1
|
|
240
|
+
|
|
241
|
+
failures = []
|
|
242
|
+
with sync_playwright() as pw:
|
|
243
|
+
try:
|
|
244
|
+
pw.chromium.launch(headless=True).close()
|
|
245
|
+
except Exception as e:
|
|
246
|
+
msg = f"chromium unavailable ({e}); run: python -m playwright install chromium"
|
|
247
|
+
print(f"SKIP: {msg}" if args.allow_skip else f"FAIL: {msg}")
|
|
248
|
+
return 0 if args.allow_skip else 1
|
|
249
|
+
|
|
250
|
+
run_overview_proof(pw, failures)
|
|
251
|
+
run_work_proof(pw, failures)
|
|
252
|
+
|
|
253
|
+
if failures:
|
|
254
|
+
print("FAIL:")
|
|
255
|
+
for f in failures:
|
|
256
|
+
print(" -", f)
|
|
257
|
+
return 1
|
|
258
|
+
|
|
259
|
+
print("PROVEN: (a) Overview WaveTelemetryProgress mounts "
|
|
260
|
+
"(b) wave phase/blocker displayed "
|
|
261
|
+
"(c) Work WaveTelemetryCost mounts "
|
|
262
|
+
"(d) tokens/model/rate displayed "
|
|
263
|
+
"(e) no console errors")
|
|
264
|
+
return 0
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
if __name__ == "__main__":
|
|
268
|
+
sys.exit(main())
|