@iseoane/agent-depot 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +117 -3
  3. package/dist/src/app-cli.js +172 -0
  4. package/dist/src/app-flow.js +457 -0
  5. package/dist/src/app-latest.js +63 -0
  6. package/dist/src/app-owned-skills.js +42 -0
  7. package/dist/src/app-recipes.js +116 -0
  8. package/dist/src/app-schema.js +45 -0
  9. package/dist/src/app-version.js +46 -0
  10. package/dist/src/atomic-file.js +26 -0
  11. package/dist/src/cli.js +1256 -0
  12. package/dist/src/git-source.js +409 -0
  13. package/dist/src/path-safety.js +31 -0
  14. package/dist/src/process-runner.js +41 -0
  15. package/dist/src/project-installation.js +1411 -0
  16. package/dist/src/project-manifest.js +511 -0
  17. package/dist/src/skill-adoption.js +77 -0
  18. package/dist/src/skill-discovery.js +467 -0
  19. package/dist/src/skill-hosts.js +103 -0
  20. package/dist/src/skill-install.js +374 -0
  21. package/dist/src/skill-removal.js +119 -0
  22. package/dist/src/skill-update.js +244 -0
  23. package/dist/src/source-state.js +240 -0
  24. package/dist/src/sources.js +391 -0
  25. package/dist/src/tui/adoption-actions.js +144 -0
  26. package/dist/src/tui/app-actions.js +157 -0
  27. package/dist/src/tui/app.js +63 -0
  28. package/dist/src/tui/batch.js +17 -0
  29. package/dist/src/tui/bulk-actions.js +243 -0
  30. package/dist/src/tui/catalog-actions.js +9 -0
  31. package/dist/src/tui/catalog-browse.js +59 -0
  32. package/dist/src/tui/catalog-filter.js +20 -0
  33. package/dist/src/tui/catalog-flow.js +107 -0
  34. package/dist/src/tui/catalog-installs.js +193 -0
  35. package/dist/src/tui/catalog-panel.js +47 -0
  36. package/dist/src/tui/catalog-state.js +92 -0
  37. package/dist/src/tui/catalog-tree.js +35 -0
  38. package/dist/src/tui/catalog-view.js +89 -0
  39. package/dist/src/tui/environment.js +1 -0
  40. package/dist/src/tui/installations-browse.js +136 -0
  41. package/dist/src/tui/installations-mode.js +1 -0
  42. package/dist/src/tui/installations-panel.js +27 -0
  43. package/dist/src/tui/installations-rows.js +11 -0
  44. package/dist/src/tui/installations-tree.js +80 -0
  45. package/dist/src/tui/installations-view.js +154 -0
  46. package/dist/src/tui/installations.js +172 -0
  47. package/dist/src/tui/keys.js +31 -0
  48. package/dist/src/tui/manage-actions.js +174 -0
  49. package/dist/src/tui/manage-panel.js +65 -0
  50. package/dist/src/tui/mode-keys.js +61 -0
  51. package/dist/src/tui/panel-parts.js +23 -0
  52. package/dist/src/tui/render.js +8 -0
  53. package/dist/src/tui/sources-mode.js +17 -0
  54. package/dist/src/tui/sources-view.js +276 -0
  55. package/dist/src/tui/theme.js +29 -0
  56. package/dist/src/tui/tree-navigation.js +68 -0
  57. package/dist/src/tui/tree.js +53 -0
  58. package/dist/src/tui/unmanaged-actions.js +178 -0
  59. package/dist/src/tui/updates-flow.js +89 -0
  60. package/dist/src/tui/updates-keys.js +104 -0
  61. package/dist/src/tui/updates-mode.js +1 -0
  62. package/dist/src/tui/updates-model.js +48 -0
  63. package/dist/src/tui/updates-panel.js +44 -0
  64. package/dist/src/tui/updates-view.js +101 -0
  65. package/dist/src/tui/updates.js +230 -0
  66. package/dist/src/tui/view-state.js +42 -0
  67. package/dist/src/tui/window.js +31 -0
  68. package/dist/src/unmanaged-removal.js +60 -0
  69. package/dist/src/update-batch.js +376 -0
  70. package/dist/src/update-flow.js +135 -0
  71. package/dist/src/usage-error.js +7 -0
  72. package/dist/src/user-global-skill-inventory.js +565 -0
  73. package/dist/src/user-method.js +14 -0
  74. package/package.json +7 -4
  75. package/skills/agent-depot-apprecipe/SKILL.md +44 -0
  76. package/skills/agent-depot-apprecipe/assets/example.md +25 -0
  77. package/skills/doctor-md-agents/scripts/test_collect_sessions.py +0 -780
  78. package/skills/doctor-md-agents/scripts/test_extract_rules.py +0 -530
  79. package/skills/doctor-md-agents/scripts/test_render_report.py +0 -383
  80. package/skills/doctor-md-agents/scripts/test_scan_corrections.py +0 -343
  81. package/skills/doctor-md-agents/scripts/test_verify_citations.py +0 -236
  82. package/skills/doctor-md-agents/scripts/testdata/old_extract_rules_pre_t1.py.txt +0 -507
  83. package/skills/doctor-md-skill/scripts/test_collect_sessions.py +0 -593
  84. package/skills/doctor-md-skill/scripts/test_detect_machine_sessions.py +0 -181
  85. package/skills/doctor-md-skill/scripts/test_render_report.py +0 -241
  86. package/skills/doctor-md-skill/scripts/test_verify_citations.py +0 -196
@@ -1,181 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Tests for detect_machine_sessions.py."""
3
-
4
- import json
5
- import tempfile
6
- import unittest
7
- from datetime import datetime, timezone
8
- from pathlib import Path
9
-
10
- import detect_machine_sessions as dms
11
-
12
-
13
- def now_iso():
14
- return datetime.now(timezone.utc).isoformat()
15
-
16
-
17
- def write_claude_session(claude_home, project, session_id, user_texts, extra_record=None):
18
- """Write a minimal Claude Code JSONL session with one user turn per
19
- entry in `user_texts`, each followed by a matching assistant reply so
20
- the session looks like a normal back-and-forth."""
21
- project_dir = Path(claude_home) / "projects" / project
22
- project_dir.mkdir(parents=True, exist_ok=True)
23
- path = project_dir / f"{session_id}.jsonl"
24
- lines = []
25
- for text in user_texts:
26
- lines.append({
27
- "type": "user", "sessionId": session_id, "cwd": "/repo", "timestamp": now_iso(),
28
- "message": {"role": "user", "content": [{"type": "text", "text": text}]},
29
- })
30
- lines.append({
31
- "type": "assistant", "sessionId": session_id, "cwd": "/repo", "timestamp": now_iso(),
32
- "message": {"role": "assistant", "id": f"{session_id}-{len(lines)}",
33
- "content": [{"type": "text", "text": "ok, working on it"}]},
34
- })
35
- if extra_record:
36
- lines.append(extra_record)
37
- path.write_text("\n".join(json.dumps(line) for line in lines), encoding="utf-8")
38
- return path
39
-
40
-
41
- class LongFirstTurnTests(unittest.TestCase):
42
- def test_long_first_turn_is_flagged(self):
43
- with tempfile.TemporaryDirectory() as tmp:
44
- claude_home = Path(tmp) / "claude"
45
- write_claude_session(claude_home, "proj", "s-long", ["x" * 5000])
46
-
47
- records = dms.collect_candidates(
48
- {"claude_home": claude_home, "codex_home": Path(tmp) / "nope",
49
- "pi_home": Path(tmp) / "nope", "grok_home": Path(tmp) / "nope",
50
- "zcode_home": Path(tmp) / "nope"},
51
- datetime.now(timezone.utc).replace(year=2000),
52
- )
53
- flagged, _templates, stats = dms.flag_sessions(records, 1200, 3, 160)
54
-
55
- self.assertEqual(stats["flagged_long_first_turn"], 1)
56
- keys = [f["session_key"] for f in flagged]
57
- self.assertIn("claude:s-long", keys)
58
- entry = next(f for f in flagged if f["session_key"] == "claude:s-long")
59
- self.assertIn("long_first_turn", entry["reasons"])
60
-
61
- def test_short_first_turn_is_not_flagged(self):
62
- with tempfile.TemporaryDirectory() as tmp:
63
- claude_home = Path(tmp) / "claude"
64
- write_claude_session(claude_home, "proj", "s-short", ["please fix the login bug"])
65
-
66
- records = dms.collect_candidates(
67
- {"claude_home": claude_home, "codex_home": Path(tmp) / "nope",
68
- "pi_home": Path(tmp) / "nope", "grok_home": Path(tmp) / "nope",
69
- "zcode_home": Path(tmp) / "nope"},
70
- datetime.now(timezone.utc).replace(year=2000),
71
- )
72
- flagged, _templates, _stats = dms.flag_sessions(records, 1200, 3, 160)
73
-
74
- self.assertEqual(flagged, [])
75
-
76
-
77
- class TemplateDetectionTests(unittest.TestCase):
78
- def test_same_prefix_in_three_distinct_sessions_is_flagged_as_template(self):
79
- with tempfile.TemporaryDirectory() as tmp:
80
- claude_home = Path(tmp) / "claude"
81
- template_text = "Score this transcript against the efficiency rubric and report a label."
82
- for i in range(3):
83
- write_claude_session(claude_home, "proj", f"s{i}", [template_text])
84
- write_claude_session(claude_home, "proj", "s-oneoff", ["completely unrelated one-off request"])
85
-
86
- records = dms.collect_candidates(
87
- {"claude_home": claude_home, "codex_home": Path(tmp) / "nope",
88
- "pi_home": Path(tmp) / "nope", "grok_home": Path(tmp) / "nope",
89
- "zcode_home": Path(tmp) / "nope"},
90
- datetime.now(timezone.utc).replace(year=2000),
91
- )
92
- flagged, dropped_templates, stats = dms.flag_sessions(records, 1200, 3, 160)
93
-
94
- self.assertEqual(stats["flagged_templated"], 3)
95
- flagged_keys = {f["session_key"] for f in flagged}
96
- self.assertEqual(flagged_keys, {"claude:s0", "claude:s1", "claude:s2"})
97
- self.assertNotIn("claude:s-oneoff", flagged_keys)
98
- self.assertEqual(len(dropped_templates), 1)
99
- self.assertEqual(dropped_templates[0]["sessions"], 3)
100
-
101
- def test_same_text_repeated_in_one_session_is_not_a_template(self):
102
- with tempfile.TemporaryDirectory() as tmp:
103
- claude_home = Path(tmp) / "claude"
104
- repeated = "please continue with the next step"
105
- write_claude_session(claude_home, "proj", "s-repeat", [repeated, repeated, repeated])
106
-
107
- records = dms.collect_candidates(
108
- {"claude_home": claude_home, "codex_home": Path(tmp) / "nope",
109
- "pi_home": Path(tmp) / "nope", "grok_home": Path(tmp) / "nope",
110
- "zcode_home": Path(tmp) / "nope"},
111
- datetime.now(timezone.utc).replace(year=2000),
112
- )
113
- flagged, dropped_templates, stats = dms.flag_sessions(records, 1200, 3, 160)
114
-
115
- self.assertEqual(stats["flagged_templated"], 0)
116
- self.assertEqual(dropped_templates, [])
117
- self.assertEqual(flagged, [])
118
-
119
-
120
- class SentinelTests(unittest.TestCase):
121
- def test_session_carrying_sentinel_is_flagged(self):
122
- with tempfile.TemporaryDirectory() as tmp:
123
- claude_home = Path(tmp) / "claude"
124
- write_claude_session(
125
- claude_home, "proj", "s-self",
126
- [dms.MACHINE_SESSION_SENTINEL + "\nScore this transcript."],
127
- )
128
-
129
- records = dms.collect_candidates(
130
- {"claude_home": claude_home, "codex_home": Path(tmp) / "nope",
131
- "pi_home": Path(tmp) / "nope", "grok_home": Path(tmp) / "nope",
132
- "zcode_home": Path(tmp) / "nope"},
133
- datetime.now(timezone.utc).replace(year=2000),
134
- )
135
- flagged, _templates, stats = dms.flag_sessions(records, 1200, 3, 160)
136
-
137
- self.assertEqual(stats["flagged_self_session_sentinel"], 1)
138
- entry = next(f for f in flagged if f["session_key"] == "claude:s-self")
139
- self.assertIn("self_session_sentinel", entry["reasons"])
140
-
141
- def test_session_without_sentinel_is_not_flagged_for_it(self):
142
- with tempfile.TemporaryDirectory() as tmp:
143
- claude_home = Path(tmp) / "claude"
144
- write_claude_session(claude_home, "proj", "s-normal", ["a perfectly normal human request"])
145
-
146
- records = dms.collect_candidates(
147
- {"claude_home": claude_home, "codex_home": Path(tmp) / "nope",
148
- "pi_home": Path(tmp) / "nope", "grok_home": Path(tmp) / "nope",
149
- "zcode_home": Path(tmp) / "nope"},
150
- datetime.now(timezone.utc).replace(year=2000),
151
- )
152
- flagged, _templates, stats = dms.flag_sessions(records, 1200, 3, 160)
153
-
154
- self.assertEqual(stats["flagged_self_session_sentinel"], 0)
155
- self.assertEqual(flagged, [])
156
-
157
-
158
- class CliTests(unittest.TestCase):
159
- def test_main_writes_machine_sessions_json(self):
160
- with tempfile.TemporaryDirectory() as tmp:
161
- claude_home = Path(tmp) / "claude"
162
- write_claude_session(claude_home, "proj", "s-long", ["y" * 3000])
163
- out_dir = Path(tmp) / "out"
164
-
165
- dms.main([
166
- "--out", str(out_dir),
167
- "--claude-home", str(claude_home),
168
- "--codex-home", str(Path(tmp) / "nope"),
169
- "--pi-home", str(Path(tmp) / "nope"),
170
- "--grok-home", str(Path(tmp) / "nope"),
171
- "--zcode-home", str(Path(tmp) / "nope"),
172
- ])
173
-
174
- report = json.loads((out_dir / "machine_sessions.json").read_text())
175
- self.assertEqual(report["stats"]["sessions_scanned"], 1)
176
- self.assertEqual(report["stats"]["sessions_flagged"], 1)
177
- self.assertEqual(report["sentinel"], dms.MACHINE_SESSION_SENTINEL)
178
-
179
-
180
- if __name__ == "__main__":
181
- unittest.main()
@@ -1,241 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Tests for doctor-md-skill report rendering."""
3
-
4
- import unittest
5
- from pathlib import Path
6
- from unittest.mock import patch
7
-
8
- from render_report import (
9
- embedded_diffs_script,
10
- format_generated_at,
11
- open_report,
12
- parse_args,
13
- render_page,
14
- )
15
-
16
-
17
- class ReportRendererTests(unittest.TestCase):
18
- def test_skill_startup_contract_is_centralized(self):
19
- skill_root = Path(__file__).resolve().parent.parent
20
- skill_text = (skill_root / "SKILL.md").read_text()
21
- harness_text = (
22
- skill_root / "references" / "supported-harnesses.md"
23
- ).read_text()
24
-
25
- self.assertIn(
26
- "$SKILL_ROOT/references/supported-harnesses.md",
27
- skill_text,
28
- )
29
- self.assertIn("Conversations in this repository", skill_text)
30
- self.assertIn("All conversations", skill_text)
31
- self.assertIn("Choose projects to analyze", skill_text)
32
- self.assertIn(
33
- "Project skills + global skills",
34
- skill_text,
35
- )
36
- self.assertIn("Project skills only", skill_text)
37
- self.assertIn(
38
- "Process datasets of 50 transcripts or fewer in a single batch",
39
- skill_text,
40
- )
41
- self.assertIn(
42
- "For datasets with more than 50 transcripts, use parallel batches "
43
- "(20 transcripts per batch recommended)",
44
- skill_text,
45
- )
46
- self.assertNotIn("--claude-home PATH", skill_text)
47
- self.assertIn("| Claude Code | `claude` |", harness_text)
48
- self.assertIn("| Codex | `codex` |", harness_text)
49
- self.assertIn("stop before creating a report directory", harness_text)
50
-
51
- def test_code_diffs_follow_os_theme(self):
52
- bundle = embedded_diffs_script()
53
-
54
- self.assertIn('themeType:"system"', bundle)
55
- self.assertIn(
56
- 'theme:{dark:"pierre-dark",light:"pierre-light"}',
57
- bundle,
58
- )
59
-
60
- def test_report_follows_os_theme(self):
61
- page = render_page({
62
- "scores": {
63
- "efficiency": 1.0,
64
- "code_quality": 1.0,
65
- "skill_coverage": 1.0,
66
- "overall": 1.0,
67
- },
68
- })
69
-
70
- self.assertIn('<meta name="color-scheme" content="light dark">', page)
71
- self.assertIn("@media (prefers-color-scheme: dark)", page)
72
- self.assertIn("--page-bg: #0f0d14", page)
73
- self.assertIn("background: var(--surface)", page)
74
- self.assertIn(
75
- "--mono-font: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace",
76
- page,
77
- )
78
- self.assertIn("--diffs-font-family: var(--mono-font)", page)
79
- self.assertIn("--diffs-header-font-family: var(--mono-font)", page)
80
-
81
- def test_report_carries_no_product_advertising(self):
82
- report = {
83
- "title": "Agent Skill Report",
84
- "generated_at": "2026-08-25T00:00:00Z",
85
- "harness": "codex",
86
- "handle": "example",
87
- "stats": {
88
- "sessions_analyzed": 1,
89
- "sessions_scanned": 1,
90
- "skills_found": 1,
91
- "skills_used": 1,
92
- "window_days": 45,
93
- },
94
- "scores": {
95
- "efficiency": 1.0,
96
- "code_quality": 1.0,
97
- "skill_coverage": 1.0,
98
- "overall": 1.0,
99
- },
100
- "top_findings": ["No material waste detected."],
101
- "suggestions": [],
102
- }
103
-
104
- page = render_page(report)
105
-
106
- self.assertNotIn("factories-footer", page)
107
- self.assertNotIn("Request access", page)
108
- self.assertNotIn("cta_url", page)
109
- self.assertIn(">Share</button>", page)
110
- self.assertNotIn("all analysis ran locally", page)
111
- self.assertIn(
112
- "Generated August 25, 2026 at 12:00 AM UTC &middot; harness: codex",
113
- page,
114
- )
115
-
116
- def test_generated_timestamp_formatting(self):
117
- self.assertEqual(
118
- format_generated_at("2026-08-27T22:06:10.421941+00"),
119
- "August 27, 2026 at 10:06 PM UTC",
120
- )
121
- self.assertEqual(format_generated_at("not-a-date"), "not-a-date")
122
-
123
- def test_open_report_uses_default_browser_with_file_uri(self):
124
- report_path = Path("/tmp/skill doctor/report.html")
125
- args = parse_args([str(report_path), "--open"])
126
-
127
- self.assertEqual(args.report_path, str(report_path))
128
- self.assertTrue(args.open_browser)
129
-
130
- with patch("render_report.webbrowser.open", return_value=True) as browser_open:
131
- self.assertTrue(open_report(report_path))
132
-
133
- browser_open.assert_called_once_with(
134
- report_path.absolute().as_uri(),
135
- new=2,
136
- )
137
-
138
- with patch("render_report.webbrowser.open", side_effect=OSError):
139
- self.assertFalse(open_report(report_path))
140
-
141
- def test_share_card_uses_doctor_md_skill_attribution(self):
142
- page = render_page({
143
- "scores": {
144
- "efficiency": 1.0,
145
- "code_quality": 1.0,
146
- "skill_coverage": 1.0,
147
- "overall": 1.0,
148
- },
149
- })
150
-
151
- self.assertIn(
152
- '"stamp": ["Generated with doctor-md-skill", '
153
- '"local run"]',
154
- page,
155
- )
156
- self.assertIn('"eyebrow": "doctor-md-skill"', page)
157
- self.assertIn("text('# ' + CARD.eyebrow", page)
158
-
159
- def test_report_metric_lines_animate_like_skill_doctor_landing_page(self):
160
- page = render_page({
161
- "scores": {
162
- "efficiency": 0.75,
163
- "code_quality": 0.93,
164
- "skill_coverage": 0.74,
165
- "overall": 0.82,
166
- },
167
- })
168
-
169
- self.assertIn(
170
- "animation: report-fill 700ms "
171
- "cubic-bezier(0.22, 1, 0.36, 1) var(--metric-delay) both",
172
- page,
173
- )
174
- self.assertIn("@keyframes report-fill", page)
175
- self.assertIn("from { transform: scaleX(0); }", page)
176
- self.assertIn("to { transform: scaleX(1); }", page)
177
- self.assertIn("width:75%;--metric-delay:180ms", page)
178
- self.assertIn("width:93%;--metric-delay:290ms", page)
179
- self.assertIn("width:74%;--metric-delay:400ms", page)
180
- self.assertIn("@media (prefers-reduced-motion: reduce)", page)
181
- self.assertIn(".bar-fill { animation: none; }", page)
182
-
183
- def test_skill_output_uses_report_path_only(self):
184
- skill_path = Path(__file__).resolve().parent.parent / "SKILL.md"
185
- skill_text = skill_path.read_text()
186
-
187
- self.assertIn(
188
- 'render_report.py" "$REPORT_DIR/report.json" --open',
189
- skill_text,
190
- )
191
- self.assertIn(
192
- "- Your agent skill report: file://$REPORT_DIR/report.html",
193
- skill_text,
194
- )
195
- self.assertNotIn("cta_url", skill_text)
196
- self.assertNotIn("[View in browser]", skill_text)
197
-
198
- def test_skill_edits_only_use_failed_conversations(self):
199
- skill_path = Path(__file__).resolve().parent.parent / "SKILL.md"
200
- skill_text = skill_path.read_text()
201
-
202
- self.assertIn(
203
- "`raw_efficiency` = mean of efficiency scores across all **verified** sessions",
204
- skill_text,
205
- )
206
- self.assertIn(
207
- "`curve(score) = 0.5 + 0.5 * score`",
208
- skill_text,
209
- )
210
- self.assertIn(
211
- "`overall = 0.5 * efficiency + 0.35 * code_quality + "
212
- "0.15 * skill_coverage.`",
213
- skill_text,
214
- )
215
- self.assertIn(
216
- "from each conversation's raw, uncurved, **verified** scorer results",
217
- skill_text,
218
- )
219
- self.assertIn(
220
- "Use only `failed_conversations` as evidence for "
221
- "skill-improvement suggestions and draft skill edits",
222
- skill_text,
223
- )
224
- self.assertIn("Only `verified: true` results are aggregated", skill_text)
225
-
226
- def test_report_renders_letter_grade(self):
227
- page = render_page({
228
- "scores": {
229
- "efficiency": 0.7,
230
- "code_quality": 0.7,
231
- "skill_coverage": 0.8,
232
- "overall": 0.7,
233
- },
234
- })
235
-
236
- self.assertIn('<div class="grade">C-</div>', page)
237
- self.assertIn('<div class="grade-label">overall 70</div>', page)
238
-
239
-
240
- if __name__ == "__main__":
241
- unittest.main()
@@ -1,196 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Tests for verify_citations.py."""
3
-
4
- import json
5
- import tempfile
6
- import unittest
7
- from pathlib import Path
8
- from unittest import mock
9
-
10
- import verify_citations as vc
11
-
12
-
13
- def write_transcript(path, text):
14
- path.parent.mkdir(parents=True, exist_ok=True)
15
- path.write_text(text, encoding="utf-8")
16
-
17
-
18
- def write_results(path, results):
19
- path.parent.mkdir(parents=True, exist_ok=True)
20
- path.write_text(json.dumps({"results": results}), encoding="utf-8")
21
-
22
-
23
- def run_verify(results_path, out_dir, **kwargs):
24
- argv = [str(results_path), "--out", str(out_dir)]
25
- for key, value in kwargs.items():
26
- argv.extend([f"--{key.replace('_', '-')}", str(value)])
27
- vc.main(argv)
28
- return json.loads((Path(out_dir) / "results.verified.json").read_text())
29
-
30
-
31
- class WhitespaceFoldingTests(unittest.TestCase):
32
- def test_quote_matching_only_after_whitespace_folding_verifies(self):
33
- with tempfile.TemporaryDirectory() as tmp:
34
- tmp = Path(tmp)
35
- transcript = tmp / "t1.md"
36
- write_transcript(
37
- transcript,
38
- "[user] Please remember: always use pnpm\nfor the frontend build.\n",
39
- )
40
- results_path = tmp / "results.json"
41
- write_results(results_path, [
42
- {"scorer": "efficiency", "transcript_path": str(transcript), "label": "highly_efficient",
43
- "score": 1.0, "quote": "always use pnpm\n for the frontend build.",
44
- "reason": "the agent followed the stated convention directly", "session_id": "s1"},
45
- ])
46
- report = run_verify(results_path, tmp / "out")
47
- self.assertTrue(report["results"][0]["verified"])
48
- self.assertIsNone(report["results"][0]["reason_code"])
49
-
50
- def test_paraphrase_does_not_verify(self):
51
- with tempfile.TemporaryDirectory() as tmp:
52
- tmp = Path(tmp)
53
- transcript = tmp / "t1.md"
54
- write_transcript(transcript, "[user] Reminder: always use pnpm for the frontend build.\n")
55
- results_path = tmp / "results.json"
56
- write_results(results_path, [
57
- {"scorer": "code-quality", "transcript_path": str(transcript), "label": "block",
58
- "score": 0.2,
59
- "quote": "the agent should always prefer pnpm instead of npm here",
60
- "reason": "paraphrased, not quoted", "session_id": "s1"},
61
- ])
62
- report = run_verify(results_path, tmp / "out")
63
- self.assertFalse(report["results"][0]["verified"])
64
- self.assertEqual(report["results"][0]["reason_code"], "not_found")
65
-
66
- def test_case_difference_does_not_verify(self):
67
- # Explicit assertion that the check is case-sensitive by design.
68
- with tempfile.TemporaryDirectory() as tmp:
69
- tmp = Path(tmp)
70
- transcript = tmp / "t1.md"
71
- write_transcript(transcript, "[user] reminder: always use pnpm here for the build.\n")
72
- results_path = tmp / "results.json"
73
- write_results(results_path, [
74
- {"scorer": "efficiency", "transcript_path": str(transcript), "label": "mostly_inefficient",
75
- "score": 0.4, "quote": "ALWAYS USE PNPM HERE",
76
- "reason": "case differs from transcript", "session_id": "s1"},
77
- ])
78
- report = run_verify(results_path, tmp / "out")
79
- result = report["results"][0]
80
- self.assertFalse(result["verified"])
81
- self.assertEqual(result["reason_code"], "not_found")
82
- # Prove the ONLY thing wrong is case: the lowercase form does verify.
83
- self.assertIn(
84
- vc.fold_whitespace("always use pnpm here"),
85
- vc.fold_whitespace(transcript.read_text(encoding="utf-8")),
86
- )
87
-
88
-
89
- class ReasonCodeTests(unittest.TestCase):
90
- def test_too_short_quote_gets_its_own_code(self):
91
- with tempfile.TemporaryDirectory() as tmp:
92
- tmp = Path(tmp)
93
- transcript = tmp / "t1.md"
94
- write_transcript(transcript, "[user] Always use pnpm here.\n")
95
- results_path = tmp / "results.json"
96
- write_results(results_path, [
97
- {"scorer": "efficiency", "transcript_path": str(transcript), "label": "highly_efficient",
98
- "score": 1.0, "quote": "pnpm", "reason": "too short to prove anything",
99
- "session_id": "s1"},
100
- ])
101
- report = run_verify(results_path, tmp / "out")
102
- result = report["results"][0]
103
- self.assertFalse(result["verified"])
104
- self.assertEqual(result["reason_code"], "quote_too_short")
105
-
106
- def test_missing_quote_field_gets_quote_missing(self):
107
- with tempfile.TemporaryDirectory() as tmp:
108
- tmp = Path(tmp)
109
- transcript = tmp / "t1.md"
110
- write_transcript(transcript, "[user] Always use pnpm here for the frontend.\n")
111
- results_path = tmp / "results.json"
112
- write_results(results_path, [
113
- {"scorer": "efficiency", "transcript_path": str(transcript), "label": "highly_efficient",
114
- "score": 1.0, "quote": "", "reason": "no citation given", "session_id": "s1"},
115
- ])
116
- report = run_verify(results_path, tmp / "out")
117
- self.assertEqual(report["results"][0]["reason_code"], "quote_missing")
118
-
119
- def test_missing_transcript_yields_transcript_unreadable_not_not_found(self):
120
- with tempfile.TemporaryDirectory() as tmp:
121
- tmp = Path(tmp)
122
- missing_transcript = tmp / "does-not-exist.md"
123
- results_path = tmp / "results.json"
124
- write_results(results_path, [
125
- {"scorer": "code-quality", "transcript_path": str(missing_transcript), "label": "block",
126
- "score": 0.2, "quote": "always use pnpm here for the build",
127
- "reason": "transcript file is gone", "session_id": "s1"},
128
- ])
129
- report = run_verify(results_path, tmp / "out")
130
- result = report["results"][0]
131
- self.assertFalse(result["verified"])
132
- self.assertEqual(result["reason_code"], "transcript_unreadable")
133
- self.assertEqual(report["stats"]["by_reason_code"]["transcript_unreadable"], 1)
134
- self.assertEqual(report["stats"]["by_reason_code"]["not_found"], 0)
135
-
136
-
137
- class CachingTests(unittest.TestCase):
138
- def test_shared_transcript_is_read_from_disk_once(self):
139
- with tempfile.TemporaryDirectory() as tmp:
140
- tmp = Path(tmp)
141
- transcript = tmp / "shared.md"
142
- write_transcript(transcript, "[user] Always use pnpm for the frontend build, please.\n")
143
- results_path = tmp / "results.json"
144
- write_results(results_path, [
145
- {"scorer": "efficiency", "transcript_path": str(transcript), "label": "highly_efficient",
146
- "score": 1.0, "quote": "Always use pnpm for the frontend build",
147
- "reason": "first citation", "session_id": "s1"},
148
- {"scorer": "code-quality", "transcript_path": str(transcript), "label": "approve",
149
- "score": 1.0, "quote": "for the frontend build, please",
150
- "reason": "second citation, same transcript", "session_id": "s1"},
151
- ])
152
- with mock.patch.object(vc, "load_transcript_text", wraps=vc.load_transcript_text) as spy:
153
- report = run_verify(results_path, tmp / "out")
154
- self.assertEqual(spy.call_count, 1)
155
- self.assertTrue(all(r["verified"] for r in report["results"]))
156
- self.assertEqual(report["stats"]["by_transcript"][str(transcript)]["total"], 2)
157
-
158
-
159
- class MalformedInputTests(unittest.TestCase):
160
- def test_invalid_json_exits_non_zero(self):
161
- with tempfile.TemporaryDirectory() as tmp:
162
- tmp = Path(tmp)
163
- results_path = tmp / "results.json"
164
- results_path.write_text("{not json", encoding="utf-8")
165
- with self.assertRaises(SystemExit) as ctx:
166
- vc.main([str(results_path), "--out", str(tmp / "out")])
167
- self.assertNotEqual(ctx.exception.code, 0)
168
-
169
- def test_missing_results_key_exits_non_zero(self):
170
- with tempfile.TemporaryDirectory() as tmp:
171
- tmp = Path(tmp)
172
- results_path = tmp / "results.json"
173
- results_path.write_text(json.dumps({"nope": []}), encoding="utf-8")
174
- with self.assertRaises(SystemExit) as ctx:
175
- vc.main([str(results_path), "--out", str(tmp / "out")])
176
- self.assertNotEqual(ctx.exception.code, 0)
177
-
178
- def test_verification_failures_still_exit_zero(self):
179
- # Failing verification is data, not a script error -- main() must
180
- # not raise SystemExit at all on a structurally valid input file.
181
- with tempfile.TemporaryDirectory() as tmp:
182
- tmp = Path(tmp)
183
- results_path = tmp / "results.json"
184
- write_results(results_path, [
185
- {"scorer": "code-quality", "transcript_path": str(tmp / "missing.md"), "label": "block",
186
- "score": 0.2, "quote": "this quote will not verify at all",
187
- "reason": "bad citation", "session_id": "s1"},
188
- ])
189
- try:
190
- vc.main([str(results_path), "--out", str(tmp / "out")])
191
- except SystemExit as exc:
192
- self.fail(f"main() exited with {exc.code} on a merely-unverified result")
193
-
194
-
195
- if __name__ == "__main__":
196
- unittest.main()