@malloy-publisher/skills 0.1.38 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/eval-answer/SKILL.md +1 -1
- package/skills/eval-answer/reference/ledger-schema.md +4 -3
- package/skills/eval-answer/scripts/ledger.py +5 -1
- package/skills/eval-answer/scripts/publisher_rest.py +39 -0
- package/skills/eval-diagnose/SKILL.md +43 -18
- package/skills/eval-diagnose/scripts/cluster_failures.py +2 -0
- package/skills/eval-diagnose/scripts/diagnose.py +11 -4
- package/skills/eval-import/SKILL.md +1 -1
- package/skills/eval-improve/SKILL.md +46 -6
- package/skills/eval-improve/scripts/improve.py +45 -16
- package/skills/eval-improve/scripts/improve_test.py +35 -0
- package/skills/eval-judge/SKILL.md +1 -1
- package/skills/eval-loop/SKILL.md +3 -2
- package/skills/eval-loop/reference/acceptance-check.md +12 -0
- package/skills/eval-loop/scripts/agent_harness.py +38 -1
- package/skills/eval-loop/scripts/agent_harness_test.py +12 -1
- package/skills/eval-loop/scripts/package_skills_test.py +263 -0
- package/skills/eval-loop/scripts/run_baseline.py +185 -16
- package/skills/eval-report/SKILL.md +1 -1
- package/skills/malloy-analysis/SKILL.md +1 -1
- package/skills/malloy-analysis-report/SKILL.md +1 -1
- package/skills/malloy-charts/SKILL.md +1 -1
- package/skills/malloy-dashboards/SKILL.md +1 -1
- package/skills/malloy-define/SKILL.md +1 -1
- package/skills/malloy-discover/SKILL.md +1 -1
- package/skills/malloy-document/SKILL.md +1 -1
- package/skills/malloy-getting-started/SKILL.md +2 -2
- package/skills/malloy-gotchas-modeling/SKILL.md +1 -1
- package/skills/malloy-html-data-apps/SKILL.md +1 -1
- package/skills/malloy-lookml-review/SKILL.md +1 -1
- package/skills/malloy-materialization/SKILL.md +1 -1
- package/skills/malloy-model/SKILL.md +1 -1
- package/skills/malloy-model-as-you-go/SKILL.md +1 -1
- package/skills/malloy-modeling/SKILL.md +1 -1
- package/skills/malloy-notebooks/SKILL.md +1 -1
- package/skills/malloy-phrase-detection/SKILL.md +1 -1
- package/skills/malloy-powerbi-review/SKILL.md +1 -1
- package/skills/malloy-publish/SKILL.md +1 -1
- package/skills/malloy-queries/SKILL.md +1 -1
- package/skills/malloy-reports/SKILL.md +1 -1
- package/skills/malloy-review/SKILL.md +1 -1
- package/skills/malloy-run-agent/SKILL.md +58 -0
- package/skills/malloy-source-unreachable/SKILL.md +1 -1
|
@@ -346,7 +346,9 @@ def build_workspace(skills: Iterable[str],
|
|
|
346
346
|
skills_root: pathlib.Path | Iterable[pathlib.Path],
|
|
347
347
|
mcp_url: str | None, prefix: str = "agent-",
|
|
348
348
|
depth: int = 1,
|
|
349
|
-
mcp_server: str = "publisher"
|
|
349
|
+
mcp_server: str = "publisher",
|
|
350
|
+
package_skills_dir: pathlib.Path | None = None
|
|
351
|
+
) -> pathlib.Path:
|
|
350
352
|
"""A scratch cwd holding `.claude/skills/` and an MCP config.
|
|
351
353
|
|
|
352
354
|
COPIED, not symlinked. A symlink loads SKILL.md fine -- the skills mechanism
|
|
@@ -377,6 +379,41 @@ def build_workspace(skills: Iterable[str],
|
|
|
377
379
|
else list(skills_root))
|
|
378
380
|
for name in present:
|
|
379
381
|
shutil.copytree(find_skill(name, roots), dest / name, symlinks=False)
|
|
382
|
+
# A package's own skills, installed last so one whose name matches a
|
|
383
|
+
# manifest skill REPLACES it -- the same shadowing the server does when it
|
|
384
|
+
# serves them, so the installed path and the tool path agree about which
|
|
385
|
+
# guide is in force.
|
|
386
|
+
#
|
|
387
|
+
# Only the package's `skills/` subtree is copied, never the package. An
|
|
388
|
+
# eval set lives at `<package>/evals/`, so copying the package would hand
|
|
389
|
+
# the answerer the goldens.
|
|
390
|
+
if package_skills_dir is not None:
|
|
391
|
+
if not package_skills_dir.is_dir():
|
|
392
|
+
raise FileNotFoundError(
|
|
393
|
+
f"--package-skills=install was asked for but "
|
|
394
|
+
f"{package_skills_dir} is not a directory. Installing nothing "
|
|
395
|
+
f"would silently measure the 'off' arm under the 'install' "
|
|
396
|
+
f"label, so this is fatal.")
|
|
397
|
+
# A skills/ that is a link can point anywhere, `evals/` included, and
|
|
398
|
+
# still pass a name check on the unresolved path.
|
|
399
|
+
if (package_skills_dir.is_symlink()
|
|
400
|
+
or package_skills_dir.resolve().name != "skills"):
|
|
401
|
+
raise ValueError(
|
|
402
|
+
f"refusing to install {package_skills_dir}: expected a real "
|
|
403
|
+
f"directory named 'skills', not a link. Anything wider risks "
|
|
404
|
+
f"copying the eval set into the answerer's workspace.")
|
|
405
|
+
for child in sorted(package_skills_dir.iterdir()):
|
|
406
|
+
if (not child.is_dir() or child.is_symlink()
|
|
407
|
+
or child.name.startswith(".")):
|
|
408
|
+
continue
|
|
409
|
+
target = dest / child.name
|
|
410
|
+
if target.exists():
|
|
411
|
+
shutil.rmtree(target)
|
|
412
|
+
# A link inside a skill could point at the eval set; copy none.
|
|
413
|
+
shutil.copytree(child, target, symlinks=True,
|
|
414
|
+
ignore=lambda src, names: [
|
|
415
|
+
n for n in names
|
|
416
|
+
if (pathlib.Path(src) / n).is_symlink()])
|
|
380
417
|
if mcp_url:
|
|
381
418
|
(work / "mcp.json").write_text(json.dumps(
|
|
382
419
|
{"mcpServers": {mcp_server: {"type": "http", "url": mcp_url}}}))
|
|
@@ -165,12 +165,23 @@ class TheAnswerersMcpSurface(unittest.TestCase):
|
|
|
165
165
|
tuple(t.split("__")[-1] for t in rb.ANSWER_TOOLS),
|
|
166
166
|
rb.HOSTED_TOOLS_DEFAULT)
|
|
167
167
|
|
|
168
|
+
def test_the_tool_arm_is_the_default_plus_get_skill(self):
|
|
169
|
+
for mode, extra in (("tool", ["get_skill"]), ("install", []),
|
|
170
|
+
("off", [])):
|
|
171
|
+
with self.subTest(mode=mode):
|
|
172
|
+
a = argparse.Namespace(target="local", package_skills=mode)
|
|
173
|
+
self.assertEqual(
|
|
174
|
+
[t.split("__")[-1] for t in rb.answer_tools(a)],
|
|
175
|
+
list(rb.HOSTED_TOOLS_DEFAULT) + extra)
|
|
176
|
+
denied = {t.split("__")[-1] for t in rb.answer_denied(a)}
|
|
177
|
+
self.assertEqual("get_skill" in denied, mode != "tool")
|
|
178
|
+
|
|
168
179
|
def test_the_lists_partition_the_publisher_surface(self):
|
|
169
180
|
# The alarm for the one weakness of an enumerated deny-list: a tool
|
|
170
181
|
# Publisher gains later is offered to the answerer unless it is placed
|
|
171
182
|
# on one side or the other, and this is what says so out loud.
|
|
172
183
|
placed = {t.split("__")[-1] for t in
|
|
173
|
-
(*rb.ANSWER_TOOLS, *rb.ANSWER_DENIED)}
|
|
184
|
+
(*rb.ANSWER_TOOLS, *rb.TOOL_ARM_TOOLS, *rb.ANSWER_DENIED)}
|
|
174
185
|
self.assertEqual(placed, set(rb.PUBLISHER_MCP_TOOLS),
|
|
175
186
|
"every Publisher MCP tool must be either allowed to "
|
|
176
187
|
"the answerer or explicitly denied; an unplaced one "
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Installing a package's own skills into an answerer workspace.
|
|
3
|
+
|
|
4
|
+
Two things can go wrong here and neither announces itself on a live run.
|
|
5
|
+
|
|
6
|
+
The first is contamination. An eval set lives at `<package>/evals/`, so a copy
|
|
7
|
+
that reached one directory too wide would put `cases.jsonl` inside the
|
|
8
|
+
answerer's cwd, where `Read` reaches it. The run would still complete and still
|
|
9
|
+
report a number.
|
|
10
|
+
|
|
11
|
+
The second is a mislabelled arm. `--package-skills=install` that installs
|
|
12
|
+
nothing is the `off` arm wearing the `install` label, and the comparison the
|
|
13
|
+
mode exists to support silently measures the same thing twice.
|
|
14
|
+
|
|
15
|
+
Stdlib only: python3 package_skills_test.py
|
|
16
|
+
"""
|
|
17
|
+
import pathlib
|
|
18
|
+
import shutil
|
|
19
|
+
import sys
|
|
20
|
+
import tempfile
|
|
21
|
+
import unittest
|
|
22
|
+
|
|
23
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
24
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
|
|
25
|
+
/ "eval-answer" / "scripts"))
|
|
26
|
+
import argparse # noqa: E402
|
|
27
|
+
from agent_harness import build_workspace # noqa: E402
|
|
28
|
+
import run_baseline as rb # noqa: E402
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def write_skill(root: pathlib.Path, name: str, body: str = "body") -> None:
|
|
32
|
+
d = root / name
|
|
33
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
(d / "SKILL.md").write_text(
|
|
35
|
+
f"---\nname: {name}\ndescription: d\n---\n\n{body}\n")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class InstallPackageSkills(unittest.TestCase):
|
|
39
|
+
def setUp(self) -> None:
|
|
40
|
+
self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="pkgskills-test-"))
|
|
41
|
+
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
|
|
42
|
+
# The manifest side: skills the harness would install anyway.
|
|
43
|
+
self.skills_root = self.tmp / "skills-root"
|
|
44
|
+
write_skill(self.skills_root, "malloy-analysis", "BUNDLED ANALYSIS")
|
|
45
|
+
write_skill(self.skills_root, "malloy-queries", "BUNDLED QUERIES")
|
|
46
|
+
# The package side, with an eval set beside it.
|
|
47
|
+
self.package = self.tmp / "pkg"
|
|
48
|
+
self.pkg_skills = self.package / "skills"
|
|
49
|
+
write_skill(self.pkg_skills, "house-conventions", "OURS")
|
|
50
|
+
evals = self.package / "evals" / "set-a"
|
|
51
|
+
evals.mkdir(parents=True)
|
|
52
|
+
(evals / "cases.jsonl").write_text('{"qid":"q1","golden":42}\n')
|
|
53
|
+
|
|
54
|
+
def build(self, **kw) -> pathlib.Path:
|
|
55
|
+
work = build_workspace(["malloy-analysis", "malloy-queries"],
|
|
56
|
+
self.skills_root, mcp_url=None, **kw)
|
|
57
|
+
self.addCleanup(shutil.rmtree, work, ignore_errors=True)
|
|
58
|
+
return work
|
|
59
|
+
|
|
60
|
+
def test_installs_the_package_skill_alongside_the_manifest_ones(self):
|
|
61
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
62
|
+
installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
|
|
63
|
+
self.assertEqual(
|
|
64
|
+
installed, ["house-conventions", "malloy-analysis", "malloy-queries"])
|
|
65
|
+
|
|
66
|
+
def test_package_skill_shadows_a_manifest_skill_of_the_same_name(self):
|
|
67
|
+
# The server resolves a same-named package skill by replacing the
|
|
68
|
+
# bundled one. The installed path has to agree, or the two arms of the
|
|
69
|
+
# comparison disagree about which guide is in force.
|
|
70
|
+
write_skill(self.pkg_skills, "malloy-analysis", "OURS WINS")
|
|
71
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
72
|
+
body = (work / ".claude" / "skills" / "malloy-analysis"
|
|
73
|
+
/ "SKILL.md").read_text()
|
|
74
|
+
self.assertIn("OURS WINS", body)
|
|
75
|
+
self.assertNotIn("BUNDLED ANALYSIS", body)
|
|
76
|
+
|
|
77
|
+
def test_never_copies_the_eval_set(self):
|
|
78
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
79
|
+
found = [str(p.relative_to(work)) for p in work.rglob("*")
|
|
80
|
+
if p.name in ("cases.jsonl", "set.json", "events.jsonl")
|
|
81
|
+
or p.name == "evals"]
|
|
82
|
+
self.assertEqual(found, [], f"eval artefacts reached the workspace: {found}")
|
|
83
|
+
|
|
84
|
+
def test_does_not_follow_a_link_inside_a_skill_into_the_eval_set(self):
|
|
85
|
+
# skills/x/evals -> ../../evals would otherwise copy the goldens.
|
|
86
|
+
link = self.pkg_skills / "house-conventions" / "evals"
|
|
87
|
+
link.symlink_to(self.package / "evals", target_is_directory=True)
|
|
88
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
89
|
+
found = [str(p.relative_to(work)) for p in work.rglob("*")
|
|
90
|
+
if p.name in ("cases.jsonl", "set.json", "evals")]
|
|
91
|
+
self.assertEqual(found, [], f"a link carried the eval set in: {found}")
|
|
92
|
+
|
|
93
|
+
def test_refuses_a_directory_that_is_not_named_skills(self):
|
|
94
|
+
# The guard is on the directory name rather than on its contents,
|
|
95
|
+
# because "does this tree contain goldens" is a question that can be
|
|
96
|
+
# answered wrongly and "is this the skills directory" cannot.
|
|
97
|
+
with self.assertRaises(ValueError):
|
|
98
|
+
self.build(package_skills_dir=self.package)
|
|
99
|
+
|
|
100
|
+
def test_refuses_a_skills_link_that_points_at_the_eval_set(self):
|
|
101
|
+
shutil.rmtree(self.pkg_skills)
|
|
102
|
+
self.pkg_skills.symlink_to(self.package / "evals", target_is_directory=True)
|
|
103
|
+
with self.assertRaises(ValueError):
|
|
104
|
+
self.build(package_skills_dir=self.pkg_skills)
|
|
105
|
+
|
|
106
|
+
def test_refuses_a_skills_link_even_when_it_resolves_to_a_skills_dir(self):
|
|
107
|
+
real = self.tmp / "elsewhere" / "skills"
|
|
108
|
+
write_skill(real, "x")
|
|
109
|
+
shutil.rmtree(self.pkg_skills)
|
|
110
|
+
self.pkg_skills.symlink_to(real, target_is_directory=True)
|
|
111
|
+
with self.assertRaises(ValueError):
|
|
112
|
+
self.build(package_skills_dir=self.pkg_skills)
|
|
113
|
+
|
|
114
|
+
def test_refuses_a_missing_directory_rather_than_installing_nothing(self):
|
|
115
|
+
with self.assertRaises(FileNotFoundError):
|
|
116
|
+
self.build(package_skills_dir=self.package / "skills-typo")
|
|
117
|
+
|
|
118
|
+
def test_installs_nothing_when_not_asked(self):
|
|
119
|
+
work = self.build()
|
|
120
|
+
installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
|
|
121
|
+
self.assertEqual(installed, ["malloy-analysis", "malloy-queries"])
|
|
122
|
+
|
|
123
|
+
def test_copies_reference_files_beside_the_skill(self):
|
|
124
|
+
# Symlinking put reference/*.md outside the session's allowed
|
|
125
|
+
# directory, so the agent could load SKILL.md and read nothing beside
|
|
126
|
+
# it. Package skills must not reintroduce that.
|
|
127
|
+
ref = self.pkg_skills / "house-conventions" / "reference"
|
|
128
|
+
ref.mkdir(parents=True)
|
|
129
|
+
(ref / "margin.md").write_text("# Margin\n")
|
|
130
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
131
|
+
target = (work / ".claude" / "skills" / "house-conventions"
|
|
132
|
+
/ "reference" / "margin.md")
|
|
133
|
+
self.assertTrue(target.is_file())
|
|
134
|
+
self.assertFalse(target.is_symlink())
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class ResolveArm(unittest.TestCase):
|
|
138
|
+
"""`--package-skills` must describe the run that actually happened.
|
|
139
|
+
|
|
140
|
+
Each mode is a claim about three things that live apart: the flag, the
|
|
141
|
+
files, and whether the deployment serves them at all. When they disagree the
|
|
142
|
+
run still completes and still reports a number, under the wrong label, and
|
|
143
|
+
nothing downstream can tell. So a mismatch has to stop the run.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
def setUp(self) -> None:
|
|
147
|
+
self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="arm-test-"))
|
|
148
|
+
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
|
|
149
|
+
self.pkg_skills = self.tmp / "pkg" / "skills"
|
|
150
|
+
write_skill(self.pkg_skills, "house", "OURS")
|
|
151
|
+
# Stub the HTTP layer: nothing here may reach a network.
|
|
152
|
+
self.served_dir = None
|
|
153
|
+
for name, fake in (
|
|
154
|
+
("package_skill_names", lambda *a, **k: self.served_names),
|
|
155
|
+
("served_package_dir", lambda *a, **k: self.served_dir)):
|
|
156
|
+
real = getattr(rb, name)
|
|
157
|
+
setattr(rb, name, fake)
|
|
158
|
+
self.addCleanup(setattr, rb, name, real)
|
|
159
|
+
self.served_names: list[str] = []
|
|
160
|
+
|
|
161
|
+
def serve(self, names: list[str]) -> None:
|
|
162
|
+
self.served_names = names
|
|
163
|
+
|
|
164
|
+
def args(self, mode: str, override=None) -> argparse.Namespace:
|
|
165
|
+
return argparse.Namespace(
|
|
166
|
+
package_skills=mode, target="local", publisher="http://x",
|
|
167
|
+
no_answerer_skills=False,
|
|
168
|
+
environment="env", package="pkg",
|
|
169
|
+
package_skills_dir_override=str(override) if override else None)
|
|
170
|
+
|
|
171
|
+
def test_install_with_a_tree_resolves_and_pins_a_sha(self):
|
|
172
|
+
self.serve(["house"])
|
|
173
|
+
a = self.args("install", self.pkg_skills)
|
|
174
|
+
rb.resolve_package_skills(a)
|
|
175
|
+
self.assertEqual(a.package_skills_dir, self.pkg_skills)
|
|
176
|
+
self.assertRegex(a.package_skills_sha, r"^[0-9a-f]{64}$")
|
|
177
|
+
|
|
178
|
+
def test_the_sha_moves_when_a_guide_changes(self):
|
|
179
|
+
self.serve(["house"])
|
|
180
|
+
a = self.args("install", self.pkg_skills)
|
|
181
|
+
rb.resolve_package_skills(a)
|
|
182
|
+
before = a.package_skills_sha
|
|
183
|
+
write_skill(self.pkg_skills, "house", "EDITED")
|
|
184
|
+
b = self.args("install", self.pkg_skills)
|
|
185
|
+
rb.resolve_package_skills(b)
|
|
186
|
+
self.assertNotEqual(before, b.package_skills_sha)
|
|
187
|
+
|
|
188
|
+
def test_install_without_a_tree_refuses(self):
|
|
189
|
+
self.serve(["house"])
|
|
190
|
+
a = self.args("install", self.tmp / "nope" / "skills")
|
|
191
|
+
with self.assertRaises(SystemExit):
|
|
192
|
+
rb.resolve_package_skills(a)
|
|
193
|
+
|
|
194
|
+
def test_tool_with_no_package_skills_is_recorded_as_off(self):
|
|
195
|
+
# get_skill is denied under `off`, so that is what the run measures.
|
|
196
|
+
self.serve([])
|
|
197
|
+
a = self.args("tool")
|
|
198
|
+
rb.resolve_package_skills(a)
|
|
199
|
+
self.assertEqual(a.package_skills, "off")
|
|
200
|
+
self.assertNotIn(rb.TOOL_ARM_TOOLS[0], rb.answer_tools(a))
|
|
201
|
+
self.assertIn(rb.TOOL_ARM_TOOLS[0], rb.answer_denied(a))
|
|
202
|
+
|
|
203
|
+
def test_install_with_no_answerer_skills_is_fatal(self):
|
|
204
|
+
self.serve(["house"])
|
|
205
|
+
a = self.args("install", self.pkg_skills)
|
|
206
|
+
a.no_answerer_skills = True
|
|
207
|
+
with self.assertRaises(SystemExit):
|
|
208
|
+
rb.resolve_package_skills(a)
|
|
209
|
+
|
|
210
|
+
def test_a_replay_carries_the_prior_run_and_resolves_nothing(self):
|
|
211
|
+
# Prior run was install; the server and files have since changed.
|
|
212
|
+
self.serve([])
|
|
213
|
+
a = self.args("tool")
|
|
214
|
+
rb.resolve_package_skills(a, {"packageSkillsMode": "install",
|
|
215
|
+
"packageSkillsSha": "abc"})
|
|
216
|
+
self.assertEqual((a.package_skills, a.package_skills_sha),
|
|
217
|
+
("install", "abc"))
|
|
218
|
+
self.assertIsNone(a.package_skills_dir)
|
|
219
|
+
|
|
220
|
+
def test_a_replay_of_a_run_that_recorded_nothing_records_nothing(self):
|
|
221
|
+
a = self.args("tool")
|
|
222
|
+
rb.resolve_package_skills(a, {})
|
|
223
|
+
self.assertIsNone(a.package_skills)
|
|
224
|
+
self.assertIsNone(a.package_skills_sha)
|
|
225
|
+
|
|
226
|
+
def test_a_platform_run_records_null_without_probing(self):
|
|
227
|
+
def boom(*a, **k):
|
|
228
|
+
raise AssertionError("probed the REST API")
|
|
229
|
+
rb.package_skill_names = boom
|
|
230
|
+
a = self.args("tool")
|
|
231
|
+
a.target = "platform"
|
|
232
|
+
rb.resolve_package_skills(a)
|
|
233
|
+
self.assertIsNone(a.package_skills)
|
|
234
|
+
|
|
235
|
+
def test_refuses_a_served_skills_dir_that_is_a_link(self):
|
|
236
|
+
pkg = self.tmp / "served"
|
|
237
|
+
(pkg / "evals").mkdir(parents=True)
|
|
238
|
+
(pkg / "skills").symlink_to(pkg / "evals", target_is_directory=True)
|
|
239
|
+
self.served_dir = pkg
|
|
240
|
+
with self.assertRaises(SystemExit):
|
|
241
|
+
rb.resolve_package_skills(self.args("tool"))
|
|
242
|
+
|
|
243
|
+
def test_tool_passes_when_the_server_serves_them(self):
|
|
244
|
+
self.serve(["house"])
|
|
245
|
+
a = self.args("tool")
|
|
246
|
+
rb.resolve_package_skills(a)
|
|
247
|
+
self.assertIsNone(a.package_skills_dir)
|
|
248
|
+
|
|
249
|
+
def test_off_refuses_when_the_server_is_still_serving_them(self):
|
|
250
|
+
# A baseline the answerer can call get_skill against is not a baseline.
|
|
251
|
+
self.serve(["house"])
|
|
252
|
+
with self.assertRaises(SystemExit):
|
|
253
|
+
rb.resolve_package_skills(self.args("off"))
|
|
254
|
+
|
|
255
|
+
def test_off_passes_when_the_server_withholds_them(self):
|
|
256
|
+
self.serve([])
|
|
257
|
+
a = self.args("off")
|
|
258
|
+
rb.resolve_package_skills(a)
|
|
259
|
+
self.assertIsNone(a.package_skills_dir)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
if __name__ == "__main__":
|
|
263
|
+
unittest.main(verbosity=2)
|
|
@@ -114,6 +114,11 @@ ANSWER_TOOLS = ("mcp__publisher__get_context",
|
|
|
114
114
|
"mcp__publisher__execute_query",
|
|
115
115
|
"mcp__publisher__search_malloy_docs")
|
|
116
116
|
|
|
117
|
+
# Granted on top of ANSWER_TOOLS only to the `--package-skills=tool` arm, and
|
|
118
|
+
# denied everywhere else, so the one divergence from the hosted default is
|
|
119
|
+
# explicit and local to that arm.
|
|
120
|
+
TOOL_ARM_TOOLS = ("mcp__publisher__get_skill",)
|
|
121
|
+
|
|
117
122
|
# Publisher tools the answerer must NOT hold. `--allowedTools` grants
|
|
118
123
|
# permission, it does not restrict availability, so the four above were the
|
|
119
124
|
# allow list while all EIGHT of this server's tools were offered. Two of the
|
|
@@ -126,7 +131,9 @@ ANSWER_TOOLS = ("mcp__publisher__get_context",
|
|
|
126
131
|
# reload_package recompiles the package mid-attempt, so the model
|
|
127
132
|
# under measurement is not the one the run pinned.
|
|
128
133
|
#
|
|
129
|
-
# compile_model authoring surface
|
|
134
|
+
# compile_model authoring surface.
|
|
135
|
+
# get_agent adopting a package agent changes who is answering;
|
|
136
|
+
# the run measures the model, not a persona.
|
|
130
137
|
#
|
|
131
138
|
# `get_status` and `list_packages` reveal the other packages on the server and
|
|
132
139
|
# are no use to an answerer given its scope in the prompt.
|
|
@@ -135,6 +142,7 @@ ANSWER_TOOLS = ("mcp__publisher__get_context",
|
|
|
135
142
|
# or `mcp__publisher__*`) removes ALL of its tools including the ones wanted,
|
|
136
143
|
# verified 2026-09-08.
|
|
137
144
|
ANSWER_DENIED = ("mcp__publisher__compile_model",
|
|
145
|
+
"mcp__publisher__get_agent",
|
|
138
146
|
"mcp__publisher__search_database_schema",
|
|
139
147
|
"mcp__publisher__reload_package",
|
|
140
148
|
"mcp__publisher__get_status",
|
|
@@ -148,8 +156,9 @@ ANSWER_DENIED = ("mcp__publisher__compile_model",
|
|
|
148
156
|
# side it belongs on. Read from `get_status`/`tools/list` on a running server
|
|
149
157
|
# when this needs re-checking; the authority is the server's registration, and
|
|
150
158
|
# a mismatch here is a decision, not a lint.
|
|
151
|
-
PUBLISHER_MCP_TOOLS = ("compile_model", "execute_query", "
|
|
152
|
-
"
|
|
159
|
+
PUBLISHER_MCP_TOOLS = ("compile_model", "execute_query", "get_agent",
|
|
160
|
+
"get_context", "get_skill", "get_status",
|
|
161
|
+
"list_packages", "reload_package",
|
|
153
162
|
"search_database_schema", "search_malloy_docs")
|
|
154
163
|
# The platform target: a hosted MCP server exposing the same two operations
|
|
155
164
|
# under its own names. The CLI addresses a tool as `mcp__<server>__<tool>`, so
|
|
@@ -183,7 +192,8 @@ import ledger # noqa: E402
|
|
|
183
192
|
from ledger import read_jsonl # noqa: E402
|
|
184
193
|
from mcp_payload import (doc_tokens, entity_hits, # noqa: E402
|
|
185
194
|
entity_ids, search_terms, target_shapes)
|
|
186
|
-
from publisher_rest import package_identity,
|
|
195
|
+
from publisher_rest import (package_identity, package_skill_names, # noqa: E402
|
|
196
|
+
served_model_path, served_package_dir, try_query)
|
|
187
197
|
from score_retrieval import ( # noqa: E402
|
|
188
198
|
cascade, coverage_report_summary, load_coverage_report, score_case,
|
|
189
199
|
summarise)
|
|
@@ -2161,7 +2171,8 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
|
|
|
2161
2171
|
# the cwd, so `Read` reaches the doctrine and never the eval set.
|
|
2162
2172
|
if a.answerer_skills:
|
|
2163
2173
|
work = str(build_workspace(a.answerer_skills, a.roots,
|
|
2164
|
-
mcp_url=None, prefix=f"ans-{qid}-"
|
|
2174
|
+
mcp_url=None, prefix=f"ans-{qid}-",
|
|
2175
|
+
package_skills_dir=a.package_skills_dir))
|
|
2165
2176
|
else:
|
|
2166
2177
|
work = tempfile.mkdtemp(prefix=f"ans-{qid}-")
|
|
2167
2178
|
mcp = os.path.join(work, "mcp.json")
|
|
@@ -2180,12 +2191,12 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
|
|
|
2180
2191
|
t0 = time.time()
|
|
2181
2192
|
events = claude(
|
|
2182
2193
|
prompt, work, a.model, mcp=mcp,
|
|
2183
|
-
tools=a
|
|
2194
|
+
tools=answer_tools(a),
|
|
2184
2195
|
# Only the local arm: the hosted server's tool surface belongs to
|
|
2185
2196
|
# the platform, so there is no complement here to enumerate. That
|
|
2186
2197
|
# arm is covered by `isolation_breaches`, which RECORDS whatever it
|
|
2187
2198
|
# was granted rather than pretending to have denied it.
|
|
2188
|
-
denied=()
|
|
2199
|
+
denied=answer_denied(a),
|
|
2189
2200
|
turns=a.max_turns, effort=a.effort, timeout=a.timeout,
|
|
2190
2201
|
skills=bool(a.answerer_skills))
|
|
2191
2202
|
elapsed = round(time.time() - t0, 1)
|
|
@@ -2296,7 +2307,10 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
|
|
|
2296
2307
|
# recorded as a breach.
|
|
2297
2308
|
if name == "Skill":
|
|
2298
2309
|
sk = (c.get("input") or {}).get("skill")
|
|
2299
|
-
|
|
2310
|
+
# A package skill this run installed is under test, not a stray.
|
|
2311
|
+
in_scope = (list(a.answerer_skills or [])
|
|
2312
|
+
+ list(a.package_skill_names or []))
|
|
2313
|
+
if sk and sk not in in_scope:
|
|
2300
2314
|
foreign_skills.append(sk)
|
|
2301
2315
|
elif sk:
|
|
2302
2316
|
used_skills.append(sk)
|
|
@@ -2408,7 +2422,7 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
|
|
|
2408
2422
|
"error": res.get("subtype") if res.get("is_error") else None,
|
|
2409
2423
|
"calls": calls,
|
|
2410
2424
|
"breaches": isolation_breaches(
|
|
2411
|
-
events, a
|
|
2425
|
+
events, answer_tools(a))
|
|
2412
2426
|
+ path_breaches(events, a.set_dir)
|
|
2413
2427
|
+ [f"invoked a skill outside the manifest: {sk}"
|
|
2414
2428
|
for sk in sorted(set(foreign_skills))],
|
|
@@ -2600,13 +2614,15 @@ def prior_judge_cost(out: pathlib.Path) -> float | None:
|
|
|
2600
2614
|
Must be read BEFORE `run_config` rewrites run.json. Read afterwards it is
|
|
2601
2615
|
always None, which is the version of this that silently does nothing.
|
|
2602
2616
|
"""
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2617
|
+
return prior_run_json(out).get("judgeCostUsd")
|
|
2618
|
+
|
|
2619
|
+
|
|
2620
|
+
def prior_run_json(out: pathlib.Path) -> dict[str, Any]:
|
|
2621
|
+
"""The run.json already in this directory, or {} when absent or unreadable."""
|
|
2606
2622
|
try:
|
|
2607
|
-
return json.loads(
|
|
2623
|
+
return json.loads((out / "run.json").read_text())
|
|
2608
2624
|
except (json.JSONDecodeError, OSError):
|
|
2609
|
-
return
|
|
2625
|
+
return {}
|
|
2610
2626
|
|
|
2611
2627
|
|
|
2612
2628
|
def carry_judge_cost(judge_cost: float,
|
|
@@ -3044,6 +3060,120 @@ def run_judge(case: dict[str, Any], att: dict[str, Any], a: argparse.Namespace,
|
|
|
3044
3060
|
return {**parse_verdict(text), "judge_cost_usd": res.get("total_cost_usd")}
|
|
3045
3061
|
|
|
3046
3062
|
|
|
3063
|
+
def resolve_package_skills(a: argparse.Namespace,
|
|
3064
|
+
prior: dict[str, Any] | None = None) -> None:
|
|
3065
|
+
"""Settle --package-skills against what the server actually serves.
|
|
3066
|
+
|
|
3067
|
+
The mode is a claim about the answerer's environment, and its parts live in
|
|
3068
|
+
different places: the flag, the files in the model repo, and whether the
|
|
3069
|
+
package under test ships them at all. A run that asked
|
|
3070
|
+
for one arm and measured another is worse than a run that refused, because
|
|
3071
|
+
the label on the result is wrong and nothing downstream can tell.
|
|
3072
|
+
|
|
3073
|
+
So each mode is checked, not assumed:
|
|
3074
|
+
|
|
3075
|
+
- `install`: the package must have a skills/ directory to copy. Nothing to
|
|
3076
|
+
copy means this arm is silently the `off` arm.
|
|
3077
|
+
- `tool`: the server must be serving the package's skills, or the answerer
|
|
3078
|
+
has nothing to fetch and this arm is silently the `off` arm too.
|
|
3079
|
+
- `off`: the served package must hold no skills, or the answerer can reach
|
|
3080
|
+
guidance the baseline is supposed to exclude. Point --package at a copy
|
|
3081
|
+
of the package without skills/.
|
|
3082
|
+
|
|
3083
|
+
A run that spawns no answerer (`--rebuild`, `--rejudge`, `--from`, signalled
|
|
3084
|
+
by a non-None `prior`) resolves nothing: it carries the prior run's recorded
|
|
3085
|
+
mode and sha, since the files and the server may have moved since. A
|
|
3086
|
+
platform run records null: the hosted workspace's skills are not reachable
|
|
3087
|
+
from here. A package that ships no skills under the default `tool` mode is
|
|
3088
|
+
recorded as `off` (the answerer is denied get_skill, so that is what it
|
|
3089
|
+
measures) rather than refused.
|
|
3090
|
+
|
|
3091
|
+
Sets `package_skills_dir` (the tree to install, or None) and
|
|
3092
|
+
`package_skills_sha` (a content hash pinned on the run, so two runs
|
|
3093
|
+
differing only in guidance are distinguishable in the ledger).
|
|
3094
|
+
"""
|
|
3095
|
+
a.package_skills_dir = None
|
|
3096
|
+
a.package_skills_sha = None
|
|
3097
|
+
# Directory names of the package skills this run INSTALLS. The answerer
|
|
3098
|
+
# addresses a skill by that name, and the provenance check below needs to
|
|
3099
|
+
# tell one this run put there deliberately from one of the CLI's own.
|
|
3100
|
+
a.package_skill_names: list[str] = []
|
|
3101
|
+
|
|
3102
|
+
if prior is not None:
|
|
3103
|
+
a.package_skills = prior.get("packageSkillsMode")
|
|
3104
|
+
a.package_skills_sha = prior.get("packageSkillsSha")
|
|
3105
|
+
return
|
|
3106
|
+
if a.target == "platform":
|
|
3107
|
+
a.package_skills = None
|
|
3108
|
+
return
|
|
3109
|
+
if a.package_skills == "install" and a.no_answerer_skills:
|
|
3110
|
+
raise SystemExit(
|
|
3111
|
+
"--package-skills=install with --no-answerer-skills installs "
|
|
3112
|
+
"nothing, so the run would be labelled install while measuring "
|
|
3113
|
+
"the 'off' arm. Drop one of the two.")
|
|
3114
|
+
|
|
3115
|
+
# The SERVED copy, not the working tree. The `tool` arm fetches what the
|
|
3116
|
+
# server holds, so installing those same bytes is what makes the two arms
|
|
3117
|
+
# differ only in delivery -- which is the whole comparison. An explicit
|
|
3118
|
+
# --package-skills-dir wins, for a server whose tree is not local.
|
|
3119
|
+
local = None
|
|
3120
|
+
if a.package_skills_dir_override:
|
|
3121
|
+
# Checked here rather than left to build_workspace: this resolver runs
|
|
3122
|
+
# before the first spawn precisely so a mistyped path stops the run at
|
|
3123
|
+
# second zero instead of after it has started answering.
|
|
3124
|
+
override = pathlib.Path(a.package_skills_dir_override)
|
|
3125
|
+
if not override.is_dir():
|
|
3126
|
+
raise SystemExit(
|
|
3127
|
+
f"--package-skills-dir {override} is not a directory.")
|
|
3128
|
+
local = override
|
|
3129
|
+
else:
|
|
3130
|
+
pkg_dir = served_package_dir(a.publisher, a.environment, a.package)
|
|
3131
|
+
skills_dir = pkg_dir / "skills" if pkg_dir is not None else None
|
|
3132
|
+
if skills_dir is not None and skills_dir.is_symlink():
|
|
3133
|
+
raise SystemExit(
|
|
3134
|
+
f"{skills_dir} is a link; refusing to read or install through "
|
|
3135
|
+
f"it, since it can point at the eval set.")
|
|
3136
|
+
if skills_dir is not None and skills_dir.is_dir():
|
|
3137
|
+
local = skills_dir
|
|
3138
|
+
served = package_skill_names(a.publisher, a.environment, a.package)
|
|
3139
|
+
|
|
3140
|
+
if a.package_skills == "install":
|
|
3141
|
+
if local is None:
|
|
3142
|
+
raise SystemExit(
|
|
3143
|
+
"--package-skills=install, but no skills/ directory was found "
|
|
3144
|
+
"for the package. Installing nothing measures the 'off' arm "
|
|
3145
|
+
"under the 'install' label. Pass --package-skills-dir, or pick "
|
|
3146
|
+
"another mode.")
|
|
3147
|
+
a.package_skills_dir = local
|
|
3148
|
+
a.package_skill_names = sorted(
|
|
3149
|
+
c.name for c in local.iterdir()
|
|
3150
|
+
if c.is_dir() and not c.name.startswith("."))
|
|
3151
|
+
elif a.package_skills == "tool":
|
|
3152
|
+
if not served:
|
|
3153
|
+
print(f" ! the server serves no package skills for "
|
|
3154
|
+
f"{a.environment}/{a.package}; recording packageSkillsMode "
|
|
3155
|
+
f"'off' (get_skill is denied) rather than 'tool'.")
|
|
3156
|
+
a.package_skills = "off"
|
|
3157
|
+
elif a.package_skills == "off":
|
|
3158
|
+
if served:
|
|
3159
|
+
raise SystemExit(
|
|
3160
|
+
"--package-skills=off, but the server IS serving package "
|
|
3161
|
+
f"skills for {a.environment}/{a.package}: "
|
|
3162
|
+
f"{', '.join(served)}. The answerer could fetch them with "
|
|
3163
|
+
"get_skill, so this is not a baseline. Serve a copy of the "
|
|
3164
|
+
"package without skills/.")
|
|
3165
|
+
|
|
3166
|
+
if local is not None:
|
|
3167
|
+
h = hashlib.sha256()
|
|
3168
|
+
for f in sorted(local.rglob("*")):
|
|
3169
|
+
if f.is_file():
|
|
3170
|
+
h.update(str(f.relative_to(local)).encode())
|
|
3171
|
+
h.update(b"\0")
|
|
3172
|
+
h.update(f.read_bytes())
|
|
3173
|
+
h.update(b"\0")
|
|
3174
|
+
a.package_skills_sha = h.hexdigest()
|
|
3175
|
+
|
|
3176
|
+
|
|
3047
3177
|
def resolve_config(a: argparse.Namespace) -> config.Config:
|
|
3048
3178
|
"""Fill every unset server, name and path from the set's eval.toml.
|
|
3049
3179
|
|
|
@@ -3122,6 +3252,24 @@ def attempt_event(c: dict[str, Any], att: dict[str, Any], phase: str | None,
|
|
|
3122
3252
|
transcriptPath=att["transcriptPath"])
|
|
3123
3253
|
|
|
3124
3254
|
|
|
3255
|
+
def answer_tools(a: argparse.Namespace) -> tuple[str, ...]:
|
|
3256
|
+
"""What the answerer is granted: the hosted tools, or Publisher's four."""
|
|
3257
|
+
if a.target == "platform":
|
|
3258
|
+
return tuple(a.hosted_tools)
|
|
3259
|
+
return ANSWER_TOOLS + (TOOL_ARM_TOOLS if _tool_arm(a) else ())
|
|
3260
|
+
|
|
3261
|
+
|
|
3262
|
+
def _tool_arm(a: argparse.Namespace) -> bool:
|
|
3263
|
+
# getattr: tests build this Namespace by hand.
|
|
3264
|
+
return getattr(a, "package_skills", "off") == "tool"
|
|
3265
|
+
|
|
3266
|
+
|
|
3267
|
+
def answer_denied(a: argparse.Namespace) -> tuple[str, ...]:
|
|
3268
|
+
if a.target == "platform":
|
|
3269
|
+
return ()
|
|
3270
|
+
return ANSWER_DENIED + (() if _tool_arm(a) else TOOL_ARM_TOOLS)
|
|
3271
|
+
|
|
3272
|
+
|
|
3125
3273
|
def main(argv: list[str] | None = None) -> int:
|
|
3126
3274
|
ap = argparse.ArgumentParser()
|
|
3127
3275
|
ap.add_argument("--set", dest="set_dir", required=True, type=pathlib.Path)
|
|
@@ -3183,6 +3331,23 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
3183
3331
|
"last two are both --target platform. Default: "
|
|
3184
3332
|
"from [model] mcp_port in eval.toml, else "
|
|
3185
3333
|
f"{LOCAL_MCP_URL}")
|
|
3334
|
+
ap.add_argument("--package-skills", choices=("install", "tool", "off"),
|
|
3335
|
+
default="tool",
|
|
3336
|
+
help="how the package's own skills reach the answerer. "
|
|
3337
|
+
"install = copied into its workspace, so they load "
|
|
3338
|
+
"like any other skill. tool = not installed; the "
|
|
3339
|
+
"answerer must call get_skill, which is what a real "
|
|
3340
|
+
"agent against this server does. off = the served package "
|
|
3341
|
+
"ships none (serve a copy without skills/), the "
|
|
3342
|
+
"baseline arm. "
|
|
3343
|
+
"The default is `tool` because it is the unmodified "
|
|
3344
|
+
"product behaviour.")
|
|
3345
|
+
ap.add_argument("--package-skills-dir", dest="package_skills_dir_override",
|
|
3346
|
+
default=None,
|
|
3347
|
+
help="the skills/ tree to install for --package-skills="
|
|
3348
|
+
"install. Defaults to the served package's own, which "
|
|
3349
|
+
"is what the `tool` arm would fetch; pass this only "
|
|
3350
|
+
"when the served tree is not reachable locally.")
|
|
3186
3351
|
ap.add_argument("--publisher", default=None,
|
|
3187
3352
|
help="Publisher REST base, used to re-execute the answerer's "
|
|
3188
3353
|
"final query so the judge sees rows rather than prose. "
|
|
@@ -3511,6 +3676,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
3511
3676
|
"(--truth-publisher); goldens are taken as they stand")
|
|
3512
3677
|
|
|
3513
3678
|
served_identity = package_identity(a.publisher, a.environment, a.package)
|
|
3679
|
+
resolve_package_skills(
|
|
3680
|
+
a, prior_run_json(a.from_run or a.out) if answers_exist else None)
|
|
3514
3681
|
set_meta = {}
|
|
3515
3682
|
if (a.set_dir / "set.json").exists():
|
|
3516
3683
|
set_meta = json.loads((a.set_dir / "set.json").read_text())
|
|
@@ -3707,8 +3874,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
3707
3874
|
# repair moves this without anyone remembering to bump anything.
|
|
3708
3875
|
datasetSha=ledger.dataset_sha(a.set_dir),
|
|
3709
3876
|
# What the SERVER says it served, taken rather than recomputed.
|
|
3710
|
-
# packageSha covers every model path in the package, so
|
|
3711
|
-
# imported file moves it where a sha of
|
|
3877
|
+
# packageSha covers every model path, skill and agent in the package, so
|
|
3878
|
+
# an edit to an imported file moves it where a sha of --model-path does not.
|
|
3712
3879
|
packageSha=served_identity.get("sourceContentSha"),
|
|
3713
3880
|
servedRevision=served_identity.get("servedRevision"),
|
|
3714
3881
|
environment=a.environment, package=a.package,
|
|
@@ -3728,6 +3895,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
3728
3895
|
skillsVersion=ledger.skills_git_sha(a.roots[0]),
|
|
3729
3896
|
skillsRoot=str(a.roots[0]),
|
|
3730
3897
|
harnessVersion=ledger.skills_git_sha(),
|
|
3898
|
+
packageSkillsMode=a.package_skills,
|
|
3899
|
+
packageSkillsSha=a.package_skills_sha,
|
|
3731
3900
|
answererManifest=a.answerer_manifest if a.answerer_skills else None,
|
|
3732
3901
|
answererSkills=a.answerer_skills or [],
|
|
3733
3902
|
judgeSkills=a.judge_skills or [],
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-report
|
|
3
3
|
description: 'Write the summary a person reads after an evaluation run: what ran, what it scored, which failures were the MODEL and which were the EVAL itself (out of turns, contaminated, unestablished golden, unreadable judge), what was learned, and what to do next. Every claim links to the artifact behind it. Use at the end of any run, before quoting a number to anyone. Never scores an answer (eval-answer) or decides who owns a failure (eval-diagnose).'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.39
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Report a Run
|