@malloy-publisher/skills 0.1.38 → 0.1.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/package.json +1 -1
  2. package/skills/eval-answer/SKILL.md +1 -1
  3. package/skills/eval-answer/reference/ledger-schema.md +4 -3
  4. package/skills/eval-answer/scripts/ledger.py +5 -1
  5. package/skills/eval-answer/scripts/publisher_rest.py +39 -0
  6. package/skills/eval-diagnose/SKILL.md +43 -18
  7. package/skills/eval-diagnose/scripts/cluster_failures.py +2 -0
  8. package/skills/eval-diagnose/scripts/diagnose.py +11 -4
  9. package/skills/eval-import/SKILL.md +1 -1
  10. package/skills/eval-improve/SKILL.md +46 -6
  11. package/skills/eval-improve/scripts/improve.py +45 -16
  12. package/skills/eval-improve/scripts/improve_test.py +35 -0
  13. package/skills/eval-judge/SKILL.md +1 -1
  14. package/skills/eval-loop/SKILL.md +3 -2
  15. package/skills/eval-loop/reference/acceptance-check.md +12 -0
  16. package/skills/eval-loop/scripts/agent_harness.py +38 -1
  17. package/skills/eval-loop/scripts/agent_harness_test.py +12 -1
  18. package/skills/eval-loop/scripts/package_skills_test.py +263 -0
  19. package/skills/eval-loop/scripts/run_baseline.py +185 -16
  20. package/skills/eval-report/SKILL.md +1 -1
  21. package/skills/malloy-analysis/SKILL.md +1 -1
  22. package/skills/malloy-analysis-report/SKILL.md +1 -1
  23. package/skills/malloy-charts/SKILL.md +1 -1
  24. package/skills/malloy-dashboards/SKILL.md +1 -1
  25. package/skills/malloy-define/SKILL.md +1 -1
  26. package/skills/malloy-discover/SKILL.md +1 -1
  27. package/skills/malloy-document/SKILL.md +1 -1
  28. package/skills/malloy-getting-started/SKILL.md +2 -2
  29. package/skills/malloy-gotchas-modeling/SKILL.md +1 -1
  30. package/skills/malloy-html-data-apps/SKILL.md +1 -1
  31. package/skills/malloy-lookml-review/SKILL.md +1 -1
  32. package/skills/malloy-materialization/SKILL.md +1 -1
  33. package/skills/malloy-model/SKILL.md +1 -1
  34. package/skills/malloy-model-as-you-go/SKILL.md +1 -1
  35. package/skills/malloy-modeling/SKILL.md +1 -1
  36. package/skills/malloy-notebooks/SKILL.md +1 -1
  37. package/skills/malloy-phrase-detection/SKILL.md +1 -1
  38. package/skills/malloy-powerbi-review/SKILL.md +1 -1
  39. package/skills/malloy-publish/SKILL.md +1 -1
  40. package/skills/malloy-queries/SKILL.md +1 -1
  41. package/skills/malloy-reports/SKILL.md +1 -1
  42. package/skills/malloy-review/SKILL.md +1 -1
  43. package/skills/malloy-run-agent/SKILL.md +58 -0
  44. package/skills/malloy-source-unreachable/SKILL.md +1 -1
@@ -346,7 +346,9 @@ def build_workspace(skills: Iterable[str],
346
346
  skills_root: pathlib.Path | Iterable[pathlib.Path],
347
347
  mcp_url: str | None, prefix: str = "agent-",
348
348
  depth: int = 1,
349
- mcp_server: str = "publisher") -> pathlib.Path:
349
+ mcp_server: str = "publisher",
350
+ package_skills_dir: pathlib.Path | None = None
351
+ ) -> pathlib.Path:
350
352
  """A scratch cwd holding `.claude/skills/` and an MCP config.
351
353
 
352
354
  COPIED, not symlinked. A symlink loads SKILL.md fine -- the skills mechanism
@@ -377,6 +379,41 @@ def build_workspace(skills: Iterable[str],
377
379
  else list(skills_root))
378
380
  for name in present:
379
381
  shutil.copytree(find_skill(name, roots), dest / name, symlinks=False)
382
+ # A package's own skills, installed last so one whose name matches a
383
+ # manifest skill REPLACES it -- the same shadowing the server does when it
384
+ # serves them, so the installed path and the tool path agree about which
385
+ # guide is in force.
386
+ #
387
+ # Only the package's `skills/` subtree is copied, never the package. An
388
+ # eval set lives at `<package>/evals/`, so copying the package would hand
389
+ # the answerer the goldens.
390
+ if package_skills_dir is not None:
391
+ if not package_skills_dir.is_dir():
392
+ raise FileNotFoundError(
393
+ f"--package-skills=install was asked for but "
394
+ f"{package_skills_dir} is not a directory. Installing nothing "
395
+ f"would silently measure the 'off' arm under the 'install' "
396
+ f"label, so this is fatal.")
397
+ # A skills/ that is a link can point anywhere, `evals/` included, and
398
+ # still pass a name check on the unresolved path.
399
+ if (package_skills_dir.is_symlink()
400
+ or package_skills_dir.resolve().name != "skills"):
401
+ raise ValueError(
402
+ f"refusing to install {package_skills_dir}: expected a real "
403
+ f"directory named 'skills', not a link. Anything wider risks "
404
+ f"copying the eval set into the answerer's workspace.")
405
+ for child in sorted(package_skills_dir.iterdir()):
406
+ if (not child.is_dir() or child.is_symlink()
407
+ or child.name.startswith(".")):
408
+ continue
409
+ target = dest / child.name
410
+ if target.exists():
411
+ shutil.rmtree(target)
412
+ # A link inside a skill could point at the eval set; copy none.
413
+ shutil.copytree(child, target, symlinks=True,
414
+ ignore=lambda src, names: [
415
+ n for n in names
416
+ if (pathlib.Path(src) / n).is_symlink()])
380
417
  if mcp_url:
381
418
  (work / "mcp.json").write_text(json.dumps(
382
419
  {"mcpServers": {mcp_server: {"type": "http", "url": mcp_url}}}))
@@ -165,12 +165,23 @@ class TheAnswerersMcpSurface(unittest.TestCase):
165
165
  tuple(t.split("__")[-1] for t in rb.ANSWER_TOOLS),
166
166
  rb.HOSTED_TOOLS_DEFAULT)
167
167
 
168
+ def test_the_tool_arm_is_the_default_plus_get_skill(self):
169
+ for mode, extra in (("tool", ["get_skill"]), ("install", []),
170
+ ("off", [])):
171
+ with self.subTest(mode=mode):
172
+ a = argparse.Namespace(target="local", package_skills=mode)
173
+ self.assertEqual(
174
+ [t.split("__")[-1] for t in rb.answer_tools(a)],
175
+ list(rb.HOSTED_TOOLS_DEFAULT) + extra)
176
+ denied = {t.split("__")[-1] for t in rb.answer_denied(a)}
177
+ self.assertEqual("get_skill" in denied, mode != "tool")
178
+
168
179
  def test_the_lists_partition_the_publisher_surface(self):
169
180
  # The alarm for the one weakness of an enumerated deny-list: a tool
170
181
  # Publisher gains later is offered to the answerer unless it is placed
171
182
  # on one side or the other, and this is what says so out loud.
172
183
  placed = {t.split("__")[-1] for t in
173
- (*rb.ANSWER_TOOLS, *rb.ANSWER_DENIED)}
184
+ (*rb.ANSWER_TOOLS, *rb.TOOL_ARM_TOOLS, *rb.ANSWER_DENIED)}
174
185
  self.assertEqual(placed, set(rb.PUBLISHER_MCP_TOOLS),
175
186
  "every Publisher MCP tool must be either allowed to "
176
187
  "the answerer or explicitly denied; an unplaced one "
@@ -0,0 +1,263 @@
1
+ #!/usr/bin/env python3
2
+ """Installing a package's own skills into an answerer workspace.
3
+
4
+ Two things can go wrong here and neither announces itself on a live run.
5
+
6
+ The first is contamination. An eval set lives at `<package>/evals/`, so a copy
7
+ that reached one directory too wide would put `cases.jsonl` inside the
8
+ answerer's cwd, where `Read` reaches it. The run would still complete and still
9
+ report a number.
10
+
11
+ The second is a mislabelled arm. `--package-skills=install` that installs
12
+ nothing is the `off` arm wearing the `install` label, and the comparison the
13
+ mode exists to support silently measures the same thing twice.
14
+
15
+ Stdlib only: python3 package_skills_test.py
16
+ """
17
+ import pathlib
18
+ import shutil
19
+ import sys
20
+ import tempfile
21
+ import unittest
22
+
23
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
24
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
25
+ / "eval-answer" / "scripts"))
26
+ import argparse # noqa: E402
27
+ from agent_harness import build_workspace # noqa: E402
28
+ import run_baseline as rb # noqa: E402
29
+
30
+
31
+ def write_skill(root: pathlib.Path, name: str, body: str = "body") -> None:
32
+ d = root / name
33
+ d.mkdir(parents=True, exist_ok=True)
34
+ (d / "SKILL.md").write_text(
35
+ f"---\nname: {name}\ndescription: d\n---\n\n{body}\n")
36
+
37
+
38
+ class InstallPackageSkills(unittest.TestCase):
39
+ def setUp(self) -> None:
40
+ self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="pkgskills-test-"))
41
+ self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
42
+ # The manifest side: skills the harness would install anyway.
43
+ self.skills_root = self.tmp / "skills-root"
44
+ write_skill(self.skills_root, "malloy-analysis", "BUNDLED ANALYSIS")
45
+ write_skill(self.skills_root, "malloy-queries", "BUNDLED QUERIES")
46
+ # The package side, with an eval set beside it.
47
+ self.package = self.tmp / "pkg"
48
+ self.pkg_skills = self.package / "skills"
49
+ write_skill(self.pkg_skills, "house-conventions", "OURS")
50
+ evals = self.package / "evals" / "set-a"
51
+ evals.mkdir(parents=True)
52
+ (evals / "cases.jsonl").write_text('{"qid":"q1","golden":42}\n')
53
+
54
+ def build(self, **kw) -> pathlib.Path:
55
+ work = build_workspace(["malloy-analysis", "malloy-queries"],
56
+ self.skills_root, mcp_url=None, **kw)
57
+ self.addCleanup(shutil.rmtree, work, ignore_errors=True)
58
+ return work
59
+
60
+ def test_installs_the_package_skill_alongside_the_manifest_ones(self):
61
+ work = self.build(package_skills_dir=self.pkg_skills)
62
+ installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
63
+ self.assertEqual(
64
+ installed, ["house-conventions", "malloy-analysis", "malloy-queries"])
65
+
66
+ def test_package_skill_shadows_a_manifest_skill_of_the_same_name(self):
67
+ # The server resolves a same-named package skill by replacing the
68
+ # bundled one. The installed path has to agree, or the two arms of the
69
+ # comparison disagree about which guide is in force.
70
+ write_skill(self.pkg_skills, "malloy-analysis", "OURS WINS")
71
+ work = self.build(package_skills_dir=self.pkg_skills)
72
+ body = (work / ".claude" / "skills" / "malloy-analysis"
73
+ / "SKILL.md").read_text()
74
+ self.assertIn("OURS WINS", body)
75
+ self.assertNotIn("BUNDLED ANALYSIS", body)
76
+
77
+ def test_never_copies_the_eval_set(self):
78
+ work = self.build(package_skills_dir=self.pkg_skills)
79
+ found = [str(p.relative_to(work)) for p in work.rglob("*")
80
+ if p.name in ("cases.jsonl", "set.json", "events.jsonl")
81
+ or p.name == "evals"]
82
+ self.assertEqual(found, [], f"eval artefacts reached the workspace: {found}")
83
+
84
+ def test_does_not_follow_a_link_inside_a_skill_into_the_eval_set(self):
85
+ # skills/x/evals -> ../../evals would otherwise copy the goldens.
86
+ link = self.pkg_skills / "house-conventions" / "evals"
87
+ link.symlink_to(self.package / "evals", target_is_directory=True)
88
+ work = self.build(package_skills_dir=self.pkg_skills)
89
+ found = [str(p.relative_to(work)) for p in work.rglob("*")
90
+ if p.name in ("cases.jsonl", "set.json", "evals")]
91
+ self.assertEqual(found, [], f"a link carried the eval set in: {found}")
92
+
93
+ def test_refuses_a_directory_that_is_not_named_skills(self):
94
+ # The guard is on the directory name rather than on its contents,
95
+ # because "does this tree contain goldens" is a question that can be
96
+ # answered wrongly and "is this the skills directory" cannot.
97
+ with self.assertRaises(ValueError):
98
+ self.build(package_skills_dir=self.package)
99
+
100
+ def test_refuses_a_skills_link_that_points_at_the_eval_set(self):
101
+ shutil.rmtree(self.pkg_skills)
102
+ self.pkg_skills.symlink_to(self.package / "evals", target_is_directory=True)
103
+ with self.assertRaises(ValueError):
104
+ self.build(package_skills_dir=self.pkg_skills)
105
+
106
+ def test_refuses_a_skills_link_even_when_it_resolves_to_a_skills_dir(self):
107
+ real = self.tmp / "elsewhere" / "skills"
108
+ write_skill(real, "x")
109
+ shutil.rmtree(self.pkg_skills)
110
+ self.pkg_skills.symlink_to(real, target_is_directory=True)
111
+ with self.assertRaises(ValueError):
112
+ self.build(package_skills_dir=self.pkg_skills)
113
+
114
+ def test_refuses_a_missing_directory_rather_than_installing_nothing(self):
115
+ with self.assertRaises(FileNotFoundError):
116
+ self.build(package_skills_dir=self.package / "skills-typo")
117
+
118
+ def test_installs_nothing_when_not_asked(self):
119
+ work = self.build()
120
+ installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
121
+ self.assertEqual(installed, ["malloy-analysis", "malloy-queries"])
122
+
123
+ def test_copies_reference_files_beside_the_skill(self):
124
+ # Symlinking put reference/*.md outside the session's allowed
125
+ # directory, so the agent could load SKILL.md and read nothing beside
126
+ # it. Package skills must not reintroduce that.
127
+ ref = self.pkg_skills / "house-conventions" / "reference"
128
+ ref.mkdir(parents=True)
129
+ (ref / "margin.md").write_text("# Margin\n")
130
+ work = self.build(package_skills_dir=self.pkg_skills)
131
+ target = (work / ".claude" / "skills" / "house-conventions"
132
+ / "reference" / "margin.md")
133
+ self.assertTrue(target.is_file())
134
+ self.assertFalse(target.is_symlink())
135
+
136
+
137
+ class ResolveArm(unittest.TestCase):
138
+ """`--package-skills` must describe the run that actually happened.
139
+
140
+ Each mode is a claim about three things that live apart: the flag, the
141
+ files, and whether the deployment serves them at all. When they disagree the
142
+ run still completes and still reports a number, under the wrong label, and
143
+ nothing downstream can tell. So a mismatch has to stop the run.
144
+ """
145
+
146
+ def setUp(self) -> None:
147
+ self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="arm-test-"))
148
+ self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
149
+ self.pkg_skills = self.tmp / "pkg" / "skills"
150
+ write_skill(self.pkg_skills, "house", "OURS")
151
+ # Stub the HTTP layer: nothing here may reach a network.
152
+ self.served_dir = None
153
+ for name, fake in (
154
+ ("package_skill_names", lambda *a, **k: self.served_names),
155
+ ("served_package_dir", lambda *a, **k: self.served_dir)):
156
+ real = getattr(rb, name)
157
+ setattr(rb, name, fake)
158
+ self.addCleanup(setattr, rb, name, real)
159
+ self.served_names: list[str] = []
160
+
161
+ def serve(self, names: list[str]) -> None:
162
+ self.served_names = names
163
+
164
+ def args(self, mode: str, override=None) -> argparse.Namespace:
165
+ return argparse.Namespace(
166
+ package_skills=mode, target="local", publisher="http://x",
167
+ no_answerer_skills=False,
168
+ environment="env", package="pkg",
169
+ package_skills_dir_override=str(override) if override else None)
170
+
171
+ def test_install_with_a_tree_resolves_and_pins_a_sha(self):
172
+ self.serve(["house"])
173
+ a = self.args("install", self.pkg_skills)
174
+ rb.resolve_package_skills(a)
175
+ self.assertEqual(a.package_skills_dir, self.pkg_skills)
176
+ self.assertRegex(a.package_skills_sha, r"^[0-9a-f]{64}$")
177
+
178
+ def test_the_sha_moves_when_a_guide_changes(self):
179
+ self.serve(["house"])
180
+ a = self.args("install", self.pkg_skills)
181
+ rb.resolve_package_skills(a)
182
+ before = a.package_skills_sha
183
+ write_skill(self.pkg_skills, "house", "EDITED")
184
+ b = self.args("install", self.pkg_skills)
185
+ rb.resolve_package_skills(b)
186
+ self.assertNotEqual(before, b.package_skills_sha)
187
+
188
+ def test_install_without_a_tree_refuses(self):
189
+ self.serve(["house"])
190
+ a = self.args("install", self.tmp / "nope" / "skills")
191
+ with self.assertRaises(SystemExit):
192
+ rb.resolve_package_skills(a)
193
+
194
+ def test_tool_with_no_package_skills_is_recorded_as_off(self):
195
+ # get_skill is denied under `off`, so that is what the run measures.
196
+ self.serve([])
197
+ a = self.args("tool")
198
+ rb.resolve_package_skills(a)
199
+ self.assertEqual(a.package_skills, "off")
200
+ self.assertNotIn(rb.TOOL_ARM_TOOLS[0], rb.answer_tools(a))
201
+ self.assertIn(rb.TOOL_ARM_TOOLS[0], rb.answer_denied(a))
202
+
203
+ def test_install_with_no_answerer_skills_is_fatal(self):
204
+ self.serve(["house"])
205
+ a = self.args("install", self.pkg_skills)
206
+ a.no_answerer_skills = True
207
+ with self.assertRaises(SystemExit):
208
+ rb.resolve_package_skills(a)
209
+
210
+ def test_a_replay_carries_the_prior_run_and_resolves_nothing(self):
211
+ # Prior run was install; the server and files have since changed.
212
+ self.serve([])
213
+ a = self.args("tool")
214
+ rb.resolve_package_skills(a, {"packageSkillsMode": "install",
215
+ "packageSkillsSha": "abc"})
216
+ self.assertEqual((a.package_skills, a.package_skills_sha),
217
+ ("install", "abc"))
218
+ self.assertIsNone(a.package_skills_dir)
219
+
220
+ def test_a_replay_of_a_run_that_recorded_nothing_records_nothing(self):
221
+ a = self.args("tool")
222
+ rb.resolve_package_skills(a, {})
223
+ self.assertIsNone(a.package_skills)
224
+ self.assertIsNone(a.package_skills_sha)
225
+
226
+ def test_a_platform_run_records_null_without_probing(self):
227
+ def boom(*a, **k):
228
+ raise AssertionError("probed the REST API")
229
+ rb.package_skill_names = boom
230
+ a = self.args("tool")
231
+ a.target = "platform"
232
+ rb.resolve_package_skills(a)
233
+ self.assertIsNone(a.package_skills)
234
+
235
+ def test_refuses_a_served_skills_dir_that_is_a_link(self):
236
+ pkg = self.tmp / "served"
237
+ (pkg / "evals").mkdir(parents=True)
238
+ (pkg / "skills").symlink_to(pkg / "evals", target_is_directory=True)
239
+ self.served_dir = pkg
240
+ with self.assertRaises(SystemExit):
241
+ rb.resolve_package_skills(self.args("tool"))
242
+
243
+ def test_tool_passes_when_the_server_serves_them(self):
244
+ self.serve(["house"])
245
+ a = self.args("tool")
246
+ rb.resolve_package_skills(a)
247
+ self.assertIsNone(a.package_skills_dir)
248
+
249
+ def test_off_refuses_when_the_server_is_still_serving_them(self):
250
+ # A baseline the answerer can call get_skill against is not a baseline.
251
+ self.serve(["house"])
252
+ with self.assertRaises(SystemExit):
253
+ rb.resolve_package_skills(self.args("off"))
254
+
255
+ def test_off_passes_when_the_server_withholds_them(self):
256
+ self.serve([])
257
+ a = self.args("off")
258
+ rb.resolve_package_skills(a)
259
+ self.assertIsNone(a.package_skills_dir)
260
+
261
+
262
+ if __name__ == "__main__":
263
+ unittest.main(verbosity=2)
@@ -114,6 +114,11 @@ ANSWER_TOOLS = ("mcp__publisher__get_context",
114
114
  "mcp__publisher__execute_query",
115
115
  "mcp__publisher__search_malloy_docs")
116
116
 
117
+ # Granted on top of ANSWER_TOOLS only to the `--package-skills=tool` arm, and
118
+ # denied everywhere else, so the one divergence from the hosted default is
119
+ # explicit and local to that arm.
120
+ TOOL_ARM_TOOLS = ("mcp__publisher__get_skill",)
121
+
117
122
  # Publisher tools the answerer must NOT hold. `--allowedTools` grants
118
123
  # permission, it does not restrict availability, so the four above were the
119
124
  # allow list while all EIGHT of this server's tools were offered. Two of the
@@ -126,7 +131,9 @@ ANSWER_TOOLS = ("mcp__publisher__get_context",
126
131
  # reload_package recompiles the package mid-attempt, so the model
127
132
  # under measurement is not the one the run pinned.
128
133
  #
129
- # compile_model authoring surface, per ANSWER_TOOLS above.
134
+ # compile_model authoring surface.
135
+ # get_agent adopting a package agent changes who is answering;
136
+ # the run measures the model, not a persona.
130
137
  #
131
138
  # `get_status` and `list_packages` reveal the other packages on the server and
132
139
  # are no use to an answerer given its scope in the prompt.
@@ -135,6 +142,7 @@ ANSWER_TOOLS = ("mcp__publisher__get_context",
135
142
  # or `mcp__publisher__*`) removes ALL of its tools including the ones wanted,
136
143
  # verified 2026-09-08.
137
144
  ANSWER_DENIED = ("mcp__publisher__compile_model",
145
+ "mcp__publisher__get_agent",
138
146
  "mcp__publisher__search_database_schema",
139
147
  "mcp__publisher__reload_package",
140
148
  "mcp__publisher__get_status",
@@ -148,8 +156,9 @@ ANSWER_DENIED = ("mcp__publisher__compile_model",
148
156
  # side it belongs on. Read from `get_status`/`tools/list` on a running server
149
157
  # when this needs re-checking; the authority is the server's registration, and
150
158
  # a mismatch here is a decision, not a lint.
151
- PUBLISHER_MCP_TOOLS = ("compile_model", "execute_query", "get_context",
152
- "get_status", "list_packages", "reload_package",
159
+ PUBLISHER_MCP_TOOLS = ("compile_model", "execute_query", "get_agent",
160
+ "get_context", "get_skill", "get_status",
161
+ "list_packages", "reload_package",
153
162
  "search_database_schema", "search_malloy_docs")
154
163
  # The platform target: a hosted MCP server exposing the same two operations
155
164
  # under its own names. The CLI addresses a tool as `mcp__<server>__<tool>`, so
@@ -183,7 +192,8 @@ import ledger # noqa: E402
183
192
  from ledger import read_jsonl # noqa: E402
184
193
  from mcp_payload import (doc_tokens, entity_hits, # noqa: E402
185
194
  entity_ids, search_terms, target_shapes)
186
- from publisher_rest import package_identity, served_model_path, try_query # noqa: E402
195
+ from publisher_rest import (package_identity, package_skill_names, # noqa: E402
196
+ served_model_path, served_package_dir, try_query)
187
197
  from score_retrieval import ( # noqa: E402
188
198
  cascade, coverage_report_summary, load_coverage_report, score_case,
189
199
  summarise)
@@ -2161,7 +2171,8 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
2161
2171
  # the cwd, so `Read` reaches the doctrine and never the eval set.
2162
2172
  if a.answerer_skills:
2163
2173
  work = str(build_workspace(a.answerer_skills, a.roots,
2164
- mcp_url=None, prefix=f"ans-{qid}-"))
2174
+ mcp_url=None, prefix=f"ans-{qid}-",
2175
+ package_skills_dir=a.package_skills_dir))
2165
2176
  else:
2166
2177
  work = tempfile.mkdtemp(prefix=f"ans-{qid}-")
2167
2178
  mcp = os.path.join(work, "mcp.json")
@@ -2180,12 +2191,12 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
2180
2191
  t0 = time.time()
2181
2192
  events = claude(
2182
2193
  prompt, work, a.model, mcp=mcp,
2183
- tools=a.hosted_tools if platform else ANSWER_TOOLS,
2194
+ tools=answer_tools(a),
2184
2195
  # Only the local arm: the hosted server's tool surface belongs to
2185
2196
  # the platform, so there is no complement here to enumerate. That
2186
2197
  # arm is covered by `isolation_breaches`, which RECORDS whatever it
2187
2198
  # was granted rather than pretending to have denied it.
2188
- denied=() if platform else ANSWER_DENIED,
2199
+ denied=answer_denied(a),
2189
2200
  turns=a.max_turns, effort=a.effort, timeout=a.timeout,
2190
2201
  skills=bool(a.answerer_skills))
2191
2202
  elapsed = round(time.time() - t0, 1)
@@ -2296,7 +2307,10 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
2296
2307
  # recorded as a breach.
2297
2308
  if name == "Skill":
2298
2309
  sk = (c.get("input") or {}).get("skill")
2299
- if sk and sk not in (a.answerer_skills or []):
2310
+ # A package skill this run installed is under test, not a stray.
2311
+ in_scope = (list(a.answerer_skills or [])
2312
+ + list(a.package_skill_names or []))
2313
+ if sk and sk not in in_scope:
2300
2314
  foreign_skills.append(sk)
2301
2315
  elif sk:
2302
2316
  used_skills.append(sk)
@@ -2408,7 +2422,7 @@ def run_answerer(case: dict[str, Any], a: argparse.Namespace,
2408
2422
  "error": res.get("subtype") if res.get("is_error") else None,
2409
2423
  "calls": calls,
2410
2424
  "breaches": isolation_breaches(
2411
- events, a.hosted_tools if a.target == "platform" else ANSWER_TOOLS)
2425
+ events, answer_tools(a))
2412
2426
  + path_breaches(events, a.set_dir)
2413
2427
  + [f"invoked a skill outside the manifest: {sk}"
2414
2428
  for sk in sorted(set(foreign_skills))],
@@ -2600,13 +2614,15 @@ def prior_judge_cost(out: pathlib.Path) -> float | None:
2600
2614
  Must be read BEFORE `run_config` rewrites run.json. Read afterwards it is
2601
2615
  always None, which is the version of this that silently does nothing.
2602
2616
  """
2603
- f = out / "run.json"
2604
- if not f.exists():
2605
- return None
2617
+ return prior_run_json(out).get("judgeCostUsd")
2618
+
2619
+
2620
+ def prior_run_json(out: pathlib.Path) -> dict[str, Any]:
2621
+ """The run.json already in this directory, or {} when absent or unreadable."""
2606
2622
  try:
2607
- return json.loads(f.read_text()).get("judgeCostUsd")
2623
+ return json.loads((out / "run.json").read_text())
2608
2624
  except (json.JSONDecodeError, OSError):
2609
- return None
2625
+ return {}
2610
2626
 
2611
2627
 
2612
2628
  def carry_judge_cost(judge_cost: float,
@@ -3044,6 +3060,120 @@ def run_judge(case: dict[str, Any], att: dict[str, Any], a: argparse.Namespace,
3044
3060
  return {**parse_verdict(text), "judge_cost_usd": res.get("total_cost_usd")}
3045
3061
 
3046
3062
 
3063
+ def resolve_package_skills(a: argparse.Namespace,
3064
+ prior: dict[str, Any] | None = None) -> None:
3065
+ """Settle --package-skills against what the server actually serves.
3066
+
3067
+ The mode is a claim about the answerer's environment, and its parts live in
3068
+ different places: the flag, the files in the model repo, and whether the
3069
+ package under test ships them at all. A run that asked
3070
+ for one arm and measured another is worse than a run that refused, because
3071
+ the label on the result is wrong and nothing downstream can tell.
3072
+
3073
+ So each mode is checked, not assumed:
3074
+
3075
+ - `install`: the package must have a skills/ directory to copy. Nothing to
3076
+ copy means this arm is silently the `off` arm.
3077
+ - `tool`: the server must be serving the package's skills, or the answerer
3078
+ has nothing to fetch and this arm is silently the `off` arm too.
3079
+ - `off`: the served package must hold no skills, or the answerer can reach
3080
+ guidance the baseline is supposed to exclude. Point --package at a copy
3081
+ of the package without skills/.
3082
+
3083
+ A run that spawns no answerer (`--rebuild`, `--rejudge`, `--from`, signalled
3084
+ by a non-None `prior`) resolves nothing: it carries the prior run's recorded
3085
+ mode and sha, since the files and the server may have moved since. A
3086
+ platform run records null: the hosted workspace's skills are not reachable
3087
+ from here. A package that ships no skills under the default `tool` mode is
3088
+ recorded as `off` (the answerer is denied get_skill, so that is what it
3089
+ measures) rather than refused.
3090
+
3091
+ Sets `package_skills_dir` (the tree to install, or None) and
3092
+ `package_skills_sha` (a content hash pinned on the run, so two runs
3093
+ differing only in guidance are distinguishable in the ledger).
3094
+ """
3095
+ a.package_skills_dir = None
3096
+ a.package_skills_sha = None
3097
+ # Directory names of the package skills this run INSTALLS. The answerer
3098
+ # addresses a skill by that name, and the provenance check below needs to
3099
+ # tell one this run put there deliberately from one of the CLI's own.
3100
+ a.package_skill_names: list[str] = []
3101
+
3102
+ if prior is not None:
3103
+ a.package_skills = prior.get("packageSkillsMode")
3104
+ a.package_skills_sha = prior.get("packageSkillsSha")
3105
+ return
3106
+ if a.target == "platform":
3107
+ a.package_skills = None
3108
+ return
3109
+ if a.package_skills == "install" and a.no_answerer_skills:
3110
+ raise SystemExit(
3111
+ "--package-skills=install with --no-answerer-skills installs "
3112
+ "nothing, so the run would be labelled install while measuring "
3113
+ "the 'off' arm. Drop one of the two.")
3114
+
3115
+ # The SERVED copy, not the working tree. The `tool` arm fetches what the
3116
+ # server holds, so installing those same bytes is what makes the two arms
3117
+ # differ only in delivery -- which is the whole comparison. An explicit
3118
+ # --package-skills-dir wins, for a server whose tree is not local.
3119
+ local = None
3120
+ if a.package_skills_dir_override:
3121
+ # Checked here rather than left to build_workspace: this resolver runs
3122
+ # before the first spawn precisely so a mistyped path stops the run at
3123
+ # second zero instead of after it has started answering.
3124
+ override = pathlib.Path(a.package_skills_dir_override)
3125
+ if not override.is_dir():
3126
+ raise SystemExit(
3127
+ f"--package-skills-dir {override} is not a directory.")
3128
+ local = override
3129
+ else:
3130
+ pkg_dir = served_package_dir(a.publisher, a.environment, a.package)
3131
+ skills_dir = pkg_dir / "skills" if pkg_dir is not None else None
3132
+ if skills_dir is not None and skills_dir.is_symlink():
3133
+ raise SystemExit(
3134
+ f"{skills_dir} is a link; refusing to read or install through "
3135
+ f"it, since it can point at the eval set.")
3136
+ if skills_dir is not None and skills_dir.is_dir():
3137
+ local = skills_dir
3138
+ served = package_skill_names(a.publisher, a.environment, a.package)
3139
+
3140
+ if a.package_skills == "install":
3141
+ if local is None:
3142
+ raise SystemExit(
3143
+ "--package-skills=install, but no skills/ directory was found "
3144
+ "for the package. Installing nothing measures the 'off' arm "
3145
+ "under the 'install' label. Pass --package-skills-dir, or pick "
3146
+ "another mode.")
3147
+ a.package_skills_dir = local
3148
+ a.package_skill_names = sorted(
3149
+ c.name for c in local.iterdir()
3150
+ if c.is_dir() and not c.name.startswith("."))
3151
+ elif a.package_skills == "tool":
3152
+ if not served:
3153
+ print(f" ! the server serves no package skills for "
3154
+ f"{a.environment}/{a.package}; recording packageSkillsMode "
3155
+ f"'off' (get_skill is denied) rather than 'tool'.")
3156
+ a.package_skills = "off"
3157
+ elif a.package_skills == "off":
3158
+ if served:
3159
+ raise SystemExit(
3160
+ "--package-skills=off, but the server IS serving package "
3161
+ f"skills for {a.environment}/{a.package}: "
3162
+ f"{', '.join(served)}. The answerer could fetch them with "
3163
+ "get_skill, so this is not a baseline. Serve a copy of the "
3164
+ "package without skills/.")
3165
+
3166
+ if local is not None:
3167
+ h = hashlib.sha256()
3168
+ for f in sorted(local.rglob("*")):
3169
+ if f.is_file():
3170
+ h.update(str(f.relative_to(local)).encode())
3171
+ h.update(b"\0")
3172
+ h.update(f.read_bytes())
3173
+ h.update(b"\0")
3174
+ a.package_skills_sha = h.hexdigest()
3175
+
3176
+
3047
3177
  def resolve_config(a: argparse.Namespace) -> config.Config:
3048
3178
  """Fill every unset server, name and path from the set's eval.toml.
3049
3179
 
@@ -3122,6 +3252,24 @@ def attempt_event(c: dict[str, Any], att: dict[str, Any], phase: str | None,
3122
3252
  transcriptPath=att["transcriptPath"])
3123
3253
 
3124
3254
 
3255
+ def answer_tools(a: argparse.Namespace) -> tuple[str, ...]:
3256
+ """What the answerer is granted: the hosted tools, or Publisher's four."""
3257
+ if a.target == "platform":
3258
+ return tuple(a.hosted_tools)
3259
+ return ANSWER_TOOLS + (TOOL_ARM_TOOLS if _tool_arm(a) else ())
3260
+
3261
+
3262
+ def _tool_arm(a: argparse.Namespace) -> bool:
3263
+ # getattr: tests build this Namespace by hand.
3264
+ return getattr(a, "package_skills", "off") == "tool"
3265
+
3266
+
3267
+ def answer_denied(a: argparse.Namespace) -> tuple[str, ...]:
3268
+ if a.target == "platform":
3269
+ return ()
3270
+ return ANSWER_DENIED + (() if _tool_arm(a) else TOOL_ARM_TOOLS)
3271
+
3272
+
3125
3273
  def main(argv: list[str] | None = None) -> int:
3126
3274
  ap = argparse.ArgumentParser()
3127
3275
  ap.add_argument("--set", dest="set_dir", required=True, type=pathlib.Path)
@@ -3183,6 +3331,23 @@ def main(argv: list[str] | None = None) -> int:
3183
3331
  "last two are both --target platform. Default: "
3184
3332
  "from [model] mcp_port in eval.toml, else "
3185
3333
  f"{LOCAL_MCP_URL}")
3334
+ ap.add_argument("--package-skills", choices=("install", "tool", "off"),
3335
+ default="tool",
3336
+ help="how the package's own skills reach the answerer. "
3337
+ "install = copied into its workspace, so they load "
3338
+ "like any other skill. tool = not installed; the "
3339
+ "answerer must call get_skill, which is what a real "
3340
+ "agent against this server does. off = the served package "
3341
+ "ships none (serve a copy without skills/), the "
3342
+ "baseline arm. "
3343
+ "The default is `tool` because it is the unmodified "
3344
+ "product behaviour.")
3345
+ ap.add_argument("--package-skills-dir", dest="package_skills_dir_override",
3346
+ default=None,
3347
+ help="the skills/ tree to install for --package-skills="
3348
+ "install. Defaults to the served package's own, which "
3349
+ "is what the `tool` arm would fetch; pass this only "
3350
+ "when the served tree is not reachable locally.")
3186
3351
  ap.add_argument("--publisher", default=None,
3187
3352
  help="Publisher REST base, used to re-execute the answerer's "
3188
3353
  "final query so the judge sees rows rather than prose. "
@@ -3511,6 +3676,8 @@ def main(argv: list[str] | None = None) -> int:
3511
3676
  "(--truth-publisher); goldens are taken as they stand")
3512
3677
 
3513
3678
  served_identity = package_identity(a.publisher, a.environment, a.package)
3679
+ resolve_package_skills(
3680
+ a, prior_run_json(a.from_run or a.out) if answers_exist else None)
3514
3681
  set_meta = {}
3515
3682
  if (a.set_dir / "set.json").exists():
3516
3683
  set_meta = json.loads((a.set_dir / "set.json").read_text())
@@ -3707,8 +3874,8 @@ def main(argv: list[str] | None = None) -> int:
3707
3874
  # repair moves this without anyone remembering to bump anything.
3708
3875
  datasetSha=ledger.dataset_sha(a.set_dir),
3709
3876
  # What the SERVER says it served, taken rather than recomputed.
3710
- # packageSha covers every model path in the package, so an edit to an
3711
- # imported file moves it where a sha of the one --model-path does not.
3877
+ # packageSha covers every model path, skill and agent in the package, so
3878
+ # an edit to an imported file moves it where a sha of --model-path does not.
3712
3879
  packageSha=served_identity.get("sourceContentSha"),
3713
3880
  servedRevision=served_identity.get("servedRevision"),
3714
3881
  environment=a.environment, package=a.package,
@@ -3728,6 +3895,8 @@ def main(argv: list[str] | None = None) -> int:
3728
3895
  skillsVersion=ledger.skills_git_sha(a.roots[0]),
3729
3896
  skillsRoot=str(a.roots[0]),
3730
3897
  harnessVersion=ledger.skills_git_sha(),
3898
+ packageSkillsMode=a.package_skills,
3899
+ packageSkillsSha=a.package_skills_sha,
3731
3900
  answererManifest=a.answerer_manifest if a.answerer_skills else None,
3732
3901
  answererSkills=a.answerer_skills or [],
3733
3902
  judgeSkills=a.judge_skills or [],
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  name: eval-report
3
3
  description: 'Write the summary a person reads after an evaluation run: what ran, what it scored, which failures were the MODEL and which were the EVAL itself (out of turns, contaminated, unestablished golden, unreadable judge), what was learned, and what to do next. Every claim links to the artifact behind it. Use at the end of any run, before quoting a number to anyone. Never scores an answer (eval-answer) or decides who owns a failure (eval-diagnose).'
4
- version: 0.1.38
4
+ version: 0.1.39
5
5
  ---
6
6
 
7
7
  # Report a Run