@malloy-publisher/skills 0.1.37 → 0.1.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/package.json +1 -1
  2. package/skills/eval-answer/SKILL.md +1 -1
  3. package/skills/eval-answer/reference/ledger-schema.md +4 -3
  4. package/skills/eval-answer/scripts/ledger.py +5 -1
  5. package/skills/eval-answer/scripts/ledger_test.py +3 -0
  6. package/skills/eval-answer/scripts/publisher_rest.py +39 -0
  7. package/skills/eval-diagnose/SKILL.md +43 -18
  8. package/skills/eval-diagnose/scripts/cluster_failures.py +2 -0
  9. package/skills/eval-diagnose/scripts/diagnose.py +11 -4
  10. package/skills/eval-import/SKILL.md +1 -1
  11. package/skills/eval-improve/SKILL.md +46 -6
  12. package/skills/eval-improve/scripts/improve.py +45 -16
  13. package/skills/eval-improve/scripts/improve_test.py +35 -0
  14. package/skills/eval-judge/SKILL.md +1 -1
  15. package/skills/eval-loop/SKILL.md +3 -2
  16. package/skills/eval-loop/reference/acceptance-check.md +12 -0
  17. package/skills/eval-loop/scripts/agent_harness.py +38 -1
  18. package/skills/eval-loop/scripts/agent_harness_test.py +12 -1
  19. package/skills/eval-loop/scripts/package_skills_test.py +263 -0
  20. package/skills/eval-loop/scripts/run_baseline.py +185 -16
  21. package/skills/eval-loop/templates/eval-run-package/notebooks/eval_run.malloy +1 -1
  22. package/skills/eval-report/SKILL.md +1 -1
  23. package/skills/malloy-analysis/SKILL.md +3 -3
  24. package/skills/malloy-analysis-report/SKILL.md +14 -14
  25. package/skills/malloy-charts/SKILL.md +2 -2
  26. package/skills/malloy-dashboards/SKILL.md +10 -488
  27. package/skills/malloy-define/SKILL.md +1 -1
  28. package/skills/malloy-discover/SKILL.md +1 -1
  29. package/skills/malloy-document/SKILL.md +1 -1
  30. package/skills/malloy-getting-started/SKILL.md +4 -4
  31. package/skills/malloy-gotchas-modeling/SKILL.md +1 -1
  32. package/skills/malloy-html-data-apps/SKILL.md +2 -2
  33. package/skills/malloy-lookml-review/SKILL.md +1 -1
  34. package/skills/malloy-materialization/SKILL.md +1 -1
  35. package/skills/malloy-model/SKILL.md +1 -1
  36. package/skills/malloy-model/reference/analysis-to-model.md +3 -3
  37. package/skills/malloy-model-as-you-go/SKILL.md +3 -3
  38. package/skills/malloy-modeling/SKILL.md +3 -3
  39. package/skills/malloy-notebooks/SKILL.md +11 -323
  40. package/skills/malloy-phrase-detection/SKILL.md +1 -1
  41. package/skills/malloy-powerbi-review/SKILL.md +1 -1
  42. package/skills/malloy-publish/SKILL.md +8 -9
  43. package/skills/malloy-queries/SKILL.md +1 -1
  44. package/skills/malloy-reports/SKILL.md +238 -0
  45. package/skills/malloy-reports/reference/grid.md +59 -0
  46. package/skills/malloy-reports/reference/linear.md +180 -0
  47. package/skills/malloy-review/SKILL.md +1 -1
  48. package/skills/malloy-review/reference/rubric-structure.md +1 -1
  49. package/skills/malloy-run-agent/SKILL.md +58 -0
  50. package/skills/malloy-source-unreachable/SKILL.md +3 -3
@@ -50,6 +50,18 @@ Acceptance rules (replacing any vague "results improve"):
50
50
  - Independent deterministic justification (a probed-wrong definition
51
51
  corrected) may accept without a measured win. Record that as the acceptance check
52
52
  `reason`.
53
+ - **Package-skill edits (`class: skill`) get the full rules, never the
54
+ documentation shortcut.** A `get_context` probe proves an entity is
55
+ discoverable. Nothing deterministic proves an agent read a guide and behaved
56
+ differently, so the only evidence a skill edit has is a fresh blind
57
+ re-answer: both splits, twice, clearing the flip-count bar. Treat the
58
+ documentation shortcut as unavailable here even when the edit reads like
59
+ documentation, because a guide is prompt text written next to the case it
60
+ failed and is the easiest artifact in this loop to overfit.
61
+
62
+ Read the diff before measuring it, too. A guide that names the case, quotes
63
+ its expected number, or encodes the shape of one question fails the expert
64
+ test in `skill:eval-improve` and is rejected without a run.
53
65
 
54
66
  Write the `acceptance_check` event (decision, class, baseline and final run ids,
55
67
  regressions, holdout delta, reason) BEFORE any commit, so a rejected
@@ -346,7 +346,9 @@ def build_workspace(skills: Iterable[str],
346
346
  skills_root: pathlib.Path | Iterable[pathlib.Path],
347
347
  mcp_url: str | None, prefix: str = "agent-",
348
348
  depth: int = 1,
349
- mcp_server: str = "publisher") -> pathlib.Path:
349
+ mcp_server: str = "publisher",
350
+ package_skills_dir: pathlib.Path | None = None
351
+ ) -> pathlib.Path:
350
352
  """A scratch cwd holding `.claude/skills/` and an MCP config.
351
353
 
352
354
  COPIED, not symlinked. A symlink loads SKILL.md fine -- the skills mechanism
@@ -377,6 +379,41 @@ def build_workspace(skills: Iterable[str],
377
379
  else list(skills_root))
378
380
  for name in present:
379
381
  shutil.copytree(find_skill(name, roots), dest / name, symlinks=False)
382
+ # A package's own skills, installed last so one whose name matches a
383
+ # manifest skill REPLACES it -- the same shadowing the server does when it
384
+ # serves them, so the installed path and the tool path agree about which
385
+ # guide is in force.
386
+ #
387
+ # Only the package's `skills/` subtree is copied, never the package. An
388
+ # eval set lives at `<package>/evals/`, so copying the package would hand
389
+ # the answerer the goldens.
390
+ if package_skills_dir is not None:
391
+ if not package_skills_dir.is_dir():
392
+ raise FileNotFoundError(
393
+ f"--package-skills=install was asked for but "
394
+ f"{package_skills_dir} is not a directory. Installing nothing "
395
+ f"would silently measure the 'off' arm under the 'install' "
396
+ f"label, so this is fatal.")
397
+ # A skills/ that is a link can point anywhere, `evals/` included, and
398
+ # still pass a name check on the unresolved path.
399
+ if (package_skills_dir.is_symlink()
400
+ or package_skills_dir.resolve().name != "skills"):
401
+ raise ValueError(
402
+ f"refusing to install {package_skills_dir}: expected a real "
403
+ f"directory named 'skills', not a link. Anything wider risks "
404
+ f"copying the eval set into the answerer's workspace.")
405
+ for child in sorted(package_skills_dir.iterdir()):
406
+ if (not child.is_dir() or child.is_symlink()
407
+ or child.name.startswith(".")):
408
+ continue
409
+ target = dest / child.name
410
+ if target.exists():
411
+ shutil.rmtree(target)
412
+ # A link inside a skill could point at the eval set; copy none.
413
+ shutil.copytree(child, target, symlinks=True,
414
+ ignore=lambda src, names: [
415
+ n for n in names
416
+ if (pathlib.Path(src) / n).is_symlink()])
380
417
  if mcp_url:
381
418
  (work / "mcp.json").write_text(json.dumps(
382
419
  {"mcpServers": {mcp_server: {"type": "http", "url": mcp_url}}}))
@@ -165,12 +165,23 @@ class TheAnswerersMcpSurface(unittest.TestCase):
165
165
  tuple(t.split("__")[-1] for t in rb.ANSWER_TOOLS),
166
166
  rb.HOSTED_TOOLS_DEFAULT)
167
167
 
168
+ def test_the_tool_arm_is_the_default_plus_get_skill(self):
169
+ for mode, extra in (("tool", ["get_skill"]), ("install", []),
170
+ ("off", [])):
171
+ with self.subTest(mode=mode):
172
+ a = argparse.Namespace(target="local", package_skills=mode)
173
+ self.assertEqual(
174
+ [t.split("__")[-1] for t in rb.answer_tools(a)],
175
+ list(rb.HOSTED_TOOLS_DEFAULT) + extra)
176
+ denied = {t.split("__")[-1] for t in rb.answer_denied(a)}
177
+ self.assertEqual("get_skill" in denied, mode != "tool")
178
+
168
179
  def test_the_lists_partition_the_publisher_surface(self):
169
180
  # The alarm for the one weakness of an enumerated deny-list: a tool
170
181
  # Publisher gains later is offered to the answerer unless it is placed
171
182
  # on one side or the other, and this is what says so out loud.
172
183
  placed = {t.split("__")[-1] for t in
173
- (*rb.ANSWER_TOOLS, *rb.ANSWER_DENIED)}
184
+ (*rb.ANSWER_TOOLS, *rb.TOOL_ARM_TOOLS, *rb.ANSWER_DENIED)}
174
185
  self.assertEqual(placed, set(rb.PUBLISHER_MCP_TOOLS),
175
186
  "every Publisher MCP tool must be either allowed to "
176
187
  "the answerer or explicitly denied; an unplaced one "
@@ -0,0 +1,263 @@
1
+ #!/usr/bin/env python3
2
+ """Installing a package's own skills into an answerer workspace.
3
+
4
+ Two things can go wrong here and neither announces itself on a live run.
5
+
6
+ The first is contamination. An eval set lives at `<package>/evals/`, so a copy
7
+ that reached one directory too wide would put `cases.jsonl` inside the
8
+ answerer's cwd, where `Read` reaches it. The run would still complete and still
9
+ report a number.
10
+
11
+ The second is a mislabelled arm. `--package-skills=install` that installs
12
+ nothing is the `off` arm wearing the `install` label, and the comparison the
13
+ mode exists to support silently measures the same thing twice.
14
+
15
+ Stdlib only: python3 package_skills_test.py
16
+ """
17
+ import pathlib
18
+ import shutil
19
+ import sys
20
+ import tempfile
21
+ import unittest
22
+
23
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
24
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
25
+ / "eval-answer" / "scripts"))
26
+ import argparse # noqa: E402
27
+ from agent_harness import build_workspace # noqa: E402
28
+ import run_baseline as rb # noqa: E402
29
+
30
+
31
+ def write_skill(root: pathlib.Path, name: str, body: str = "body") -> None:
32
+ d = root / name
33
+ d.mkdir(parents=True, exist_ok=True)
34
+ (d / "SKILL.md").write_text(
35
+ f"---\nname: {name}\ndescription: d\n---\n\n{body}\n")
36
+
37
+
38
+ class InstallPackageSkills(unittest.TestCase):
39
+ def setUp(self) -> None:
40
+ self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="pkgskills-test-"))
41
+ self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
42
+ # The manifest side: skills the harness would install anyway.
43
+ self.skills_root = self.tmp / "skills-root"
44
+ write_skill(self.skills_root, "malloy-analysis", "BUNDLED ANALYSIS")
45
+ write_skill(self.skills_root, "malloy-queries", "BUNDLED QUERIES")
46
+ # The package side, with an eval set beside it.
47
+ self.package = self.tmp / "pkg"
48
+ self.pkg_skills = self.package / "skills"
49
+ write_skill(self.pkg_skills, "house-conventions", "OURS")
50
+ evals = self.package / "evals" / "set-a"
51
+ evals.mkdir(parents=True)
52
+ (evals / "cases.jsonl").write_text('{"qid":"q1","golden":42}\n')
53
+
54
+ def build(self, **kw) -> pathlib.Path:
55
+ work = build_workspace(["malloy-analysis", "malloy-queries"],
56
+ self.skills_root, mcp_url=None, **kw)
57
+ self.addCleanup(shutil.rmtree, work, ignore_errors=True)
58
+ return work
59
+
60
+ def test_installs_the_package_skill_alongside_the_manifest_ones(self):
61
+ work = self.build(package_skills_dir=self.pkg_skills)
62
+ installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
63
+ self.assertEqual(
64
+ installed, ["house-conventions", "malloy-analysis", "malloy-queries"])
65
+
66
+ def test_package_skill_shadows_a_manifest_skill_of_the_same_name(self):
67
+ # The server resolves a same-named package skill by replacing the
68
+ # bundled one. The installed path has to agree, or the two arms of the
69
+ # comparison disagree about which guide is in force.
70
+ write_skill(self.pkg_skills, "malloy-analysis", "OURS WINS")
71
+ work = self.build(package_skills_dir=self.pkg_skills)
72
+ body = (work / ".claude" / "skills" / "malloy-analysis"
73
+ / "SKILL.md").read_text()
74
+ self.assertIn("OURS WINS", body)
75
+ self.assertNotIn("BUNDLED ANALYSIS", body)
76
+
77
+ def test_never_copies_the_eval_set(self):
78
+ work = self.build(package_skills_dir=self.pkg_skills)
79
+ found = [str(p.relative_to(work)) for p in work.rglob("*")
80
+ if p.name in ("cases.jsonl", "set.json", "events.jsonl")
81
+ or p.name == "evals"]
82
+ self.assertEqual(found, [], f"eval artefacts reached the workspace: {found}")
83
+
84
+ def test_does_not_follow_a_link_inside_a_skill_into_the_eval_set(self):
85
+ # skills/x/evals -> ../../evals would otherwise copy the goldens.
86
+ link = self.pkg_skills / "house-conventions" / "evals"
87
+ link.symlink_to(self.package / "evals", target_is_directory=True)
88
+ work = self.build(package_skills_dir=self.pkg_skills)
89
+ found = [str(p.relative_to(work)) for p in work.rglob("*")
90
+ if p.name in ("cases.jsonl", "set.json", "evals")]
91
+ self.assertEqual(found, [], f"a link carried the eval set in: {found}")
92
+
93
+ def test_refuses_a_directory_that_is_not_named_skills(self):
94
+ # The guard is on the directory name rather than on its contents,
95
+ # because "does this tree contain goldens" is a question that can be
96
+ # answered wrongly and "is this the skills directory" cannot.
97
+ with self.assertRaises(ValueError):
98
+ self.build(package_skills_dir=self.package)
99
+
100
+ def test_refuses_a_skills_link_that_points_at_the_eval_set(self):
101
+ shutil.rmtree(self.pkg_skills)
102
+ self.pkg_skills.symlink_to(self.package / "evals", target_is_directory=True)
103
+ with self.assertRaises(ValueError):
104
+ self.build(package_skills_dir=self.pkg_skills)
105
+
106
+ def test_refuses_a_skills_link_even_when_it_resolves_to_a_skills_dir(self):
107
+ real = self.tmp / "elsewhere" / "skills"
108
+ write_skill(real, "x")
109
+ shutil.rmtree(self.pkg_skills)
110
+ self.pkg_skills.symlink_to(real, target_is_directory=True)
111
+ with self.assertRaises(ValueError):
112
+ self.build(package_skills_dir=self.pkg_skills)
113
+
114
+ def test_refuses_a_missing_directory_rather_than_installing_nothing(self):
115
+ with self.assertRaises(FileNotFoundError):
116
+ self.build(package_skills_dir=self.package / "skills-typo")
117
+
118
+ def test_installs_nothing_when_not_asked(self):
119
+ work = self.build()
120
+ installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
121
+ self.assertEqual(installed, ["malloy-analysis", "malloy-queries"])
122
+
123
+ def test_copies_reference_files_beside_the_skill(self):
124
+ # Symlinking put reference/*.md outside the session's allowed
125
+ # directory, so the agent could load SKILL.md and read nothing beside
126
+ # it. Package skills must not reintroduce that.
127
+ ref = self.pkg_skills / "house-conventions" / "reference"
128
+ ref.mkdir(parents=True)
129
+ (ref / "margin.md").write_text("# Margin\n")
130
+ work = self.build(package_skills_dir=self.pkg_skills)
131
+ target = (work / ".claude" / "skills" / "house-conventions"
132
+ / "reference" / "margin.md")
133
+ self.assertTrue(target.is_file())
134
+ self.assertFalse(target.is_symlink())
135
+
136
+
137
+ class ResolveArm(unittest.TestCase):
138
+ """`--package-skills` must describe the run that actually happened.
139
+
140
+ Each mode is a claim about three things that live apart: the flag, the
141
+ files, and whether the deployment serves them at all. When they disagree the
142
+ run still completes and still reports a number, under the wrong label, and
143
+ nothing downstream can tell. So a mismatch has to stop the run.
144
+ """
145
+
146
+ def setUp(self) -> None:
147
+ self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="arm-test-"))
148
+ self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
149
+ self.pkg_skills = self.tmp / "pkg" / "skills"
150
+ write_skill(self.pkg_skills, "house", "OURS")
151
+ # Stub the HTTP layer: nothing here may reach a network.
152
+ self.served_dir = None
153
+ for name, fake in (
154
+ ("package_skill_names", lambda *a, **k: self.served_names),
155
+ ("served_package_dir", lambda *a, **k: self.served_dir)):
156
+ real = getattr(rb, name)
157
+ setattr(rb, name, fake)
158
+ self.addCleanup(setattr, rb, name, real)
159
+ self.served_names: list[str] = []
160
+
161
+ def serve(self, names: list[str]) -> None:
162
+ self.served_names = names
163
+
164
+ def args(self, mode: str, override=None) -> argparse.Namespace:
165
+ return argparse.Namespace(
166
+ package_skills=mode, target="local", publisher="http://x",
167
+ no_answerer_skills=False,
168
+ environment="env", package="pkg",
169
+ package_skills_dir_override=str(override) if override else None)
170
+
171
+ def test_install_with_a_tree_resolves_and_pins_a_sha(self):
172
+ self.serve(["house"])
173
+ a = self.args("install", self.pkg_skills)
174
+ rb.resolve_package_skills(a)
175
+ self.assertEqual(a.package_skills_dir, self.pkg_skills)
176
+ self.assertRegex(a.package_skills_sha, r"^[0-9a-f]{64}$")
177
+
178
+ def test_the_sha_moves_when_a_guide_changes(self):
179
+ self.serve(["house"])
180
+ a = self.args("install", self.pkg_skills)
181
+ rb.resolve_package_skills(a)
182
+ before = a.package_skills_sha
183
+ write_skill(self.pkg_skills, "house", "EDITED")
184
+ b = self.args("install", self.pkg_skills)
185
+ rb.resolve_package_skills(b)
186
+ self.assertNotEqual(before, b.package_skills_sha)
187
+
188
+ def test_install_without_a_tree_refuses(self):
189
+ self.serve(["house"])
190
+ a = self.args("install", self.tmp / "nope" / "skills")
191
+ with self.assertRaises(SystemExit):
192
+ rb.resolve_package_skills(a)
193
+
194
+ def test_tool_with_no_package_skills_is_recorded_as_off(self):
195
+ # get_skill is denied under `off`, so that is what the run measures.
196
+ self.serve([])
197
+ a = self.args("tool")
198
+ rb.resolve_package_skills(a)
199
+ self.assertEqual(a.package_skills, "off")
200
+ self.assertNotIn(rb.TOOL_ARM_TOOLS[0], rb.answer_tools(a))
201
+ self.assertIn(rb.TOOL_ARM_TOOLS[0], rb.answer_denied(a))
202
+
203
+ def test_install_with_no_answerer_skills_is_fatal(self):
204
+ self.serve(["house"])
205
+ a = self.args("install", self.pkg_skills)
206
+ a.no_answerer_skills = True
207
+ with self.assertRaises(SystemExit):
208
+ rb.resolve_package_skills(a)
209
+
210
+ def test_a_replay_carries_the_prior_run_and_resolves_nothing(self):
211
+ # Prior run was install; the server and files have since changed.
212
+ self.serve([])
213
+ a = self.args("tool")
214
+ rb.resolve_package_skills(a, {"packageSkillsMode": "install",
215
+ "packageSkillsSha": "abc"})
216
+ self.assertEqual((a.package_skills, a.package_skills_sha),
217
+ ("install", "abc"))
218
+ self.assertIsNone(a.package_skills_dir)
219
+
220
+ def test_a_replay_of_a_run_that_recorded_nothing_records_nothing(self):
221
+ a = self.args("tool")
222
+ rb.resolve_package_skills(a, {})
223
+ self.assertIsNone(a.package_skills)
224
+ self.assertIsNone(a.package_skills_sha)
225
+
226
+ def test_a_platform_run_records_null_without_probing(self):
227
+ def boom(*a, **k):
228
+ raise AssertionError("probed the REST API")
229
+ rb.package_skill_names = boom
230
+ a = self.args("tool")
231
+ a.target = "platform"
232
+ rb.resolve_package_skills(a)
233
+ self.assertIsNone(a.package_skills)
234
+
235
+ def test_refuses_a_served_skills_dir_that_is_a_link(self):
236
+ pkg = self.tmp / "served"
237
+ (pkg / "evals").mkdir(parents=True)
238
+ (pkg / "skills").symlink_to(pkg / "evals", target_is_directory=True)
239
+ self.served_dir = pkg
240
+ with self.assertRaises(SystemExit):
241
+ rb.resolve_package_skills(self.args("tool"))
242
+
243
+ def test_tool_passes_when_the_server_serves_them(self):
244
+ self.serve(["house"])
245
+ a = self.args("tool")
246
+ rb.resolve_package_skills(a)
247
+ self.assertIsNone(a.package_skills_dir)
248
+
249
+ def test_off_refuses_when_the_server_is_still_serving_them(self):
250
+ # A baseline the answerer can call get_skill against is not a baseline.
251
+ self.serve(["house"])
252
+ with self.assertRaises(SystemExit):
253
+ rb.resolve_package_skills(self.args("off"))
254
+
255
+ def test_off_passes_when_the_server_withholds_them(self):
256
+ self.serve([])
257
+ a = self.args("off")
258
+ rb.resolve_package_skills(a)
259
+ self.assertIsNone(a.package_skills_dir)
260
+
261
+
262
+ if __name__ == "__main__":
263
+ unittest.main(verbosity=2)