@malloy-publisher/skills 0.1.37 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/eval-answer/SKILL.md +1 -1
- package/skills/eval-answer/reference/ledger-schema.md +4 -3
- package/skills/eval-answer/scripts/ledger.py +5 -1
- package/skills/eval-answer/scripts/ledger_test.py +3 -0
- package/skills/eval-answer/scripts/publisher_rest.py +39 -0
- package/skills/eval-diagnose/SKILL.md +43 -18
- package/skills/eval-diagnose/scripts/cluster_failures.py +2 -0
- package/skills/eval-diagnose/scripts/diagnose.py +11 -4
- package/skills/eval-import/SKILL.md +1 -1
- package/skills/eval-improve/SKILL.md +46 -6
- package/skills/eval-improve/scripts/improve.py +45 -16
- package/skills/eval-improve/scripts/improve_test.py +35 -0
- package/skills/eval-judge/SKILL.md +1 -1
- package/skills/eval-loop/SKILL.md +3 -2
- package/skills/eval-loop/reference/acceptance-check.md +12 -0
- package/skills/eval-loop/scripts/agent_harness.py +38 -1
- package/skills/eval-loop/scripts/agent_harness_test.py +12 -1
- package/skills/eval-loop/scripts/package_skills_test.py +263 -0
- package/skills/eval-loop/scripts/run_baseline.py +185 -16
- package/skills/eval-loop/templates/eval-run-package/notebooks/eval_run.malloy +1 -1
- package/skills/eval-report/SKILL.md +1 -1
- package/skills/malloy-analysis/SKILL.md +3 -3
- package/skills/malloy-analysis-report/SKILL.md +14 -14
- package/skills/malloy-charts/SKILL.md +2 -2
- package/skills/malloy-dashboards/SKILL.md +10 -488
- package/skills/malloy-define/SKILL.md +1 -1
- package/skills/malloy-discover/SKILL.md +1 -1
- package/skills/malloy-document/SKILL.md +1 -1
- package/skills/malloy-getting-started/SKILL.md +4 -4
- package/skills/malloy-gotchas-modeling/SKILL.md +1 -1
- package/skills/malloy-html-data-apps/SKILL.md +2 -2
- package/skills/malloy-lookml-review/SKILL.md +1 -1
- package/skills/malloy-materialization/SKILL.md +1 -1
- package/skills/malloy-model/SKILL.md +1 -1
- package/skills/malloy-model/reference/analysis-to-model.md +3 -3
- package/skills/malloy-model-as-you-go/SKILL.md +3 -3
- package/skills/malloy-modeling/SKILL.md +3 -3
- package/skills/malloy-notebooks/SKILL.md +11 -323
- package/skills/malloy-phrase-detection/SKILL.md +1 -1
- package/skills/malloy-powerbi-review/SKILL.md +1 -1
- package/skills/malloy-publish/SKILL.md +8 -9
- package/skills/malloy-queries/SKILL.md +1 -1
- package/skills/malloy-reports/SKILL.md +238 -0
- package/skills/malloy-reports/reference/grid.md +59 -0
- package/skills/malloy-reports/reference/linear.md +180 -0
- package/skills/malloy-review/SKILL.md +1 -1
- package/skills/malloy-review/reference/rubric-structure.md +1 -1
- package/skills/malloy-run-agent/SKILL.md +58 -0
- package/skills/malloy-source-unreachable/SKILL.md +3 -3
|
@@ -50,6 +50,18 @@ Acceptance rules (replacing any vague "results improve"):
|
|
|
50
50
|
- Independent deterministic justification (a probed-wrong definition
|
|
51
51
|
corrected) may accept without a measured win. Record that as the acceptance check
|
|
52
52
|
`reason`.
|
|
53
|
+
- **Package-skill edits (`class: skill`) get the full rules, never the
|
|
54
|
+
documentation shortcut.** A `get_context` probe proves an entity is
|
|
55
|
+
discoverable. Nothing deterministic proves an agent read a guide and behaved
|
|
56
|
+
differently, so the only evidence a skill edit has is a fresh blind
|
|
57
|
+
re-answer: both splits, twice, clearing the flip-count bar. Treat the
|
|
58
|
+
documentation shortcut as unavailable here even when the edit reads like
|
|
59
|
+
documentation, because a guide is prompt text written next to the case it
|
|
60
|
+
failed and is the easiest artifact in this loop to overfit.
|
|
61
|
+
|
|
62
|
+
Read the diff before measuring it, too. A guide that names the case, quotes
|
|
63
|
+
its expected number, or encodes the shape of one question fails the expert
|
|
64
|
+
test in `skill:eval-improve` and is rejected without a run.
|
|
53
65
|
|
|
54
66
|
Write the `acceptance_check` event (decision, class, baseline and final run ids,
|
|
55
67
|
regressions, holdout delta, reason) BEFORE any commit, so a rejected
|
|
@@ -346,7 +346,9 @@ def build_workspace(skills: Iterable[str],
|
|
|
346
346
|
skills_root: pathlib.Path | Iterable[pathlib.Path],
|
|
347
347
|
mcp_url: str | None, prefix: str = "agent-",
|
|
348
348
|
depth: int = 1,
|
|
349
|
-
mcp_server: str = "publisher"
|
|
349
|
+
mcp_server: str = "publisher",
|
|
350
|
+
package_skills_dir: pathlib.Path | None = None
|
|
351
|
+
) -> pathlib.Path:
|
|
350
352
|
"""A scratch cwd holding `.claude/skills/` and an MCP config.
|
|
351
353
|
|
|
352
354
|
COPIED, not symlinked. A symlink loads SKILL.md fine -- the skills mechanism
|
|
@@ -377,6 +379,41 @@ def build_workspace(skills: Iterable[str],
|
|
|
377
379
|
else list(skills_root))
|
|
378
380
|
for name in present:
|
|
379
381
|
shutil.copytree(find_skill(name, roots), dest / name, symlinks=False)
|
|
382
|
+
# A package's own skills, installed last so one whose name matches a
|
|
383
|
+
# manifest skill REPLACES it -- the same shadowing the server does when it
|
|
384
|
+
# serves them, so the installed path and the tool path agree about which
|
|
385
|
+
# guide is in force.
|
|
386
|
+
#
|
|
387
|
+
# Only the package's `skills/` subtree is copied, never the package. An
|
|
388
|
+
# eval set lives at `<package>/evals/`, so copying the package would hand
|
|
389
|
+
# the answerer the goldens.
|
|
390
|
+
if package_skills_dir is not None:
|
|
391
|
+
if not package_skills_dir.is_dir():
|
|
392
|
+
raise FileNotFoundError(
|
|
393
|
+
f"--package-skills=install was asked for but "
|
|
394
|
+
f"{package_skills_dir} is not a directory. Installing nothing "
|
|
395
|
+
f"would silently measure the 'off' arm under the 'install' "
|
|
396
|
+
f"label, so this is fatal.")
|
|
397
|
+
# A skills/ that is a link can point anywhere, `evals/` included, and
|
|
398
|
+
# still pass a name check on the unresolved path.
|
|
399
|
+
if (package_skills_dir.is_symlink()
|
|
400
|
+
or package_skills_dir.resolve().name != "skills"):
|
|
401
|
+
raise ValueError(
|
|
402
|
+
f"refusing to install {package_skills_dir}: expected a real "
|
|
403
|
+
f"directory named 'skills', not a link. Anything wider risks "
|
|
404
|
+
f"copying the eval set into the answerer's workspace.")
|
|
405
|
+
for child in sorted(package_skills_dir.iterdir()):
|
|
406
|
+
if (not child.is_dir() or child.is_symlink()
|
|
407
|
+
or child.name.startswith(".")):
|
|
408
|
+
continue
|
|
409
|
+
target = dest / child.name
|
|
410
|
+
if target.exists():
|
|
411
|
+
shutil.rmtree(target)
|
|
412
|
+
# A link inside a skill could point at the eval set; copy none.
|
|
413
|
+
shutil.copytree(child, target, symlinks=True,
|
|
414
|
+
ignore=lambda src, names: [
|
|
415
|
+
n for n in names
|
|
416
|
+
if (pathlib.Path(src) / n).is_symlink()])
|
|
380
417
|
if mcp_url:
|
|
381
418
|
(work / "mcp.json").write_text(json.dumps(
|
|
382
419
|
{"mcpServers": {mcp_server: {"type": "http", "url": mcp_url}}}))
|
|
@@ -165,12 +165,23 @@ class TheAnswerersMcpSurface(unittest.TestCase):
|
|
|
165
165
|
tuple(t.split("__")[-1] for t in rb.ANSWER_TOOLS),
|
|
166
166
|
rb.HOSTED_TOOLS_DEFAULT)
|
|
167
167
|
|
|
168
|
+
def test_the_tool_arm_is_the_default_plus_get_skill(self):
|
|
169
|
+
for mode, extra in (("tool", ["get_skill"]), ("install", []),
|
|
170
|
+
("off", [])):
|
|
171
|
+
with self.subTest(mode=mode):
|
|
172
|
+
a = argparse.Namespace(target="local", package_skills=mode)
|
|
173
|
+
self.assertEqual(
|
|
174
|
+
[t.split("__")[-1] for t in rb.answer_tools(a)],
|
|
175
|
+
list(rb.HOSTED_TOOLS_DEFAULT) + extra)
|
|
176
|
+
denied = {t.split("__")[-1] for t in rb.answer_denied(a)}
|
|
177
|
+
self.assertEqual("get_skill" in denied, mode != "tool")
|
|
178
|
+
|
|
168
179
|
def test_the_lists_partition_the_publisher_surface(self):
|
|
169
180
|
# The alarm for the one weakness of an enumerated deny-list: a tool
|
|
170
181
|
# Publisher gains later is offered to the answerer unless it is placed
|
|
171
182
|
# on one side or the other, and this is what says so out loud.
|
|
172
183
|
placed = {t.split("__")[-1] for t in
|
|
173
|
-
(*rb.ANSWER_TOOLS, *rb.ANSWER_DENIED)}
|
|
184
|
+
(*rb.ANSWER_TOOLS, *rb.TOOL_ARM_TOOLS, *rb.ANSWER_DENIED)}
|
|
174
185
|
self.assertEqual(placed, set(rb.PUBLISHER_MCP_TOOLS),
|
|
175
186
|
"every Publisher MCP tool must be either allowed to "
|
|
176
187
|
"the answerer or explicitly denied; an unplaced one "
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Installing a package's own skills into an answerer workspace.
|
|
3
|
+
|
|
4
|
+
Two things can go wrong here and neither announces itself on a live run.
|
|
5
|
+
|
|
6
|
+
The first is contamination. An eval set lives at `<package>/evals/`, so a copy
|
|
7
|
+
that reached one directory too wide would put `cases.jsonl` inside the
|
|
8
|
+
answerer's cwd, where `Read` reaches it. The run would still complete and still
|
|
9
|
+
report a number.
|
|
10
|
+
|
|
11
|
+
The second is a mislabelled arm. `--package-skills=install` that installs
|
|
12
|
+
nothing is the `off` arm wearing the `install` label, and the comparison the
|
|
13
|
+
mode exists to support silently measures the same thing twice.
|
|
14
|
+
|
|
15
|
+
Stdlib only: python3 package_skills_test.py
|
|
16
|
+
"""
|
|
17
|
+
import pathlib
|
|
18
|
+
import shutil
|
|
19
|
+
import sys
|
|
20
|
+
import tempfile
|
|
21
|
+
import unittest
|
|
22
|
+
|
|
23
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
24
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
|
|
25
|
+
/ "eval-answer" / "scripts"))
|
|
26
|
+
import argparse # noqa: E402
|
|
27
|
+
from agent_harness import build_workspace # noqa: E402
|
|
28
|
+
import run_baseline as rb # noqa: E402
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def write_skill(root: pathlib.Path, name: str, body: str = "body") -> None:
|
|
32
|
+
d = root / name
|
|
33
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
(d / "SKILL.md").write_text(
|
|
35
|
+
f"---\nname: {name}\ndescription: d\n---\n\n{body}\n")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class InstallPackageSkills(unittest.TestCase):
|
|
39
|
+
def setUp(self) -> None:
|
|
40
|
+
self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="pkgskills-test-"))
|
|
41
|
+
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
|
|
42
|
+
# The manifest side: skills the harness would install anyway.
|
|
43
|
+
self.skills_root = self.tmp / "skills-root"
|
|
44
|
+
write_skill(self.skills_root, "malloy-analysis", "BUNDLED ANALYSIS")
|
|
45
|
+
write_skill(self.skills_root, "malloy-queries", "BUNDLED QUERIES")
|
|
46
|
+
# The package side, with an eval set beside it.
|
|
47
|
+
self.package = self.tmp / "pkg"
|
|
48
|
+
self.pkg_skills = self.package / "skills"
|
|
49
|
+
write_skill(self.pkg_skills, "house-conventions", "OURS")
|
|
50
|
+
evals = self.package / "evals" / "set-a"
|
|
51
|
+
evals.mkdir(parents=True)
|
|
52
|
+
(evals / "cases.jsonl").write_text('{"qid":"q1","golden":42}\n')
|
|
53
|
+
|
|
54
|
+
def build(self, **kw) -> pathlib.Path:
|
|
55
|
+
work = build_workspace(["malloy-analysis", "malloy-queries"],
|
|
56
|
+
self.skills_root, mcp_url=None, **kw)
|
|
57
|
+
self.addCleanup(shutil.rmtree, work, ignore_errors=True)
|
|
58
|
+
return work
|
|
59
|
+
|
|
60
|
+
def test_installs_the_package_skill_alongside_the_manifest_ones(self):
|
|
61
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
62
|
+
installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
|
|
63
|
+
self.assertEqual(
|
|
64
|
+
installed, ["house-conventions", "malloy-analysis", "malloy-queries"])
|
|
65
|
+
|
|
66
|
+
def test_package_skill_shadows_a_manifest_skill_of_the_same_name(self):
|
|
67
|
+
# The server resolves a same-named package skill by replacing the
|
|
68
|
+
# bundled one. The installed path has to agree, or the two arms of the
|
|
69
|
+
# comparison disagree about which guide is in force.
|
|
70
|
+
write_skill(self.pkg_skills, "malloy-analysis", "OURS WINS")
|
|
71
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
72
|
+
body = (work / ".claude" / "skills" / "malloy-analysis"
|
|
73
|
+
/ "SKILL.md").read_text()
|
|
74
|
+
self.assertIn("OURS WINS", body)
|
|
75
|
+
self.assertNotIn("BUNDLED ANALYSIS", body)
|
|
76
|
+
|
|
77
|
+
def test_never_copies_the_eval_set(self):
|
|
78
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
79
|
+
found = [str(p.relative_to(work)) for p in work.rglob("*")
|
|
80
|
+
if p.name in ("cases.jsonl", "set.json", "events.jsonl")
|
|
81
|
+
or p.name == "evals"]
|
|
82
|
+
self.assertEqual(found, [], f"eval artefacts reached the workspace: {found}")
|
|
83
|
+
|
|
84
|
+
def test_does_not_follow_a_link_inside_a_skill_into_the_eval_set(self):
|
|
85
|
+
# skills/x/evals -> ../../evals would otherwise copy the goldens.
|
|
86
|
+
link = self.pkg_skills / "house-conventions" / "evals"
|
|
87
|
+
link.symlink_to(self.package / "evals", target_is_directory=True)
|
|
88
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
89
|
+
found = [str(p.relative_to(work)) for p in work.rglob("*")
|
|
90
|
+
if p.name in ("cases.jsonl", "set.json", "evals")]
|
|
91
|
+
self.assertEqual(found, [], f"a link carried the eval set in: {found}")
|
|
92
|
+
|
|
93
|
+
def test_refuses_a_directory_that_is_not_named_skills(self):
|
|
94
|
+
# The guard is on the directory name rather than on its contents,
|
|
95
|
+
# because "does this tree contain goldens" is a question that can be
|
|
96
|
+
# answered wrongly and "is this the skills directory" cannot.
|
|
97
|
+
with self.assertRaises(ValueError):
|
|
98
|
+
self.build(package_skills_dir=self.package)
|
|
99
|
+
|
|
100
|
+
def test_refuses_a_skills_link_that_points_at_the_eval_set(self):
|
|
101
|
+
shutil.rmtree(self.pkg_skills)
|
|
102
|
+
self.pkg_skills.symlink_to(self.package / "evals", target_is_directory=True)
|
|
103
|
+
with self.assertRaises(ValueError):
|
|
104
|
+
self.build(package_skills_dir=self.pkg_skills)
|
|
105
|
+
|
|
106
|
+
def test_refuses_a_skills_link_even_when_it_resolves_to_a_skills_dir(self):
|
|
107
|
+
real = self.tmp / "elsewhere" / "skills"
|
|
108
|
+
write_skill(real, "x")
|
|
109
|
+
shutil.rmtree(self.pkg_skills)
|
|
110
|
+
self.pkg_skills.symlink_to(real, target_is_directory=True)
|
|
111
|
+
with self.assertRaises(ValueError):
|
|
112
|
+
self.build(package_skills_dir=self.pkg_skills)
|
|
113
|
+
|
|
114
|
+
def test_refuses_a_missing_directory_rather_than_installing_nothing(self):
|
|
115
|
+
with self.assertRaises(FileNotFoundError):
|
|
116
|
+
self.build(package_skills_dir=self.package / "skills-typo")
|
|
117
|
+
|
|
118
|
+
def test_installs_nothing_when_not_asked(self):
|
|
119
|
+
work = self.build()
|
|
120
|
+
installed = sorted(p.name for p in (work / ".claude" / "skills").iterdir())
|
|
121
|
+
self.assertEqual(installed, ["malloy-analysis", "malloy-queries"])
|
|
122
|
+
|
|
123
|
+
def test_copies_reference_files_beside_the_skill(self):
|
|
124
|
+
# Symlinking put reference/*.md outside the session's allowed
|
|
125
|
+
# directory, so the agent could load SKILL.md and read nothing beside
|
|
126
|
+
# it. Package skills must not reintroduce that.
|
|
127
|
+
ref = self.pkg_skills / "house-conventions" / "reference"
|
|
128
|
+
ref.mkdir(parents=True)
|
|
129
|
+
(ref / "margin.md").write_text("# Margin\n")
|
|
130
|
+
work = self.build(package_skills_dir=self.pkg_skills)
|
|
131
|
+
target = (work / ".claude" / "skills" / "house-conventions"
|
|
132
|
+
/ "reference" / "margin.md")
|
|
133
|
+
self.assertTrue(target.is_file())
|
|
134
|
+
self.assertFalse(target.is_symlink())
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class ResolveArm(unittest.TestCase):
|
|
138
|
+
"""`--package-skills` must describe the run that actually happened.
|
|
139
|
+
|
|
140
|
+
Each mode is a claim about three things that live apart: the flag, the
|
|
141
|
+
files, and whether the deployment serves them at all. When they disagree the
|
|
142
|
+
run still completes and still reports a number, under the wrong label, and
|
|
143
|
+
nothing downstream can tell. So a mismatch has to stop the run.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
def setUp(self) -> None:
|
|
147
|
+
self.tmp = pathlib.Path(tempfile.mkdtemp(prefix="arm-test-"))
|
|
148
|
+
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
|
|
149
|
+
self.pkg_skills = self.tmp / "pkg" / "skills"
|
|
150
|
+
write_skill(self.pkg_skills, "house", "OURS")
|
|
151
|
+
# Stub the HTTP layer: nothing here may reach a network.
|
|
152
|
+
self.served_dir = None
|
|
153
|
+
for name, fake in (
|
|
154
|
+
("package_skill_names", lambda *a, **k: self.served_names),
|
|
155
|
+
("served_package_dir", lambda *a, **k: self.served_dir)):
|
|
156
|
+
real = getattr(rb, name)
|
|
157
|
+
setattr(rb, name, fake)
|
|
158
|
+
self.addCleanup(setattr, rb, name, real)
|
|
159
|
+
self.served_names: list[str] = []
|
|
160
|
+
|
|
161
|
+
def serve(self, names: list[str]) -> None:
|
|
162
|
+
self.served_names = names
|
|
163
|
+
|
|
164
|
+
def args(self, mode: str, override=None) -> argparse.Namespace:
|
|
165
|
+
return argparse.Namespace(
|
|
166
|
+
package_skills=mode, target="local", publisher="http://x",
|
|
167
|
+
no_answerer_skills=False,
|
|
168
|
+
environment="env", package="pkg",
|
|
169
|
+
package_skills_dir_override=str(override) if override else None)
|
|
170
|
+
|
|
171
|
+
def test_install_with_a_tree_resolves_and_pins_a_sha(self):
|
|
172
|
+
self.serve(["house"])
|
|
173
|
+
a = self.args("install", self.pkg_skills)
|
|
174
|
+
rb.resolve_package_skills(a)
|
|
175
|
+
self.assertEqual(a.package_skills_dir, self.pkg_skills)
|
|
176
|
+
self.assertRegex(a.package_skills_sha, r"^[0-9a-f]{64}$")
|
|
177
|
+
|
|
178
|
+
def test_the_sha_moves_when_a_guide_changes(self):
|
|
179
|
+
self.serve(["house"])
|
|
180
|
+
a = self.args("install", self.pkg_skills)
|
|
181
|
+
rb.resolve_package_skills(a)
|
|
182
|
+
before = a.package_skills_sha
|
|
183
|
+
write_skill(self.pkg_skills, "house", "EDITED")
|
|
184
|
+
b = self.args("install", self.pkg_skills)
|
|
185
|
+
rb.resolve_package_skills(b)
|
|
186
|
+
self.assertNotEqual(before, b.package_skills_sha)
|
|
187
|
+
|
|
188
|
+
def test_install_without_a_tree_refuses(self):
|
|
189
|
+
self.serve(["house"])
|
|
190
|
+
a = self.args("install", self.tmp / "nope" / "skills")
|
|
191
|
+
with self.assertRaises(SystemExit):
|
|
192
|
+
rb.resolve_package_skills(a)
|
|
193
|
+
|
|
194
|
+
def test_tool_with_no_package_skills_is_recorded_as_off(self):
|
|
195
|
+
# get_skill is denied under `off`, so that is what the run measures.
|
|
196
|
+
self.serve([])
|
|
197
|
+
a = self.args("tool")
|
|
198
|
+
rb.resolve_package_skills(a)
|
|
199
|
+
self.assertEqual(a.package_skills, "off")
|
|
200
|
+
self.assertNotIn(rb.TOOL_ARM_TOOLS[0], rb.answer_tools(a))
|
|
201
|
+
self.assertIn(rb.TOOL_ARM_TOOLS[0], rb.answer_denied(a))
|
|
202
|
+
|
|
203
|
+
def test_install_with_no_answerer_skills_is_fatal(self):
|
|
204
|
+
self.serve(["house"])
|
|
205
|
+
a = self.args("install", self.pkg_skills)
|
|
206
|
+
a.no_answerer_skills = True
|
|
207
|
+
with self.assertRaises(SystemExit):
|
|
208
|
+
rb.resolve_package_skills(a)
|
|
209
|
+
|
|
210
|
+
def test_a_replay_carries_the_prior_run_and_resolves_nothing(self):
|
|
211
|
+
# Prior run was install; the server and files have since changed.
|
|
212
|
+
self.serve([])
|
|
213
|
+
a = self.args("tool")
|
|
214
|
+
rb.resolve_package_skills(a, {"packageSkillsMode": "install",
|
|
215
|
+
"packageSkillsSha": "abc"})
|
|
216
|
+
self.assertEqual((a.package_skills, a.package_skills_sha),
|
|
217
|
+
("install", "abc"))
|
|
218
|
+
self.assertIsNone(a.package_skills_dir)
|
|
219
|
+
|
|
220
|
+
def test_a_replay_of_a_run_that_recorded_nothing_records_nothing(self):
|
|
221
|
+
a = self.args("tool")
|
|
222
|
+
rb.resolve_package_skills(a, {})
|
|
223
|
+
self.assertIsNone(a.package_skills)
|
|
224
|
+
self.assertIsNone(a.package_skills_sha)
|
|
225
|
+
|
|
226
|
+
def test_a_platform_run_records_null_without_probing(self):
|
|
227
|
+
def boom(*a, **k):
|
|
228
|
+
raise AssertionError("probed the REST API")
|
|
229
|
+
rb.package_skill_names = boom
|
|
230
|
+
a = self.args("tool")
|
|
231
|
+
a.target = "platform"
|
|
232
|
+
rb.resolve_package_skills(a)
|
|
233
|
+
self.assertIsNone(a.package_skills)
|
|
234
|
+
|
|
235
|
+
def test_refuses_a_served_skills_dir_that_is_a_link(self):
|
|
236
|
+
pkg = self.tmp / "served"
|
|
237
|
+
(pkg / "evals").mkdir(parents=True)
|
|
238
|
+
(pkg / "skills").symlink_to(pkg / "evals", target_is_directory=True)
|
|
239
|
+
self.served_dir = pkg
|
|
240
|
+
with self.assertRaises(SystemExit):
|
|
241
|
+
rb.resolve_package_skills(self.args("tool"))
|
|
242
|
+
|
|
243
|
+
def test_tool_passes_when_the_server_serves_them(self):
|
|
244
|
+
self.serve(["house"])
|
|
245
|
+
a = self.args("tool")
|
|
246
|
+
rb.resolve_package_skills(a)
|
|
247
|
+
self.assertIsNone(a.package_skills_dir)
|
|
248
|
+
|
|
249
|
+
def test_off_refuses_when_the_server_is_still_serving_them(self):
|
|
250
|
+
# A baseline the answerer can call get_skill against is not a baseline.
|
|
251
|
+
self.serve(["house"])
|
|
252
|
+
with self.assertRaises(SystemExit):
|
|
253
|
+
rb.resolve_package_skills(self.args("off"))
|
|
254
|
+
|
|
255
|
+
def test_off_passes_when_the_server_withholds_them(self):
|
|
256
|
+
self.serve([])
|
|
257
|
+
a = self.args("off")
|
|
258
|
+
rb.resolve_package_skills(a)
|
|
259
|
+
self.assertIsNone(a.package_skills_dir)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
if __name__ == "__main__":
|
|
263
|
+
unittest.main(verbosity=2)
|