@malloy-publisher/skills 0.1.49 → 0.1.51

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/skills",
    "description": "Agent skills for Malloy and the Malloy Publisher",
-   "version": "0.1.49",
+   "version": "0.1.51",
    "license": "MIT",
    "type": "module",
    "main": "dist/index.js",

package/skills/eval-answer/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-answer
 description: 'Score one analytical answer against a verified golden, and score which of the entities the golden depends on retrieval delivered to the answerer. Run the contamination checklist, re-execute the submitted query yourself, then spawn a judge subagent per skill:eval-judge. Append attempt, tool_call, score, and retrieval_score events to the file ledger (reference/ledger-schema.md). Never explain the failure (eval-diagnose) or edit the model (eval-improve). Use when asked whether an answer was correct, to score a run, or to baseline a model.'
-version: 0.1.49
+version: 0.1.51
 ---
 
 # Evaluate One Answer
@@ -19,8 +19,9 @@ revision.
 
 Take the question from the stored case (`evals/<set>/cases.jsonl`), never from
 memory or a truncated console line. Record `question_sha` of the exact text
-the answerer saw. Record `servedRevision` from `get_context` or reload, not
-the package name: a same-named decoy has been measured for hours.
+the answerer saw. Record `servedRevision` from the package's serving state,
+`GET …/packages/{pkg}/status`, or the `reload_package` result, not the package name: a
+same-named decoy has been measured for hours.
 
 ## Step 1: Contamination check, before any score
 

package/skills/eval-answer/scripts/check_findable.py

@@ -209,15 +209,25 @@ def embedding_index(rest_url: str, environment: str, package: str,
     server-side state as a broken answer key. None also covers a server too old
     to send the field.
     """
-    url = (f"{rest_url.rstrip('/')}/api/v0/environments/"
-           f"{urllib.parse.quote(environment)}/packages/"
-           f"{urllib.parse.quote(package)}")
+    package_url = (f"{rest_url.rstrip('/')}/api/v0/environments/"
+                   f"{urllib.parse.quote(environment)}/packages/"
+                   f"{urllib.parse.quote(package)}")
+    # Its own route; a server from before the route carries it on the
+    # package resource instead.
     try:
-        with urllib.request.urlopen(url, timeout=timeout) as r:
-            body = json.loads(r.read())
+        with urllib.request.urlopen(f"{package_url}/embedding-index",
+                                    timeout=timeout) as r:
+            index = json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        if e.code != 404:
+            return None
+        try:
+            with urllib.request.urlopen(package_url, timeout=timeout) as r:
+                index = json.loads(r.read()).get("embeddingIndex")
+        except (urllib.error.URLError, OSError, ValueError):
+            return None
     except (urllib.error.URLError, OSError, ValueError):
         return None
-    index = body.get("embeddingIndex")
     if not isinstance(index, dict) or not isinstance(index.get("status"), str):
         return None
     return index

package/skills/eval-answer/scripts/check_findable_test.py

@@ -531,17 +531,53 @@ class IndexWait(unittest.TestCase):
 
 class EmbeddingIndexRead(unittest.TestCase):
     def read(self, payload):
+        """Read the index from a server holding `payload` as its package
+        resource. A payload carrying `embeddingIndex` is a current server,
+        which serves the object from `GET …/embedding-index`; one without it
+        is an older server, whose `/embedding-index` 404s."""
         import json as _json
 
         class Resp:
+            def __init__(self, body): self.body = body
             def __enter__(self): return self
             def __exit__(self, *a): return False
-            def read(self): return _json.dumps(payload).encode()
+            def read(self): return _json.dumps(self.body).encode()
+
+        def urlopen(url, timeout=None):
+            if url.endswith("/embedding-index"):
+                if "embeddingIndex" in payload:
+                    return Resp(payload["embeddingIndex"])
+                raise check_findable.urllib.error.HTTPError(
+                    url, 404, "Not Found", None, None)
+            return Resp(payload)
 
         with mock.patch.object(check_findable.urllib.request, "urlopen",
-                               lambda *a, **k: Resp()):
+                               urlopen):
             return check_findable.embedding_index("http://x", "e", "p")
 
+    def test_an_older_server_is_read_from_the_package_resource(self):
+        idx = {"status": "ready"}
+        import json as _json
+        calls = []
+
+        class Resp:
+            def __enter__(self): return self
+            def __exit__(self, *a): return False
+            def read(self): return _json.dumps({"embeddingIndex": idx}).encode()
+
+        def urlopen(url, timeout=None):
+            calls.append(url)
+            if url.endswith("/embedding-index"):
+                raise check_findable.urllib.error.HTTPError(
+                    url, 404, "Not Found", None, None)
+            return Resp()
+
+        with mock.patch.object(check_findable.urllib.request, "urlopen",
+                               urlopen):
+            self.assertEqual(
+                check_findable.embedding_index("http://x", "e", "p"), idx)
+        self.assertEqual(len(calls), 2)
+
     def test_the_whole_object_comes_back_so_reason_is_not_lost(self):
         idx = {"status": "error", "reason": "cooldown",
                "lastError": {"message": "m"}}

package/skills/eval-answer/scripts/publisher_rest.py

@@ -205,7 +205,8 @@ def served_model_path(base: str, environment: str, package: str,
 
 
 def package_identity(base: str, environment: str, package: str) -> dict[str, str]:
-    """What the server says it is serving: `sourceContentSha`, `servedRevision`.
+    """What the server says it is serving: `sourceContentSha`, `servedRevision`,
+    read from the package's serving state (`GET …/packages/{pkg}/status`).
 
     Take these rather than computing your own. `sourceContentSha` is a content
     hash over EVERY model path in the package, so it moves when an imported file
@@ -218,10 +219,35 @@ def package_identity(base: str, environment: str, package: str) -> dict[str, str
     directory. So `publisher.json` version pins nothing (nothing reads it) and a
     set of goldens living inside the package needs its own hash.
     """
+    keys = ("sourceContentSha", "servedRevision")
+    pkg_path = (f"api/v0/environments/{urllib.parse.quote(environment)}"
+                f"/packages/{urllib.parse.quote(package)}")
+    # The package's serving state; for a versioned package, its `latest`'s.
     try:
-        d = get_json(base, f"api/v0/environments/{urllib.parse.quote(environment)}"
-                           f"/packages/{urllib.parse.quote(package)}")
+        d = get_json(base, f"{pkg_path}/status")
+        if any(isinstance(d.get(k), str) for k in keys):
+            return {k: d[k] for k in keys if isinstance(d.get(k), str)}
+    except Exception:                                   # noqa: BLE001
+        pass
+    # A server from before that route: the package's `/status` entry carries
+    # them, one entry per version held, and the one serving `latest` answers.
+    try:
+        status = get_json(base, "api/v0/status")
+        for env in status.get("environments") or []:
+            if env.get("name") != environment:
+                continue
+            entries = [p for p in env.get("packages") or []
+                       if p.get("name") == package]
+            entry = next((p for p in entries
+                          if p.get("versionId") in (None, p.get("latestVersion"))),
+                         entries[0] if entries else None)
+            if entry and any(isinstance(entry.get(k), str) for k in keys):
+                return {k: entry[k] for k in keys if isinstance(entry.get(k), str)}
+    except Exception:                                   # noqa: BLE001
+        pass
+    # Older still: on the package resource.
+    try:
+        d = get_json(base, pkg_path)
     except Exception:                                   # noqa: BLE001
         return {}
-    return {k: d[k] for k in ("sourceContentSha", "servedRevision")
-            if isinstance(d.get(k), str)}
+    return {k: d[k] for k in keys if isinstance(d.get(k), str)}

package/skills/eval-answer/scripts/publisher_rest_test.py

@@ -149,6 +149,51 @@ class CompactJson(unittest.TestCase):
         self.assertEqual(rows, [{"n": 7}])
 
 
+
+class PackageIdentityTests(unittest.TestCase):
+    """The served identity comes from the package's serving state, and on older
+    servers from its `/status` entry or the package resource."""
+
+    def identity(self, responses):
+        def get_json(base, path):
+            if path in responses:
+                return responses[path]
+            raise OSError("HTTP Error 404")
+        with unittest.mock.patch.object(publisher_rest, "get_json", get_json):
+            return publisher_rest.package_identity("http://x", "env", "pkg")
+
+    def test_the_serving_state_answers(self):
+        state = {"name": "pkg", "sourceContentSha": "abc", "servedRevision": "r1"}
+        self.assertEqual(
+            self.identity({"api/v0/environments/env/packages/pkg/status": state}),
+            {"sourceContentSha": "abc", "servedRevision": "r1"})
+
+    def test_a_server_without_the_route_answers_from_its_status_entry(self):
+        status = {"environments": [{"name": "env", "packages": [
+            {"name": "other", "sourceContentSha": "zzz"},
+            {"name": "pkg", "sourceContentSha": "abc", "servedRevision": "r1"},
+        ]}]}
+        self.assertEqual(self.identity({"api/v0/status": status}),
+                         {"sourceContentSha": "abc", "servedRevision": "r1"})
+
+    def test_a_versioned_package_answers_from_its_latest_version(self):
+        status = {"environments": [{"name": "env", "packages": [
+            {"name": "pkg", "versionId": "1.0.0", "latestVersion": "1.1.0",
+             "sourceContentSha": "old"},
+            {"name": "pkg", "versionId": "1.1.0", "latestVersion": "1.1.0",
+             "sourceContentSha": "new"},
+        ]}]}
+        self.assertEqual(self.identity({"api/v0/status": status}),
+                         {"sourceContentSha": "new"})
+
+    def test_an_older_server_answers_from_the_package_resource(self):
+        status = {"environments": [{"name": "env", "packages": [{"name": "pkg"}]}]}
+        pkg = {"sourceContentSha": "abc", "servedRevision": "r1"}
+        self.assertEqual(
+            self.identity({"api/v0/status": status,
+                           "api/v0/environments/env/packages/pkg": pkg}),
+            {"sourceContentSha": "abc", "servedRevision": "r1"})
+
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
     unittest.main()

package/skills/eval-diagnose/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-diagnose
 description: 'Diagnose why a scored answer failed and who owns the fix, then cluster the failures by shared root cause. Walk dataset, agent-call, get_context/model, get_context/retrieval, construction, then model-definition. Append issue events to the file ledger, linked by traceId, one per cluster. Use after eval-answer, when triaging a run, or before changing a model. Does not edit the model (eval-improve).'
-version: 0.1.49
+version: 0.1.51
 ---
 
 # Diagnose One Answer

package/skills/eval-import/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-import
 description: 'Turn a list of questions into an eval set, whatever shape it arrived in: a JSONL a customer sent, a CSV, a spreadsheet export, a markdown doc, an email thread, or a pull from production logs. Classify each item by what came WITH the question (a query, a number, prose criteria, or nothing), write cases.jsonl and set.json per reference/case-format.md, keep the file as it arrived, and seal each question so a later edit is detectable. Never marks a value-bearing golden verified and never invents a question. Use when questions arrive from outside and need to become a set, or before the first run against a set nobody here authored.'
-version: 0.1.49
+version: 0.1.51
 ---
 
 # Import questions into an eval set

package/skills/eval-improve/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-improve
 description: 'Make the smallest safe edit that closes a diagnosed gap the model repo owns: a Malloy model edit, or a package skill under skills/. Every factual claim carries a probe receipt. Use after eval-diagnose, or when asked to fix a model so an agent can discover the right answer. Never accepts its own edit; the acceptance check belongs to eval-loop. Does not decide whether an answer was wrong (eval-answer) or why (eval-diagnose).'
-version: 0.1.49
+version: 0.1.51
 ---
 
 # Improve the Model

package/skills/eval-improve/scripts/improve.py

@@ -77,9 +77,34 @@ IMPROVE_TOOLS = ("mcp__publisher__get_context",
                  "Read", "Edit", "Write", "Grep", "Glob",
                  "Bash(bash ./sync_and_reload.sh)")
 
-# REST, not MCP: an MCP reply escapes its inner JSON, so grepping it for the sha matches nothing.
-RELOAD = """curl -s -m 60 "{rest_url}/api/v0/environments/{environment}/packages/{package}?reload=true" \\
-  | grep -o '"sourceContentSha":"[^"]*"' | head -1"""
+# REST, not MCP: an MCP reply escapes its inner JSON. The sha is read from the
+# package's serving state after the reload (the reload response itself carries
+# it only while the server inlines the serving fields). A failed reload prints
+# its error and stops, so the previous load's sha is never shown for it.
+RELOAD = """reload_body=$(mktemp)
+reload_code=$(curl -s -m 60 -o "$reload_body" -w "%{{http_code}}" "{rest_url}/api/v0/environments/{environment}/packages/{package}?reload=true" || echo 000)
+if [ "$reload_code" != "200" ]; then
+  echo "reload failed (HTTP $reload_code):"; cat "$reload_body"; echo; rm -f "$reload_body"; exit 1
+fi
+rm -f "$reload_body"
+python3 -c '
+import json, urllib.request
+base = "{rest_url}/api/v0"
+def get(path):
+    try:
+        with urllib.request.urlopen(base + path, timeout=30) as r:
+            return json.load(r)
+    except Exception:
+        return {{}}
+sha = get("/environments/{environment}/packages/{package}/status").get("sourceContentSha")
+if not sha:  # a server from before that route: the package`s /status entry
+    for e in get("/status").get("environments") or []:
+        for p in (e.get("packages") or []) if e.get("name") == "{environment}" else []:
+            if p.get("name") == "{package}" and p.get("sourceContentSha"):
+                sha = p["sourceContentSha"]
+if sha:
+    print("\\"sourceContentSha\\":\\"%s\\"" % sha)
+'"""
 
 SYNC_SCRIPT = """#!/bin/bash
 set -euo pipefail

package/skills/eval-improve/scripts/improve_test.py

@@ -282,5 +282,68 @@ class Grants(unittest.TestCase):
             self.assertIn(f"mcp__publisher__{t}", improve.IMPROVE_TOOLS)
 
 
+class ReloadReceipt(unittest.TestCase):
+    """The reload helper prints the served sha: from the package's serving
+    state, or from its `/status` entry on a server from before that route.
+    A reload that fails prints its error instead, never the old sha."""
+
+    PKG = "/api/v0/environments/env/packages/pkg"
+
+    def run_reload(self, routes):
+        import http.server
+        import json
+        import threading
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_GET(self):
+                path = self.path.split("?")[0]
+                body = routes.get(path)
+                code, body = body if isinstance(body, tuple) else (
+                    200 if body is not None else 404, body)
+                self.send_response(code)
+                self.end_headers()
+                self.wfile.write(json.dumps(body or {}).encode())
+
+            def log_message(self, *args):
+                pass
+
+        server = http.server.HTTPServer(("127.0.0.1", 0), Handler)
+        threading.Thread(target=server.serve_forever, daemon=True).start()
+        try:
+            cmd = improve.RELOAD.format(
+                rest_url=f"http://127.0.0.1:{server.server_port}",
+                environment="env", package="pkg")
+            return subprocess.run(["bash", "-c", cmd], capture_output=True,
+                                  text=True, timeout=60).stdout.strip()
+        finally:
+            server.shutdown()
+
+    def test_reads_the_serving_state(self):
+        out = self.run_reload({
+            self.PKG: {"name": "pkg"},
+            "/api/v0/environments/env/packages/pkg/status":
+                {"sourceContentSha": "abc"},
+        })
+        self.assertEqual(out, '"sourceContentSha":"abc"')
+
+    def test_falls_back_to_the_status_entry_on_an_older_server(self):
+        out = self.run_reload({
+            self.PKG: {"name": "pkg"},
+            "/api/v0/status": {"environments": [{"name": "env", "packages": [
+                {"name": "pkg", "sourceContentSha": "old"}]}]},
+        })
+        self.assertEqual(out, '"sourceContentSha":"old"')
+
+    def test_a_failed_reload_prints_its_error_and_no_sha(self):
+        out = self.run_reload({
+            self.PKG: (400, {"message": "bad.malloy: refused"}),
+            "/api/v0/environments/env/packages/pkg/status":
+                {"sourceContentSha": "stale"},
+        })
+        self.assertIn("reload failed (HTTP 400)", out)
+        self.assertIn("bad.malloy: refused", out)
+        self.assertNotIn("stale", out)
+
+
 if __name__ == "__main__":
     unittest.main()

package/skills/eval-judge/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-judge
 description: 'Decide whether ONE answer matches its golden, and say whether you believe the golden. Read this before emitting any verdict. Covers containment, column pairing, near_match, refusals, and the gold_status judgement. Use when scoring an attempt in an evaluation run; never to conduct a run (eval-loop), diagnose a failure (eval-diagnose) or edit a model (eval-improve).'
-version: 0.1.49
+version: 0.1.51
 ---
 
 # The judge

package/skills/eval-loop/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-loop
 description: 'Conduct a local Publisher evaluation loop in five steps: scrape/run, eval, diagnose, improve, checkpoint. You are the conductor: import cases into the file ledger, spawn a blind answerer, then run eval-answer, eval-diagnose, and eval-improve. Persistence is plain files: the set in the model package''s evals/ directory, runs in the set''s workdir; checkpoints are git commits of the model repo. `eval.py` runs each step from the set''s eval.toml. Use to score a model, diagnose failures, improve behind an acceptance check, or roll back a bad direction.'
-version: 0.1.49
+version: 0.1.51
 ---
 
 # The Evaluation Loop

package/skills/eval-loop/scripts/serve.py

@@ -53,8 +53,9 @@ run that starts the moment the REST root replies would record those empty
 answers as retrieval misses.
 
 It drives one ranking call (older servers only started the sync on the first
-one), then polls the package resource until `embeddingIndex.status` reaches a
-terminal value and prints it. The values are `lexical` (no provider: a mode),
+one), then polls `GET …/packages/{package}/embedding-index` (the package
+resource's `embeddingIndex` on a server from before that route) until its
+`status` reaches a terminal value and prints it. The values are `lexical` (no provider: a mode),
 `indexing` (the only non-terminal one), `ready`, and `error`, which carries a
 `reason` and `lastError.message` that are printed. Only `ready` licenses
 reading a run's discoverability findings. A package resource with no
@@ -95,6 +96,7 @@ import socket
 import subprocess
 import sys
 import time
+import urllib.error
 import urllib.request
 
 sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent
@@ -203,6 +205,27 @@ def warm_arguments(environment: str, package: str) -> dict:
     }
 
 
+def read_index(port: int, environment: str, package: str) -> dict | None:
+    """The package's embedding-index object, or None when the server has none.
+
+    Read from `GET …/embedding-index`; a server from before that route carries
+    the object on the package resource as `embeddingIndex`, so a 404 there
+    falls back to it. A transport failure raises, for the caller's retry.
+    """
+    package_url = (f"http://localhost:{port}/api/v0/environments/"
+                   f"{environment}/packages/{package}")
+    try:
+        with urllib.request.urlopen(f"{package_url}/embedding-index",
+                                    timeout=10) as r:
+            return json.loads(r.read().decode())
+    except urllib.error.HTTPError as e:
+        if e.code != 404:
+            raise
+    with urllib.request.urlopen(package_url, timeout=10) as r:
+        index = json.loads(r.read().decode()).get("embeddingIndex")
+    return index if isinstance(index, dict) else None
+
+
 def index_status(package_payload: dict) -> str | None:
     """`embeddingIndex.status` from a package resource, or None if absent.
 
@@ -235,13 +258,12 @@ def warm_retrieval(port: int, mcp_port: int, environment: str, package: str,
         return None, f"warm-up call failed: {e}"
 
     url = (f"http://localhost:{port}/api/v0/environments/{environment}"
-           f"/packages/{package}")
+           f"/packages/{package}/embedding-index")
     deadline = time.time() + wait
     status, last_error, read_one, index = None, None, False, {}
     while time.time() < deadline:
         try:
-            with urllib.request.urlopen(url, timeout=10) as r:
-                payload = json.loads(r.read().decode())
+            read = read_index(port, environment, package)
         except Exception as e:  # noqa: BLE001
             # A transport failure is not a capability verdict. Collapsing it
             # into `status = None` handed None a second meaning `index_status`
@@ -255,13 +277,13 @@ def warm_retrieval(port: int, mcp_port: int, environment: str, package: str,
             time.sleep(2)
             continue
         read_one = True
-        status = index_status(payload)
+        status = index_status({"embeddingIndex": read})
         if status is None:
             return None, ("no embeddingIndex on the package resource; this "
                           "server has no embedding provider or predates the "
                           "status field, so every run against it is LEXICAL "
                           "-- do not report discoverability findings from it")
-        index = payload["embeddingIndex"]
+        index = read
         if status in TERMINAL_INDEX_STATES:
             break
         time.sleep(2)

package/skills/eval-loop/scripts/serve_test.py

@@ -163,10 +163,24 @@ class WarmRetrievalSeparatesFailureFromAbsence(unittest.TestCase):
             def read(self):
                 return json.dumps(self.payload).encode()
 
+        # Each scripted outcome is a package-resource payload or an error. A
+        # payload carrying `embeddingIndex` is served the way a current server
+        # serves it, from `GET …/embedding-index`; one without it is an older
+        # server, whose `/embedding-index` 404s and whose package resource
+        # answers instead.
+        pending = []
+
         def urlopen(req, timeout=None):
-            r = next(it)
+            url = req if isinstance(req, str) else req.full_url
+            r = pending.pop() if pending else next(it)
             if isinstance(r, Exception):
                 raise r
+            if url.endswith("/embedding-index"):
+                if "embeddingIndex" in r:
+                    return Resp(r["embeddingIndex"])
+                pending.append(r)
+                raise serve.urllib.error.HTTPError(url, 404, "Not Found",
+                                                   None, None)
             return Resp(r)
 
         with mock.patch.object(serve, "_mcp_call", lambda *a, **k: None), \

package/skills/eval-report/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-report
 description: 'Write the summary a person reads after an evaluation run: what ran, what it scored, which failures were the MODEL and which were the EVAL itself (out of turns, contaminated, unestablished golden, unreadable judge), what was learned, and what to do next. Every claim links to the artifact behind it. Use at the end of any run, before quoting a number to anyone. Never scores an answer (eval-answer) or decides who owns a failure (eval-diagnose).'
-version: 0.1.49
+version: 0.1.51
 ---
 
 # Report a Run

package/skills/malloy-analysis/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: malloy-analysis
 description: "Read before your first get_context call on any business or data question. The required workflow, not background: which source, which rows, how to verify. Skipping it causes confident wrong answers."
-version: 0.1.49
+version: 0.1.51
 ---
 
 <!-- Copyright (c) Credible Data Inc. SPDX-License-Identifier: MIT -->

package/skills/malloy-analysis-report/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: malloy-analysis-report
 description: Combine validated Malloy queries into a report (a linear report, read top to bottom). Use when the user asks to "create a report", "combine these into a report", or wants a persistent multi-query artifact.
-version: 0.1.49
+version: 0.1.51
 ---
 
 <!--

package/skills/malloy-charts/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: malloy-charts
 description: Read before choosing a chart or adding a chart annotation. Chart types, tag syntax and scale rules, KPI cards, dashboards, sparklines, and the renderer mistakes that fail silently.
-version: 0.1.49
+version: 0.1.51
 ---
 <!--
 Copyright (c) Credible Data Inc.

package/skills/malloy-classify/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: malloy-classify
 description: 'Propose #(classification) annotations for a Malloy model from an access policy''s class scheme, backed by aggregate value evidence from the served data, and show them as a diff before writing anything. Use when the user asks to "classify the model", "tag PHI/PII", "label sensitive fields", or adds fields to a package that a policy governs, and before running malloy-compliance-review on a model nobody has classified.'
-version: 0.1.49
+version: 0.1.51
 ---
 
 <!--