@malloy-publisher/skills 0.1.21 → 0.1.23

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/skills",
    "description": "Agent skills for Malloy and the Malloy Publisher",
-   "version": "0.1.21",
+   "version": "0.1.23",
    "license": "MIT",
    "type": "module",
    "main": "dist/index.js",

package/skills/eval-answer/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-answer
 description: 'Score one analytical answer against a verified golden, and score which of the entities the golden depends on retrieval delivered to the answerer. Run the contamination checklist, re-execute the submitted query yourself, then spawn a judge subagent per skill:eval-judge. Append attempt, tool_call, score, and retrieval_score events to the file ledger (reference/ledger-schema.md). Never explain the failure (eval-diagnose) or edit the model (eval-improve). Use when asked whether an answer was correct, to score a run, or to baseline a model.'
-version: 0.1.21
+version: 0.1.23
 ---
 
 # Evaluate One Answer

package/skills/eval-answer/scripts/check_findable.py

@@ -51,6 +51,7 @@ import argparse
 import json
 import pathlib
 import sys
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -176,6 +177,30 @@ def compiled_entities(rest_url: str, environment: str,
     return out or None
 
 
+def embedding_index_status(rest_url: str, environment: str, package: str,
+                           timeout: int = 15) -> str | None:
+    """`embeddingIndex.status` for one package, or None if it cannot be read.
+
+    Retrieval is semantic only once this reaches `ready`; before that the
+    server answers the same calls from the lexical matcher without saying so,
+    and fewer entities come back. A checker that does not wait for it reports
+    the difference as a broken answer key.
+    """
+    url = (f"{rest_url.rstrip('/')}/api/v0/environments/"
+           f"{urllib.parse.quote(environment)}/packages/"
+           f"{urllib.parse.quote(package)}")
+    try:
+        with urllib.request.urlopen(url, timeout=timeout) as r:
+            body = json.loads(r.read())
+    except (urllib.error.URLError, OSError, ValueError):
+        return None
+    index = body.get("embeddingIndex")
+    if not isinstance(index, dict):
+        return None
+    status = index.get("status")
+    return status if isinstance(status, str) else None
+
+
 def stale_packages(rest_url: str, timeout: int = 30) -> set[tuple[str, str]] | None:
     """`{(environment, package)}` serving a model older than their files.
 
@@ -214,6 +239,26 @@ def declared_findings(cases: list[dict[str, Any]],
             continue
         if f"{kind}:{name}" in declared[src]:
             continue
+        # A DOTTED name is a join path, not a field name. `declared[src]` holds
+        # the source's OWN fields, so `dimension:order_items:products.brand`
+        # never matched and every dotted id was reported as a missing field --
+        # five of them on one set, against a get_context that returns each as
+        # its top result. Resolve it where it lives: the last segment is the
+        # field, the segment before it names the joined source.
+        if "." in name:
+            *path, leaf = name.split(".")
+            hop = path[-1]
+            if hop in declared and f"{kind}:{leaf}" in declared[hop]:
+                continue
+            if hop not in declared:
+                out.append(f"{eid}: the join path names {hop!r}, which the "
+                           f"compiled model has no source for "
+                           f"(required by {', '.join(qids)})")
+            else:
+                out.append(f"{eid}: the compiled {hop} source declares no "
+                           f"{kind} {leaf!r} to reach through that join "
+                           f"(required by {', '.join(qids)})")
+            continue
         other = sorted(k.split(":", 1)[0] for k in declared[src]
                        if k.split(":", 1)[1] == name)
         if other:
@@ -310,6 +355,11 @@ def main(argv: list[str] | None = None) -> int:
                          "both directions.")
     ap.add_argument("--out", default=None,
                     help="write the per-entity rows as JSON")
+    ap.add_argument("--index-wait", type=int, default=300,
+                    help="seconds to wait for the embedding index to be ready "
+                         "before searching (needs --publisher). 0 to skip the "
+                         "wait, which risks reading the lexical matcher as a "
+                         "missing entity")
     a = ap.parse_args(argv)
 
     f = a.set_dir / "cases.jsonl"
@@ -318,6 +368,55 @@ def main(argv: list[str] | None = None) -> int:
         return 3
     cases = [json.loads(line) for line in f.read_text().splitlines() if line.strip()]
 
+    # The embedding index is built lazily, on the first call that ranks. A
+    # search run seconds after a server start is answered by the LEXICAL
+    # matcher, which finds fewer things -- and this check reads "fewer things"
+    # as "the key names entities that cannot be retrieved" and tells you to fix
+    # the key. Three ids were reported missing that way on one set and all
+    # three passed on a re-run. So wait for it, and say so when it cannot.
+    if a.publisher and a.index_wait:
+        # The server's enum is indexing | ready | cooldown | too-many-entities.
+        # `cooldown` (the embedding provider failed recently) and
+        # `too-many-entities` (over the cap, permanently lexical) will never
+        # become `ready`, so waiting on them burns the whole timeout for a
+        # result that is knowable on the first read.
+        TERMINAL = ("ready", "cooldown", "too-many-entities")
+        # `indexing` does NOT clear on its own: the docs say it clears "once a
+        # get_context question triggers a sync, which is what builds the
+        # index". Polling the package resource never triggers one, so a wait
+        # that only polls is a guaranteed stall on a freshly loaded package.
+        # One ranking call first, as serve.py --warm-retrieval does.
+        try:
+            get_context(a.mcp_url, [{"target_type": "source"}],
+                        a.environment, a.package)
+        except (urllib.error.URLError, RuntimeError, OSError):
+            pass                      # the search below reports it properly
+        deadline = time.monotonic() + a.index_wait
+        status = embedding_index_status(a.publisher, a.environment, a.package)
+        while status not in TERMINAL and status is not None \
+                and time.monotonic() < deadline:
+            time.sleep(2)
+            status = embedding_index_status(a.publisher, a.environment,
+                                            a.package)
+        if status is None:
+            print("! the embedding index status could not be read; a miss "
+                  "below may be the lexical matcher rather than the key",
+                  file=sys.stderr)
+        elif status == "too-many-entities":
+            print("! this package is over the embedding cap, so retrieval is "
+                  "permanently lexical here. The misses below are real for "
+                  "this server and say nothing about a semantic one",
+                  file=sys.stderr)
+        elif status == "cooldown":
+            print("! the embedding provider is in cooldown, so retrieval is "
+                  "lexical for now. Re-run rather than acting on a miss below",
+                  file=sys.stderr)
+        elif status != "ready":
+            print(f"! the embedding index is {status!r} after "
+                  f"{a.index_wait}s, so this search may be answered "
+                  f"lexically; a miss below may be the index, not the key",
+                  file=sys.stderr)
+
     declared_out: list[str] = []
     if a.publisher:
         # Staleness first. A stale package answers the models endpoint normally

package/skills/eval-answer/scripts/check_findable_test.py

@@ -322,5 +322,43 @@ class Staleness(unittest.TestCase):
         self.assertIsNone(check_findable.stale_packages("http://127.0.0.1:9"))
 
 
+class DottedJoinPaths(unittest.TestCase):
+    """A dotted name is a join path, resolved on the joined source.
+
+    `declared[src]` holds a source's OWN fields, so every dotted id read as a
+    field the source does not declare. On the storefront tour set that was five
+    of eight reported misses, each one an id get_context returns as its top
+    result -- and the check ends by telling you to fix the key, so a false miss
+    here sends you to break an answer key that is correct.
+    """
+
+    DECLARED = {
+        "order_items": {"source:order_items", "measure:total_sales"},
+        "products": {"source:products", "dimension:brand", "measure:product_count"},
+    }
+
+    def findings(self, eid):
+        cases = [{"qid": "q1", "expectedEntities": {"required": [eid]}}]
+        return check_findable.declared_findings(cases, self.DECLARED)
+
+    def test_a_reachable_join_path_is_not_a_finding(self):
+        self.assertEqual(self.findings("dimension:order_items:products.brand"), [])
+
+    def test_the_kind_still_has_to_match_on_the_joined_source(self):
+        out = self.findings("measure:order_items:products.brand")
+        self.assertEqual(len(out), 1)
+        self.assertIn("declares no measure 'brand'", out[0])
+
+    def test_an_unknown_hop_is_named_as_the_hop(self):
+        out = self.findings("dimension:order_items:suppliers.name")
+        self.assertEqual(len(out), 1)
+        self.assertIn("'suppliers'", out[0])
+
+    def test_a_field_missing_on_the_joined_source_is_a_finding(self):
+        out = self.findings("dimension:order_items:products.colour")
+        self.assertEqual(len(out), 1)
+        self.assertIn("'colour'", out[0])
+
+
 if __name__ == "__main__":
     unittest.main()

package/skills/eval-answer/scripts/score_retrieval.py

@@ -528,7 +528,17 @@ def cascade(rows: list[dict[str, Any]]) -> dict[str, int]:
          # pins that identity never caught it because every row it builds
          # supplies a required entity.
          "passed_not_covered": 0, "passed_not_retrieved": 0,
-         "passed_unmeasured": 0, "passed_no_entities_named": 0}
+         "passed_unmeasured": 0, "passed_no_entities_named": 0,
+         # Recall, tallied OUTSIDE the funnel. The funnel is an elif chain, so a
+         # row whose coverage was never measured stops at rung 1 and never
+         # reaches the recall check -- `not retrieved` is then structurally 0,
+         # and a display that subtracts it from a denominator reports every
+         # retrieval as a success. That is what happened: 12 cases, coverage
+         # unknown, three of them at recall 0.5, printed "retrieved? 12 yes, 0
+         # no". These two count every row whose recall was actually computed and
+         # whose coverage is not a KNOWN gap, so the rung says what was measured
+         # whether or not coverage ran.
+         "recall_scored": 0, "recall_short": 0}
     for r in rows:
         cov = r["coverage"]
         passed = not r["failed"] and r["verdict"] not in UNSCORED
@@ -550,6 +560,9 @@ def cascade(rows: list[dict[str, Any]]) -> dict[str, int]:
             c["delivered, wrong"] += 1
         else:
             c["delivered, right"] += 1
+        if cov not in MEASURED_GAPS and r["recall"] is not None:
+            c["recall_scored"] += 1
+            c["recall_short"] += r["recall"] < 1.0
     return c
 
 

package/skills/eval-answer/scripts/score_retrieval_test.py

@@ -437,8 +437,15 @@ class Cascade(unittest.TestCase):
     # `passed_not_covered` and `passed_not_retrieved` OVERLAY the rungs: they
     # re-count rows already counted by "not covered" and "not retrieved". They
     # are not rungs and must never be added to them.
+    #
+    # `recall_scored` / `recall_short` are a parallel TALLY, not a rung either.
+    # The rungs are an elif chain, so a row whose coverage was never measured
+    # stops at the first one and its recall is never classified; these two
+    # count recall over every row that carried one, which is what the retrieval
+    # line reports. Same rows, counted on a second axis -- so they overlay.
     OVERLAY = ("total", "passed_not_covered", "passed_not_retrieved",
-               "passed_unmeasured", "passed_no_entities_named")
+               "passed_unmeasured", "passed_no_entities_named",
+               "recall_scored", "recall_short")
 
     def test_every_row_lands_on_exactly_one_rung(self):
         c = cascade(self.rows())

package/skills/eval-answer/scripts/verify_goldens.py

@@ -518,6 +518,49 @@ def promotion_blocker(case: dict[str, Any], set_dir: pathlib.Path) -> str | None
     return None
 
 
+# A figure is a number a reader would check: 8,817.56, 943, 25%. Years and
+# small ordinals are not -- "top 3", "2025" -- so they do not raise a finding.
+FIGURE = re.compile(r"(?<![\w.])(\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+\.\d+|\d{3,})")
+
+
+def criteria_figure_findings(case: dict[str, Any]) -> list[str]:
+    """Numbers quoted inside a `criteria` golden, which nothing can verify.
+
+    A criteria golden is exempt from both numeric checks and legitimately so:
+    it holds no value to re-derive, and the rubric-figure check compares
+    figures against the golden's ROWS, which it does not have. The clauses are
+    prose and prose cannot drift.
+
+    Its prose can still assert numbers, and then the exemption is a hole rather
+    than a simplification. This set shipped one: a criteria golden's note said
+    "by order count it is Amelia Cohen (80 orders)". Both halves were wrong --
+    at the grain the same rubric requires, the top by order count is the same
+    customer who tops spend, and 80 is three customers sharing a name, summed
+    by exactly the grouping the neighbouring case exists to forbid. Every check
+    passed it, because every check skips this kind of golden.
+
+    Reported as review, not failure: quoting a figure is often right, and the
+    point is that a human has to be the one who checked it.
+    """
+    g = case.get("golden") or {}
+    if g.get("kind") != "criteria":
+        return []
+    out = []
+    for field, text in (("rubric", g.get("rubric")),
+                        ("verification.note", (g.get("verification") or {}).get("note"))):
+        if not isinstance(text, str):
+            continue
+        figs = sorted(set(FIGURE.findall(text)))
+        if figs:
+            out.append(f"review {case['qid']}: the criteria golden's {field} quotes "
+                       f"{', '.join(figs[:6])}"
+                       f"{' and more' if len(figs) > 6 else ''}, and NOTHING in this "
+                       f"toolchain can check a figure in a criteria golden -- it has no "
+                       f"rows for the rubric-figure check and no value to re-derive. "
+                       f"Query each one at the grain this rubric requires, or drop it")
+    return out
+
+
 def axis_findings(case: dict[str, Any], set_dir: pathlib.Path) -> list[str]:
     g = case.get("golden") or {}
     if g.get("kind") == "unanswerable":
@@ -1021,6 +1064,7 @@ def verify(set_dir: pathlib.Path, publisher: str, environment: str,
     for c in chosen:
         findings += rubric_number_findings(c)
         findings += axis_findings(c, set_dir)
+        findings += criteria_figure_findings(c)
     findings += rubric_alternative_findings(chosen)
     findings += stale_rubric_claims(chosen, model_definitions(model))
     findings += unknown_name_findings(chosen, model_text(model))

package/skills/eval-diagnose/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-diagnose
 description: 'Diagnose why a scored answer failed and who owns the fix, then cluster the failures by shared root cause. Walk dataset, agent-call, get_context/model, get_context/retrieval, construction, then model-definition. Append issue events to the file ledger, linked by traceId, one per cluster. Use after eval-answer, when triaging a run, or before changing a model. Does not edit the model (eval-improve).'
-version: 0.1.21
+version: 0.1.23
 ---
 
 # Diagnose One Answer
@@ -221,6 +221,21 @@ If the agent could not reasonably have known to ask, that is a model gap.
 | `GUIDANCE-NOT-RETRIEVED` | entities came back, governing guidance did not | model: put guidance on the entities agents search for |
 | `GUIDANCE-DECLINED` | guidance was retrieved and judged inapplicable | model: state the business default, not a caveat |
 
+**Before diagnosing a `NOT-RETURNED`, check the call actually returned
+nothing.** The `tool_call` event carries `retrieval_mode` and `rankedSummary`.
+A `rankedSummary` of `null` means the response could not be read, not that it
+was empty -- a body too large for the model's context is written to a file, and
+the run records no summary for it. There is nothing to diagnose there: say the
+call was not measured.
+
+And **never attribute a miss to the embedding index unless `retrieval_mode`
+says `lexical`.** The mode is recorded per call precisely so this is checkable.
+A diagnosis once explained a phantom empty result with "the index was not ready
+yet" on a run whose index was ready before the first question and whose
+response did hold results; the empty list was a parsing bug in the harness. An
+unverifiable cause that sounds right is worse than `needs_human`, because it
+closes the finding.
+
 A missing join is coverage, not an agent-call miss. The model has to volunteer
 relationships. A declared join is not a retrieval entity; do not look for it in
 `get_context` results.

package/skills/eval-import/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-import
 description: 'Turn a list of questions into an eval set, whatever shape it arrived in: a JSONL a customer sent, a CSV, a spreadsheet export, a markdown doc, an email thread, or a pull from production logs. Classify each item by what came WITH the question (a query, a number, prose criteria, or nothing), write cases.jsonl and set.json per reference/case-format.md, keep the file as it arrived, and seal each question so a later edit is detectable. Never marks a value-bearing golden verified and never invents a question. Use when questions arrive from outside and need to become a set, or before the first run against a set nobody here authored.'
-version: 0.1.21
+version: 0.1.23
 ---
 
 # Import questions into an eval set

package/skills/eval-improve/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-improve
 description: 'Make the smallest safe Malloy model edit that closes a diagnosed model-owned gap, with a probe receipt for every factual claim. Use after eval-diagnose, or when asked to fix a model so an agent can discover the right answer. Never accepts its own edit; the acceptance check belongs to eval-loop. Does not decide whether an answer was wrong (eval-answer) or why (eval-diagnose).'
-version: 0.1.21
+version: 0.1.23
 ---
 
 # Improve the Model

package/skills/eval-judge/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-judge
 description: 'Decide whether ONE answer matches its golden, and say whether you believe the golden. Read this before emitting any verdict. Covers containment, column pairing, near_match, refusals, and the gold_status judgement. Use when scoring an attempt in an evaluation run; never to conduct a run (eval-loop), diagnose a failure (eval-diagnose) or edit a model (eval-improve).'
-version: 0.1.21
+version: 0.1.23
 ---
 
 # The judge
@@ -119,6 +119,27 @@ Output, exactly this shape:
    reading that produced it, that is `no_match`. Use `near_match` only when you
    can name the rubric clause that makes the difference defensible.
 
+   **A rubric clause cannot make a wrong VALUE defensible, and a clause that
+   tries is a defect in the rubric rather than a licence to you.** `near_match`
+   turns on the answer being right under a reading the QUESTION allows -- a tie
+   broken the other way, a grain the question left open, a basis the question
+   never fixed. It does not turn on the answer being transparent about how it
+   got a figure the question did not ask for. Those two look alike in a rubric
+   and are opposites in a report: one is a number a reader can act on, the
+   other is a number a reader would act on wrongly. Naming the method makes a
+   wrong figure DIAGNOSABLE, which is worth having, and it is not partial
+   credit.
+
+   The test, before you write `near_match` on a case with a value: would a
+   reader who acted on this figure be wrong? If yes, it is `no_match` however
+   plainly the answer explained itself, and however the rubric is worded. Say
+   in `why` that you are overriding a rubric clause, so the clause gets fixed.
+   This rule exists because a set shipped one: a question asked for sales over
+   the company's season, the answer gave the meteorological window 25% lower
+   and said which window it used, and a clause granting `near_match` for a
+   stated window kept a materially wrong answer out of the pass rate
+   entirely -- the arm reported 100%.
+
    It is a third outcome because as a pass it was a large share of the measured
    noise: the same unchanged answer reads `match` in one run and `near_match`
    in the next, and the pass rate moves although nothing did. A verdict whose

package/skills/eval-loop/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: eval-loop
 description: 'Conduct a local Publisher evaluation loop in five steps: scrape/run, eval, diagnose, improve, checkpoint. You are the conductor: import cases into the file ledger, spawn a blind answerer, then run eval-answer, eval-diagnose, and eval-improve. Persistence is plain files under the model package''s evals/ directory; checkpoints are git commits of the model repo. Use to score a model, diagnose failures, improve behind an acceptance check, or roll back a bad direction.'
-version: 0.1.21
+version: 0.1.23
 ---
 
 # The Evaluation Loop
@@ -328,11 +328,19 @@ the run measure something other than what it names:
    first time somebody tidies up; the set, the run directory and the write-up
    belong in git beside the model.
 
-2. The server must be up with retrieval tracing on, so a call's ranked results
-   can be recovered afterwards (open-source Publisher: `PUBLISHER_MCP_TRACE=retrieval`).
-   Confirm a trace lookup is available (absent means tracing is off).
-   Refuse to start a scored run without it: failures without traces cannot be
-   attributed.
+2. The server must be up. Where your host offers retrieval tracing, turn it on
+   and confirm a trace lookup answers, so a call's ranked results can be
+   recovered afterwards.
+
+   **Open-source Publisher has none.** `PUBLISHER_MCP_TRACE=retrieval` is set by
+   `serve.py` and read by nothing: the trace store was written and never merged,
+   so there is no trace tool and `traceId` is null on every local attempt. This
+   does not block a scored run, because attribution never depended on it:
+   `rankedSummary` is copied onto the `tool_call` event at capture, precisely so
+   the evidence survives without a store. So do not refuse a local run for want
+   of tracing -- an earlier version of this rule did, and it refused every local
+   run there has ever been. Refuse one whose `tool_call` events carry no
+   `rankedSummary`, which is what attribution actually reads.
 
 3. Health-check: your host's status check until it reports serving, and inspect
    `loadErrors`. A dead database that still answers HTTP is an environment
@@ -500,6 +508,13 @@ conductor. Do not:
 
 - The model is the only thing improve edits. No question text, qids, or
   expected values in any name, doc, or comment.
+- **You are measuring a model, not reviewing this harness.** Read a script when
+  a number you have to report cannot be explained otherwise, and stop there. Do
+  not audit the scripts, propose fixes to them, or hand back tooling critique
+  in place of a result -- a run that ends in harness feedback has not answered
+  the question it was asked. A defect that changed a number gets one sentence
+  in the report; a defect that changed nothing gets one line in chat, once.
+  Fixing it is a different task, and the user starts it.
 - When the environment misbehaves, stop. Never diagnose a sick system. The
   harness is part of the environment: a known-broken measurement does not
   become quotable by being finished. Measured against this directive, an arm

package/skills/eval-loop/scripts/build_run_package.py

@@ -396,7 +396,8 @@ def build(run_dirs: list[pathlib.Path], set_dir: pathlib.Path,
             retr.append({"attempt_key": ak, "run_id": run_id, **r})
 
             exp = case.get("expectedEntities") or {}
-            required = {e for g in groups(exp) for e in g}
+            required_groups = groups(exp)
+            required = {e for g in required_groups for e in g}
             acceptable = set(exp.get("acceptable") or []) | required
             got: dict[str, None] = {}
             mine_tokens: list[str] = []
@@ -412,11 +413,23 @@ def build(run_dirs: list[pathlib.Path], set_dir: pathlib.Path,
             # named in returned source docs (from the ledger when the run
             # recorded them, else from the transcript walked above).
             tokens = {t for t in mine_tokens} | set(_IDENT.findall(docs))
-            for eid in sorted(required):
+            # ONE ROW PER GROUP, not per entity. `groups()` is the unit recall
+            # is scored in -- a `requiredAnyOf` group is satisfied by any one
+            # member -- and flattening it here made the app's badge disagree
+            # with the recall metric printed beside it: a case with one
+            # `required` plus a three-way alternative rendered four dots and
+            # read `2/4` in red on a case whose recall was 2 of 2. Nine of the
+            # storefront set's twelve cases carry such a group.
+            RANK = {"exact": 0, "alias": 1, "in_docs": 2, "missing": 3}
+            for g in sorted(required_groups, key=lambda g: sorted(g)[0]):
+                best = min(((delivery(e, set(got), tokens), e) for e in g),
+                           key=lambda p: RANK.get(p[0], 3))
+                status, eid = best
                 required_rows.append({
                     "attempt_key": ak, "run_id": run_id, "qid": qid,
-                    "entity_id": eid,
-                    "status": delivery(eid, set(got), tokens)})
+                    "entity_id": eid if len(g) == 1 else
+                                 f"{eid} (any of {len(g)})",
+                    "status": status})
 
             roles: list[tuple[str, str]] = []
             roles += [(eid, "required") for eid in sorted(required)]