@malloy-publisher/skills 0.1.12 → 0.1.13

package/skills/eval-answer/scripts/check_contamination.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+"""Score-time contamination check. Stdlib only.
+
+An attempt is contaminated if the host-side tool-use log touched an eval path,
+gold artifact, or the model file under test, or if the answerer reported more
+MCP calls than the host observed as tool uses.
+
+  python check_contamination.py --log tool_uses.json [options]
+
+All path matching is PLAIN SUBSTRING containment against the flattened tool
+input, never glob expansion. --gold-paths and --eval-paths EXTEND the built-in
+defaults rather than replacing them, so passing extra paths can only widen the
+check.
+
+Exit 0 always when the check itself ran. Exit 1 if the log cannot be read.
+A contaminated attempt is data for the grader, not a script failure.
+
+The log is a JSON object:
+
+  {
+    "toolUses": [
+      {"name": "Read", "input": {"path": "..."}},
+      {"name": "Shell", "input": {"command": "..."}},
+      {"name": "malloy_getContext", "input": {}}
+    ],
+    "reportedCalls": 4
+  }
+
+`reportedCalls` may also be passed as --reported-calls.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Any
+
+
+DEFAULT_GOLD_PATHS = (
+    "evals/",
+    "results/gold/",
+    "gold/",
+    "cases.jsonl",
+    "intents.jsonl",
+    "set.json",
+    "events.jsonl",
+    "judge-regressions",
+)
+DEFAULT_EVAL_PATHS = (
+    "publisher.db",
+    "mcp_traces",
+)
+HOST_FILE_TOOLS = (
+    "read",
+    "write",
+    "edit",
+    "grep",
+    "glob",
+    "shell",
+    "bash",
+)
+
+
+def _tool_name(entry: dict[str, Any]) -> str:
+    return str(entry.get("name") or entry.get("tool") or "")
+
+
+def _is_host_file_tool(name: str) -> bool:
+    n = name.lower()
+    return any(n == t or n.endswith(f"_{t}") or n.endswith(f"-{t}") for t in HOST_FILE_TOOLS)
+
+
+def _is_mcp_transport(haystack: str) -> bool:
+    """A shell command that is itself an MCP call (curl JSON-RPC to /mcp).
+
+    The modelPath argument on an MCP execute_query is required, not a file
+    read, and that stays true when the MCP call rides over Shell instead of
+    a native tool. Gold and eval-path needles still apply to these entries;
+    only the model-path needle is exempted.
+    """
+    return "tools/call" in haystack and "/mcp" in haystack
+
+
+def _flatten_input(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    if isinstance(value, dict):
+        return " ".join(_flatten_input(v) for v in value.values())
+    if isinstance(value, list):
+        return " ".join(_flatten_input(v) for v in value)
+    return str(value)
+
+
+def _haystack(entry: dict[str, Any]) -> str:
+    name = str(entry.get("name") or entry.get("tool") or "")
+    raw = entry.get("input") or entry.get("arguments") or entry.get("args") or ""
+    return f"{name} {_flatten_input(raw)}".lower()
+
+
+def _matches(haystack: str, needles: list[str]) -> list[str]:
+    hits = []
+    for needle in needles:
+        n = needle.lower().strip()
+        if not n:
+            continue
+        if n in haystack:
+            hits.append(needle)
+    return hits
+
+
+def check(
+    log: dict[str, Any],
+    *,
+    gold_paths: list[str] | None = None,
+    eval_paths: list[str] | None = None,
+    model_path: str | None = None,
+    reported_calls: int | None = None,
+) -> dict[str, Any]:
+    uses = log.get("toolUses") or log.get("tool_uses") or []
+    if not isinstance(uses, list):
+        raise ValueError("toolUses must be a list")
+
+    # Custom paths extend the defaults; they never replace them. Replacing
+    # once turned the gold check into a no-op when a caller passed glob
+    # patterns that substring-match nothing.
+    needles = list(DEFAULT_GOLD_PATHS) + list(gold_paths or ())
+    needles.extend(list(DEFAULT_EVAL_PATHS) + list(eval_paths or ()))
+    # A needle like "." or "/" matches everything and voids every verdict in a
+    # run -- which is what happened the first time a caller passed `--set .`.
+    # Substring matching needs a needle that could only mean the eval set.
+    short = [n for n in needles if len(n.strip("./")) < 3]
+    if short:
+        raise ValueError(f"contamination path(s) too short to be meaningful: "
+                         f"{short!r}; pass an absolute path")
+    model_needles = []
+    if model_path:
+        model_needles = [os.path.normpath(model_path), os.path.basename(model_path)]
+
+    reasons: list[str] = []
+    for i, entry in enumerate(uses):
+        if not isinstance(entry, dict):
+            continue
+        name = _tool_name(entry) or f"use[{i}]"
+        hay = _haystack(entry)
+        hits = _matches(hay, needles)
+        # modelPath on malloy_executeQuery is required, not a file read --
+        # including when the MCP call rides over Shell as a curl JSON-RPC.
+        if model_needles and _is_host_file_tool(name) and not _is_mcp_transport(hay):
+            hits.extend(_matches(hay, model_needles))
+        if hits:
+            reasons.append(f"{name} touched {hits[0]}")
+
+    host_uses = len(uses)
+    reported = reported_calls
+    if reported is None:
+        raw = log.get("reportedCalls", log.get("reported_calls"))
+        if raw is not None:
+            reported = int(raw)
+    if reported is not None and reported > host_uses:
+        reasons.append(
+            f"reportedCalls {reported} > host tool uses {host_uses}"
+        )
+
+    return {
+        "contaminated": bool(reasons),
+        "reasons": reasons,
+        "host_tool_uses": host_uses,
+        "reported_calls": reported,
+    }
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--log", required=True, help="host tool-use JSON")
+    ap.add_argument(
+        "--gold-paths",
+        nargs="*",
+        default=None,
+        help="extra gold substrings; extends the defaults, substring match only",
+    )
+    ap.add_argument(
+        "--eval-paths",
+        nargs="*",
+        default=None,
+        help="extra eval-ledger substrings; extends the defaults",
+    )
+    ap.add_argument("--model-path", default=None)
+    ap.add_argument("--reported-calls", type=int, default=None)
+    args = ap.parse_args()
+    try:
+        with open(args.log) as f:
+            log = json.load(f)
+        out = check(
+            log,
+            gold_paths=args.gold_paths,
+            eval_paths=args.eval_paths,
+            model_path=args.model_path,
+            reported_calls=args.reported_calls,
+        )
+    except Exception as exc:  # noqa: BLE001
+        print(f"harness error: {type(exc).__name__}: {exc}", file=sys.stderr)
+        sys.exit(1)
+    print(json.dumps(out, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/skills/eval-answer/scripts/check_contamination_test.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+import os
+import unittest
+
+from check_contamination import check
+
+
+class ContaminationTests(unittest.TestCase):
+    def test_clean_mcp_only(self):
+        log = {
+            "toolUses": [
+                {"name": "malloy_getContext", "input": {"query": "orders"}},
+                {"name": "malloy_executeQuery", "input": {"query": "run: orders -> { aggregate: n is count() }"}},
+            ],
+            "reportedCalls": 2,
+        }
+        out = check(log)
+        self.assertFalse(out["contaminated"])
+        self.assertEqual(out["reasons"], [])
+
+    def test_read_gold_csv(self):
+        log = {
+            "toolUses": [
+                {"name": "Read", "input": {"path": "results/gold/beaver_full/dw_5261.csv"}},
+            ],
+            "reportedCalls": 0,
+        }
+        out = check(log)
+        self.assertTrue(out["contaminated"])
+        self.assertTrue(any("gold" in r for r in out["reasons"]))
+
+    def test_eval_rest_path(self):
+        log = {
+            "toolUses": [
+                {"name": "Shell", "input": {"command": "curl http://127.0.0.1:4000/api/v0/evals/cases/abc"}},
+            ],
+        }
+        out = check(log)
+        self.assertTrue(out["contaminated"])
+
+    def test_model_file_read(self):
+        log = {
+            "toolUses": [
+                {"name": "Read", "input": {"path": "/tmp/pkg/orders.malloy"}},
+            ],
+        }
+        out = check(log, model_path="/tmp/pkg/orders.malloy")
+        self.assertTrue(out["contaminated"])
+
+    def test_mcp_model_path_is_not_contamination(self):
+        log = {
+            "toolUses": [
+                {
+                    "name": "malloy_executeQuery",
+                    "input": {
+                        "modelPath": "storefront.malloy",
+                        "query": "run: order_items -> { aggregate: order_count }",
+                    },
+                }
+            ],
+            "reportedCalls": 1,
+        }
+        out = check(log, model_path="/tmp/pkg/storefront.malloy")
+        self.assertFalse(out["contaminated"])
+
+    def test_mcp_over_shell_model_path_is_not_contamination(self):
+        # An answerer that reaches MCP with curl JSON-RPC carries modelPath
+        # inside a Bash command; that is transport, not a file read. Found
+        # live: every clean attempt in a run was flagged before this rule.
+        log = {
+            "toolUses": [
+                {
+                    "name": "Bash",
+                    "input": {
+                        "command": "curl -s -X POST http://localhost:4040/mcp -d '{\"method\":\"tools/call\",\"params\":{\"name\":\"malloy_executeQuery\",\"arguments\":{\"modelPath\":\"storefront.malloy\"}}}'"
+                    },
+                }
+            ],
+        }
+        out = check(log, model_path="/tmp/pkg/storefront.malloy")
+        self.assertFalse(out["contaminated"])
+
+    def test_shell_cat_of_model_file_is_contamination(self):
+        log = {
+            "toolUses": [
+                {"name": "Bash", "input": {"command": "cat /tmp/pkg/storefront.malloy"}},
+            ],
+        }
+        out = check(log, model_path="/tmp/pkg/storefront.malloy")
+        self.assertTrue(out["contaminated"])
+
+    def test_custom_gold_paths_extend_defaults(self):
+        # The historical bug: passing glob patterns REPLACED the default
+        # needles, and since matching is substring-only, the gold check
+        # became a no-op. Custom paths must extend, never replace.
+        log = {
+            "toolUses": [
+                {"name": "Read", "input": {"path": "results/gold/x/dw_1.csv"}},
+            ],
+        }
+        out = check(log, gold_paths=["evals/**", "results/gold/**"])
+        self.assertTrue(out["contaminated"])
+        self.assertTrue(any("gold" in r for r in out["reasons"]))
+
+    def test_ledger_read_is_contamination(self):
+        log = {
+            "toolUses": [
+                {"name": "Read", "input": {"path": "evals/beaver/runs/r1/events.jsonl"}},
+            ],
+        }
+        out = check(log)
+        self.assertTrue(out["contaminated"])
+
+    def test_underreported_calls(self):
+        log = {
+            "toolUses": [
+                {"name": "malloy_getContext", "input": {}},
+            ],
+            "reportedCalls": 9,
+        }
+        out = check(log)
+        self.assertTrue(out["contaminated"])
+        self.assertTrue(any("reportedCalls" in r for r in out["reasons"]))
+
+
+if __name__ == "__main__":
+    os.chdir(os.path.dirname(__file__))
+    unittest.main()

package/skills/eval-answer/scripts/init_truth_package.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""Scaffold a truth package from the model under test. Stdlib only.
+
+  python3 init_truth_package.py --package <path/to/model/package> \\
+      --out <set>/../truth --name <set>-truth [--publisher-config <path>]
+
+WHY
+
+The doctrine mandates raw-table goldens: every golden is computed in a
+semantics-free package over the same raw tables, never through the model under
+test, so a model bug cannot certify its own golden. `set.json` has had a
+`truthPackage` field for that since the schema was written -- and nothing built
+one. The VideoAmp author wrote `truth.malloy`, a `publisher.json`, a second
+server config and an MCP client by hand before authoring case one: an hour of
+plumbing for a structure that is entirely derivable from the package.
+
+WHAT IT PRODUCES
+
+  truth.malloy       one `source: t_<stem> is <connection>.table('<ref>')` per
+                     distinct table the model reads, and NOTHING else -- no
+                     measures, no joins, no filters, no renames. The author
+                     adds scope filters and raw-column dimensions by hand, as
+                     the header comment says to; the script will not guess
+                     semantics, because guessed semantics are exactly what a
+                     truth package exists to exclude.
+  publisher.json     the truth package, named, with the standard description.
+  publisher.config.json  (--publisher-config) a Publisher server config
+                     serving ONLY the truth package, for the second server the
+                     answerer has no route to. Proxy connections are copied
+                     from the model package's own config when it has one, so
+                     the truth server reaches the same warehouse.
+  README.md          what is here, what the author still has to do, and the
+                     two rules: never serve this on the answerer's Publisher;
+                     never import the model under test.
+
+It refuses to write into a directory that already has a truth.malloy, because
+the hand-added scope filters are the valuable part and a rerun must not erase
+them.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import re
+import sys
+
+TABLE_REF = re.compile(r"""(?P<conn>[A-Za-z_][\w]*)\.table\(\s*['"](?P<ref>[^'"]+)['"]\s*\)""")
+
+
+def table_refs(package: pathlib.Path) -> list[tuple[str, str]]:
+    """(connection, table ref) for every distinct table the package reads."""
+    seen: dict[tuple[str, str], None] = {}
+    for f in sorted(package.rglob("*.malloy")):
+        text = f.read_text()
+        # Strip comments so a commented-out table does not become a source.
+        text = re.sub(r"//[^\n]*", "", text)
+        for m in TABLE_REF.finditer(text):
+            seen.setdefault((m.group("conn"), m.group("ref")), None)
+    return list(seen)
+
+
+# Data-file extensions a table ref may carry. Stripped before anything else
+# looks at the ref, because the order is the whole bug: a file path's last
+# DOT-segment IS its extension, so splitting first left `parquet` as the table
+# name and a package of parquet files scaffolded as t_parquet, t_parquet_2,
+# t_parquet_3 -- names no golden author can write a query against.
+DATA_EXT = re.compile(r"\.(parquet|csv|tsv|json|jsonl|ndjson|orc|avro)$", re.I)
+
+
+def stem(ref: str) -> str:
+    """t_<table>, lowercased, with v_/fact_/dim_/tbl_ noise trimmed.
+
+    Two ref shapes reach this. A PATH (`data/users.parquet`) names a file, so
+    the table is the basename without its extension. A WAREHOUSE ref
+    (`analytics.public.orders`) has no slash, and there the last dot-segment is
+    the table. Deciding which on the presence of a slash keeps both correct.
+    """
+    raw = ref.strip("/")
+    cut = DATA_EXT.sub("", raw)
+    had_ext = cut != raw
+    if "/" in cut:
+        last = cut.rsplit("/", 1)[-1]
+        # Only guess at a further extension when the known list did not match,
+        # so `data/a.b.parquet` keeps the `.b` that is part of the name.
+        if not had_ext:
+            last = re.sub(r"\.[A-Za-z0-9_]{1,10}$", "", last)
+    else:
+        last = cut if had_ext else cut.split(".")[-1]
+    last = re.sub(r"^(v_|vw_|fact_|dim_|tbl_)+", "", last, flags=re.I)
+    return "t_" + re.sub(r"\W+", "_", last).strip("_").lower()
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--package", required=True, type=pathlib.Path,
+                    help="the model package under test (holds publisher.json)")
+    ap.add_argument("--out", required=True, type=pathlib.Path,
+                    help="where to write the truth package (a SIBLING of the set, "
+                         "never inside the served tree)")
+    ap.add_argument("--name", required=True, help="truth package name, e.g. ir-truth")
+    ap.add_argument("--publisher-config", type=pathlib.Path, default=None,
+                    help="also write a Publisher server config serving only the "
+                         "truth package, to this path")
+    ap.add_argument("--environment", default="truth",
+                    help="environment name in the generated server config")
+    a = ap.parse_args()
+
+    if not (a.package / "publisher.json").exists():
+        raise SystemExit(f"{a.package} has no publisher.json; point --package at the "
+                         f"model package root")
+    if (a.out / "truth.malloy").exists():
+        raise SystemExit(f"{a.out / 'truth.malloy'} exists. The hand-added scope filters "
+                         f"are the valuable part; delete it yourself if you mean to "
+                         f"start over")
+
+    refs = table_refs(a.package)
+    if not refs:
+        raise SystemExit(f"no <connection>.table('...') references found under {a.package}")
+    names: dict[str, int] = {}
+    lines = [
+        f"// Truth sources for the {a.name} eval set. GENERATED by init_truth_package.py",
+        f"// from {a.package.name}/ -- then edited by hand, which is the point.",
+        "//",
+        "// These mirror the RAW tables the model reads and nothing else: no",
+        "// measures, no joins, no lenses, no renames, no guards. A golden must not",
+        "// be able to inherit the bug it is meant to catch.",
+        "//",
+        "// TO DO BY HAND, per source:",
+        "//   - the package's documented scope filters (a report type, an audience",
+        "//     level, a task name) -- as raw column predicates, quoting the model's",
+        "//     own doc for where each comes from",
+        "//   - raw-column dimensions for anything nested (a VARIANT column, a JSON",
+        "//     path) that a golden query will need to group by",
+        "// Keep column names in the warehouse's own case so a golden query reads as",
+        "// a warehouse query, not as a model query.",
+        "",
+    ]
+    for conn, ref in refs:
+        n = stem(ref)
+        if n in names:
+            names[n] += 1
+            n = f"{n}_{names[n]}"
+        else:
+            names[n] = 1
+        lines.append(f"source: {n} is {conn}.table('{ref}')")
+    a.out.mkdir(parents=True, exist_ok=True)
+    (a.out / "truth.malloy").write_text("\n".join(lines) + "\n")
+
+    (a.out / "publisher.json").write_text(json.dumps({
+        "name": a.name, "version": "0.0.1",
+        "description": (f"Truth package for the {a.name} eval set: the raw tables the "
+                        f"model reads, with the package's scope filters applied by hand "
+                        f"and NO semantic modelling. Goldens are computed here so a "
+                        f"model bug cannot certify its own golden. Never served on the "
+                        f"answerer's Publisher."),
+        "explores": ["truth.malloy"],
+    }, indent=2) + "\n")
+
+    (a.out / "README.md").write_text(f"""# {a.name}
+
+Truth package for the eval set beside it. `truth.malloy` was scaffolded from
+`{a.package.name}/` -- one source per raw table the model reads -- and must now be
+finished by hand: add the package's scope filters as raw predicates and any
+raw-column dimensions the goldens will group by. Add nothing else.
+
+Two rules:
+
+1. **Never serve this on the answerer's Publisher.** It is served by a second
+   server the answerer has no route to. Serving both from one config puts gold
+   within the answerer's reach.
+2. **Never import the model under test.** A truth query that reuses a model
+   measure lets a bug in the model certify its own golden.
+
+Every golden's `canonicalQuery` runs here. `verify_goldens.py --set <set>`
+re-derives them before each run.
+""")
+
+    if a.publisher_config:
+        cfg: dict = {"frozenConfig": False,
+                     "environments": [{"name": a.environment,
+                                       "packages": [{"name": a.name,
+                                                     "location": str(a.out.resolve())}]}]}
+        model_cfg = a.package / "publisher.config.json"
+        if not model_cfg.exists():
+            model_cfg = a.package.parent / "publisher.config.json"
+        if model_cfg.exists():
+            try:
+                src = json.loads(model_cfg.read_text())
+                if src.get("connections"):
+                    cfg["connections"] = src["connections"]
+            except json.JSONDecodeError:
+                pass
+        a.publisher_config.parent.mkdir(parents=True, exist_ok=True)
+        a.publisher_config.write_text(json.dumps(cfg, indent=2) + "\n")
+
+    print(f"{a.out}/truth.malloy  ({len(refs)} raw table sources)")
+    for conn, ref in refs:
+        print(f"  {conn}.table('{ref}')")
+    print(f"{a.out}/publisher.json  {a.out}/README.md")
+    if a.publisher_config:
+        print(f"{a.publisher_config}  (serve with: eval-loop/scripts/serve.py "
+              f"--server-root <root> --port 4881 --mcp-port 4882 --allow-proxy)")
+    rel = [r for _, r in refs if "/" in r and not r.startswith("/") and "://" not in r]
+    if rel:
+        print(f"\n  ! {len(rel)} table reference(s) are relative paths (e.g. {rel[0]!r}). "
+              f"They resolve against the TRUTH package directory, so point them at "
+              f"the model's data (../{a.package.name}/data/...) or symlink data/ here.")
+    print("\nNext: add scope filters and raw dimensions by hand (see the header), then "
+          "set `truthPackage` in set.json.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/eval-answer/scripts/init_truth_package_test.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""Tests for init_truth_package. Stdlib only: python3 init_truth_package_test.py"""
+import os
+import unittest
+
+from init_truth_package import stem
+
+
+class StemTests(unittest.TestCase):
+    def test_file_path_names_the_file_not_the_extension(self):
+        # The regression. Every table in a parquet package scaffolded as
+        # t_parquet, t_parquet_2, ... because the split ran before the
+        # extension strip, and a golden author cannot write against those.
+        self.assertEqual(stem("data/users.parquet"), "t_users")
+        self.assertEqual(stem("data/order_items.parquet"), "t_order_items")
+
+    def test_a_whole_package_of_one_file_type_gets_distinct_names(self):
+        refs = ["data/users.parquet", "data/products.parquet",
+                "data/inventory_items.parquet", "data/order_items.parquet"]
+        self.assertEqual(len({stem(r) for r in refs}), 4)
+
+    def test_warehouse_ref_still_takes_the_last_dot_segment(self):
+        self.assertEqual(stem("analytics.public.orders"), "t_orders")
+        self.assertEqual(stem("orders"), "t_orders")
+
+    def test_bare_filename_without_a_directory(self):
+        self.assertEqual(stem("users.parquet"), "t_users")
+
+    def test_prefix_noise_is_trimmed(self):
+        self.assertEqual(stem("data/fact_sales.csv"), "t_sales")
+        self.assertEqual(stem("/abs/path/dim_date.parquet"), "t_date")
+        self.assertEqual(stem("warehouse.dim_user"), "t_user")
+
+    def test_dots_inside_a_filename_survive(self):
+        # `.parquet` is the extension; `.b` is part of the name.
+        self.assertEqual(stem("data/a.b.parquet"), "t_a_b")
+
+    def test_path_with_no_extension(self):
+        self.assertEqual(stem("data/users"), "t_users")
+
+    def test_every_other_extension_we_know(self):
+        for ext in ("csv", "tsv", "json", "jsonl", "ndjson", "orc", "avro"):
+            self.assertEqual(stem(f"data/events.{ext}"), "t_events", ext)
+
+
+if __name__ == "__main__":
+    os.chdir(os.path.dirname(__file__))
+    unittest.main()