@malloy-publisher/skills 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/eval-answer/SKILL.md +1 -1
- package/skills/eval-answer/scripts/score_retrieval_test.py +15 -5
- package/skills/eval-diagnose/SKILL.md +1 -1
- package/skills/eval-import/SKILL.md +1 -1
- package/skills/eval-improve/SKILL.md +1 -1
- package/skills/eval-judge/SKILL.md +1 -1
- package/skills/eval-loop/SKILL.md +1 -1
- package/skills/eval-loop/reference/running-a-run.md +1 -1
- package/skills/eval-loop/scripts/build_run_package.py +4 -2
- package/skills/eval-loop/scripts/build_run_package_test.py +5 -0
- package/skills/eval-loop/scripts/run_baseline.py +1 -1
- package/skills/eval-loop/templates/eval-run-package/README.md +7 -6
- package/skills/eval-loop/templates/eval-run-package/{eval_run.malloynb → notebooks/eval_run.malloy} +55 -28
- package/skills/eval-loop/templates/eval-run-package/public/app.js +1 -1
- package/skills/eval-loop/templates/eval-run-package/public/index.html +1 -1
- package/skills/eval-report/SKILL.md +2 -2
- package/skills/malloy/SKILL.md +2 -2
- package/skills/malloy-analysis/SKILL.md +1 -1
- package/skills/malloy-analysis-pitfalls/SKILL.md +1 -1
- package/skills/malloy-analysis-report/SKILL.md +51 -17
- package/skills/malloy-analyze/SKILL.md +13 -13
- package/skills/malloy-charts/SKILL.md +1 -1
- package/skills/malloy-dashboards/SKILL.md +21 -7
- package/skills/malloy-data-app-design/SKILL.md +1 -1
- package/skills/malloy-debug/SKILL.md +1 -1
- package/skills/malloy-define/SKILL.md +2 -2
- package/skills/malloy-discover/SKILL.md +1 -1
- package/skills/malloy-document/SKILL.md +6 -6
- package/skills/malloy-getting-started/SKILL.md +1 -1
- package/skills/malloy-gotchas-modeling/SKILL.md +1 -1
- package/skills/malloy-gotchas-queries/SKILL.md +1 -1
- package/skills/malloy-gotchas-rendering/SKILL.md +1 -1
- package/skills/malloy-html-data-app-embedding/SKILL.md +1 -1
- package/skills/malloy-html-data-app-runtime/SKILL.md +1 -1
- package/skills/malloy-html-data-apps/SKILL.md +3 -3
- package/skills/malloy-lookml-review/SKILL.md +1 -1
- package/skills/malloy-materialization/SKILL.md +1 -1
- package/skills/malloy-materialization-tuning/SKILL.md +1 -1
- package/skills/malloy-model/SKILL.md +26 -61
- package/skills/malloy-model/reference/analysis-to-model.md +1 -1
- package/skills/malloy-model-as-you-go/SKILL.md +1 -1
- package/skills/malloy-modeling/SKILL.md +1 -1
- package/skills/malloy-notebook-chat/SKILL.md +1 -1
- package/skills/malloy-notebooks/SKILL.md +124 -109
- package/skills/malloy-patterns/SKILL.md +1 -1
- package/skills/malloy-phrase-detection/SKILL.md +1 -1
- package/skills/malloy-powerbi-review/SKILL.md +1 -1
- package/skills/malloy-publish/SKILL.md +22 -5
- package/skills/malloy-queries/SKILL.md +27 -5
- package/skills/malloy-review/SKILL.md +1 -1
- package/skills/malloy-review/reference/output-template.md +2 -2
- package/skills/malloy-review/reference/rubric-correctness.md +2 -2
- package/skills/malloy-review/reference/rubric-structure.md +1 -1
- package/skills/malloy-review/reference/severity-taxonomy.md +1 -1
- package/skills/malloy-scope/SKILL.md +1 -1
- package/skills/malloy-source-unreachable/SKILL.md +3 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-answer
|
|
3
3
|
description: 'Score one analytical answer against a verified golden, and score which of the entities the golden depends on retrieval delivered to the answerer. Run the contamination checklist, re-execute the submitted query yourself, then spawn a judge subagent per skill:eval-judge. Append attempt, tool_call, score, and retrieval_score events to the file ledger (reference/ledger-schema.md). Never explain the failure (eval-diagnose) or edit the model (eval-improve). Use when asked whether an answer was correct, to score a run, or to baseline a model.'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Evaluate One Answer
|
|
@@ -601,6 +601,16 @@ class LabelsMatchTheRunPackage(unittest.TestCase):
|
|
|
601
601
|
"eval-run-package", *parts)) as fh:
|
|
602
602
|
return fh.read()
|
|
603
603
|
|
|
604
|
+
def notebook_text(self):
|
|
605
|
+
"""The notebook, at its `.malloy` path or the legacy `.malloynb` one that
|
|
606
|
+
run packages built before the switch still carry."""
|
|
607
|
+
for parts in (("notebooks", "eval_run.malloy"), ("eval_run.malloynb",)):
|
|
608
|
+
try:
|
|
609
|
+
return self.package_file(*parts)
|
|
610
|
+
except FileNotFoundError:
|
|
611
|
+
continue
|
|
612
|
+
raise FileNotFoundError("eval-run-package has no notebook")
|
|
613
|
+
|
|
604
614
|
def emitted(self):
|
|
605
615
|
from score_retrieval import (DELIVERED, MODEL, NEVER_ASKED,
|
|
606
616
|
NOT_RETURNED, REFUSAL, UNMEASURED)
|
|
@@ -611,7 +621,7 @@ class LabelsMatchTheRunPackage(unittest.TestCase):
|
|
|
611
621
|
"""The legend is what a reader consults to interpret the column, so a
|
|
612
622
|
label missing from it is worse than no legend. It named `documentation`
|
|
613
623
|
-- a value that never appears -- and omitted `never asked` entirely."""
|
|
614
|
-
text = self.
|
|
624
|
+
text = self.notebook_text()
|
|
615
625
|
for label in self.emitted():
|
|
616
626
|
self.assertIn(f"**{label}**", text, label)
|
|
617
627
|
|
|
@@ -643,11 +653,11 @@ class LabelsMatchTheRunPackage(unittest.TestCase):
|
|
|
643
653
|
|
|
644
654
|
def test_no_retired_label_survives_in_the_package(self):
|
|
645
655
|
# Each of these was a real value once; each now matches nothing.
|
|
646
|
-
for
|
|
647
|
-
|
|
648
|
-
|
|
656
|
+
for text in (self.package_file("eval_run.malloy"), self.notebook_text(),
|
|
657
|
+
self.package_file("public", "app.js"),
|
|
658
|
+
self.package_file("README.md")):
|
|
649
659
|
for retired in ("query construction", "retrieval ranking"):
|
|
650
|
-
self.assertNotIn(retired, text,
|
|
660
|
+
self.assertNotIn(retired, text, retired)
|
|
651
661
|
|
|
652
662
|
|
|
653
663
|
class Summary(unittest.TestCase):
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-diagnose
|
|
3
3
|
description: 'Diagnose why a scored answer failed and who owns the fix, then cluster the failures by shared root cause. Walk dataset, agent-call, get_context/model, get_context/retrieval, construction, then model-definition. Append issue events to the file ledger, linked by traceId, one per cluster. Use after eval-answer, when triaging a run, or before changing a model. Does not edit the model (eval-improve).'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Diagnose One Answer
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-import
|
|
3
3
|
description: 'Turn a list of questions into an eval set, whatever shape it arrived in: a JSONL a customer sent, a CSV, a spreadsheet export, a markdown doc, an email thread, or a pull from production logs. Classify each item by what came WITH the question (a query, a number, prose criteria, or nothing), write cases.jsonl and set.json per reference/case-format.md, keep the file as it arrived, and seal each question so a later edit is detectable. Never marks a value-bearing golden verified and never invents a question. Use when questions arrive from outside and need to become a set, or before the first run against a set nobody here authored.'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Import questions into an eval set
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-improve
|
|
3
3
|
description: 'Make the smallest safe Malloy model edit that closes a diagnosed model-owned gap, with a probe receipt for every factual claim. Use after eval-diagnose, or when asked to fix a model so an agent can discover the right answer. Never accepts its own edit; the acceptance check belongs to eval-loop. Does not decide whether an answer was wrong (eval-answer) or why (eval-diagnose).'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Improve the Model
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-judge
|
|
3
3
|
description: 'Decide whether ONE answer matches its golden, and say whether you believe the golden. Read this before emitting any verdict. Covers containment, column pairing, near_match, refusals, and the gold_status judgement. Use when scoring an attempt in an evaluation run; never to conduct a run (eval-loop), diagnose a failure (eval-diagnose) or edit a model (eval-improve).'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# The judge
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-loop
|
|
3
3
|
description: 'Conduct a local Publisher evaluation loop in five steps: scrape/run, eval, diagnose, improve, checkpoint. You are the conductor: import cases into the file ledger, spawn a blind answerer, then run eval-answer, eval-diagnose, and eval-improve. Persistence is plain files under the model package''s evals/ directory; checkpoints are git commits of the model repo. Use to score a model, diagnose failures, improve behind an acceptance check, or roll back a bad direction.'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# The Evaluation Loop
|
|
@@ -121,7 +121,7 @@ Three layers, in this order, and the order is what makes it readable:
|
|
|
121
121
|
4. **DEEP DIVE.** A run directory is JSONL, which is a record and not a
|
|
122
122
|
report, so this layer is the two commands that turn it into something you
|
|
123
123
|
read: `build_run_package.py` (a Malloy model over the run's CSVs,
|
|
124
|
-
`eval_run.
|
|
124
|
+
`notebooks/eval_run.malloy` for the aggregate tables, and an in-package HTML app for
|
|
125
125
|
the case matrix and its per-case drawer), then a `POST .../packages` that
|
|
126
126
|
registers it on the Publisher already running, with no restart. It prints
|
|
127
127
|
the resulting URL with the run's own paths filled in.
|
|
@@ -10,7 +10,7 @@ an in-package HTML app for the case matrix.
|
|
|
10
10
|
WHY BOTH A NOTEBOOK AND AN APP
|
|
11
11
|
|
|
12
12
|
They are not redundant, and the split is not stylistic. Publisher renders a
|
|
13
|
-
`.
|
|
13
|
+
`.malloy` notebook natively, so the aggregate tables -- pass rate, effort, most-missed
|
|
14
14
|
entities, the backlog -- are best expressed as Malloy and left alone: no
|
|
15
15
|
JavaScript to drift out of step with the model. What a notebook cannot do is
|
|
16
16
|
open a drawer, and reading an eval is mostly drilling into one case. So the
|
|
@@ -530,9 +530,11 @@ def build(run_dirs: list[pathlib.Path], set_dir: pathlib.Path,
|
|
|
530
530
|
"n_cases", "proposed_edit", "evidence", "confidence"])
|
|
531
531
|
write_csv(data / "cluster_members.csv", member_rows, ["cluster_id", "qid"])
|
|
532
532
|
|
|
533
|
-
for name in ("publisher.json", "eval_run.malloy"
|
|
533
|
+
for name in ("publisher.json", "eval_run.malloy"):
|
|
534
534
|
if (TEMPLATE / name).exists():
|
|
535
535
|
shutil.copy(TEMPLATE / name, out / name)
|
|
536
|
+
if (TEMPLATE / "notebooks").exists():
|
|
537
|
+
shutil.copytree(TEMPLATE / "notebooks", out / "notebooks", dirs_exist_ok=True)
|
|
536
538
|
if (TEMPLATE / "public").exists():
|
|
537
539
|
shutil.copytree(TEMPLATE / "public", out / "public", dirs_exist_ok=True)
|
|
538
540
|
|
|
@@ -75,6 +75,7 @@ class SourcesMatchTheCsvs(unittest.TestCase):
|
|
|
75
75
|
capture_output=True, text=True, timeout=300)
|
|
76
76
|
assert p.returncode == 0, p.stderr
|
|
77
77
|
cls.data = out / "data"
|
|
78
|
+
cls.out = out
|
|
78
79
|
|
|
79
80
|
@classmethod
|
|
80
81
|
def tearDownClass(cls):
|
|
@@ -92,6 +93,10 @@ class SourcesMatchTheCsvs(unittest.TestCase):
|
|
|
92
93
|
self.assertTrue(found, "no duckdb.table sources parsed from the model")
|
|
93
94
|
return found
|
|
94
95
|
|
|
96
|
+
def test_the_package_carries_the_notebook_at_its_malloy_path(self):
|
|
97
|
+
self.assertTrue((self.out / "notebooks" / "eval_run.malloy").exists())
|
|
98
|
+
self.assertFalse((self.out / "eval_run.malloynb").exists())
|
|
99
|
+
|
|
95
100
|
def test_every_declared_column_is_actually_written(self):
|
|
96
101
|
for name, stem, block in self.sources():
|
|
97
102
|
if not (self.data / f"{stem}.csv").exists():
|
|
@@ -1748,7 +1748,7 @@ def summary_lines(*, out: pathlib.Path, set_dir: pathlib.Path, events_n: int,
|
|
|
1748
1748
|
lines += ["", "DEEP DIVE",
|
|
1749
1749
|
" A run directory is JSONL, which is a record, not a report.",
|
|
1750
1750
|
" build_run_package.py turns it into a servable Malloy package:",
|
|
1751
|
-
" a model over the run's CSVs, eval_run.
|
|
1751
|
+
" a model over the run's CSVs, notebooks/eval_run.malloy for the",
|
|
1752
1752
|
" aggregate tables, and an in-package HTML app for the case",
|
|
1753
1753
|
" matrix and its per-case drawer. Register it and open it:",
|
|
1754
1754
|
"",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Eval-run package
|
|
2
2
|
|
|
3
3
|
A finished run as a Malloy package you can open: the semantic model in
|
|
4
|
-
`eval_run.malloy`, a notebook in `eval_run.
|
|
4
|
+
`eval_run.malloy`, a notebook in `notebooks/eval_run.malloy`, an HTML data app in
|
|
5
5
|
`public/`, and CSV under `data/` written by `../../scripts/build_run_package.py`.
|
|
6
6
|
|
|
7
7
|
```bash
|
|
@@ -32,7 +32,7 @@ product, found on data we control and understand.
|
|
|
32
32
|
|
|
33
33
|
Both, and the split is not stylistic.
|
|
34
34
|
|
|
35
|
-
**`eval_run.
|
|
35
|
+
**`notebooks/eval_run.malloy`** holds the analytical tables: pass rate, cost, effort,
|
|
36
36
|
where the failures are, retrieval, the backlog. Publisher renders it natively, so
|
|
37
37
|
these are Malloy reading the model directly with no JavaScript in between and
|
|
38
38
|
nothing to drift.
|
|
@@ -121,7 +121,7 @@ hole -- that exact bug is why the tables are cross-checked rather than trusted.
|
|
|
121
121
|
```bash
|
|
122
122
|
publisher --server_root <parent-of-package> --mcp_port 4049
|
|
123
123
|
# app http://localhost:4000/environments/evals/packages/eval-run/index.html
|
|
124
|
-
# notebook http://localhost:4000/evals/eval-run/eval_run
|
|
124
|
+
# notebook http://localhost:4000/evals/eval-run/notebooks/eval_run
|
|
125
125
|
```
|
|
126
126
|
|
|
127
127
|
The two URLs are different on purpose and neither is guessable. `public/` is
|
|
@@ -138,13 +138,14 @@ file:
|
|
|
138
138
|
|
|
139
139
|
```bash
|
|
140
140
|
curl -s .../api/v0/environments/<env>/packages/<pkg>/models # compiles the model
|
|
141
|
-
curl -s .../api/v0/environments/<env>/packages/<pkg>/notebooks # lists
|
|
141
|
+
curl -s .../api/v0/environments/<env>/packages/<pkg>/notebooks # lists the notebook
|
|
142
142
|
```
|
|
143
143
|
|
|
144
144
|
`models` returns a 424 with the full compiler error when the Malloy is broken,
|
|
145
145
|
which is the fastest way to see a compile failure. Notebooks are listed by
|
|
146
|
-
`notebooks` and are **not** in `models
|
|
147
|
-
|
|
146
|
+
`notebooks` and are **not** in the `models` list. A `.malloy` notebook still compiles
|
|
147
|
+
as a model, so `models/notebooks/eval_run.malloy` shows its compile errors; only a
|
|
148
|
+
legacy `.malloynb` returns `404 "<file> is a notebook"` there. A notebook fetched from `notebooks/<file>` reports
|
|
148
149
|
its cells' compiled schemas under `modelInfo`, but only the cells it treats as
|
|
149
150
|
anonymous queries -- do not read a low count there as cells failing to compile. To
|
|
150
151
|
verify every cell, run each named query through `execute_query`.
|
package/skills/eval-loop/templates/eval-run-package/{eval_run.malloynb → notebooks/eval_run.malloy}
RENAMED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
##" One or more scored runs over one evaluation set: the analytical tables.
|
|
2
|
+
## artifact { kind=notebook title="Eval run" }
|
|
3
|
+
import "../eval_run.malloy"
|
|
4
|
+
|
|
5
|
+
##|(markdown)
|
|
2
6
|
# Eval run
|
|
3
7
|
|
|
4
8
|
One or more scored runs over one evaluation set. Publisher renders this natively,
|
|
@@ -12,9 +16,9 @@ open a drawer.
|
|
|
12
16
|
|
|
13
17
|
Read it top to bottom: the headline, then whether the headline is trustworthy,
|
|
14
18
|
then where the failures are, then what to do about them.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
19
|
+
|##
|
|
20
|
+
|
|
21
|
+
##|(markdown)
|
|
18
22
|
## 1. The headline
|
|
19
23
|
|
|
20
24
|
Pass rate over **decided** verdicts -- `match` against `no_match`. Two things are
|
|
@@ -28,9 +32,11 @@ quietly depresses every arm:
|
|
|
28
32
|
Read `confident_count` alongside the rate: two arms are only comparable when they
|
|
29
33
|
answered the same cases. Read `near_matches` too. It is the leading indicator of
|
|
30
34
|
rubrics going vague, and it moves before the pass rate does.
|
|
31
|
-
|
|
35
|
+
|##
|
|
36
|
+
|
|
32
37
|
run: run_summary
|
|
33
|
-
|
|
38
|
+
|
|
39
|
+
##|(markdown)
|
|
34
40
|
## 2. Is the headline trustworthy?
|
|
35
41
|
|
|
36
42
|
A pass rate means nothing if the attempts behind it were contaminated, if the
|
|
@@ -39,7 +45,8 @@ harness fell over, or if the arms did not face the same set.
|
|
|
39
45
|
Any contamination at all voids the run -- an attempt that saw the gold answer is
|
|
40
46
|
not evidence about the model. Harness errors are excluded from the rate rather
|
|
41
47
|
than counted as failures: a timeout is the eval failing, not the answerer.
|
|
42
|
-
|
|
48
|
+
|##
|
|
49
|
+
|
|
43
50
|
run: attempts -> {
|
|
44
51
|
group_by: arm is runs.label
|
|
45
52
|
aggregate:
|
|
@@ -50,18 +57,21 @@ run: attempts -> {
|
|
|
50
57
|
judge_version is runs.judge_version.max()
|
|
51
58
|
set_version is runs.set_version.max()
|
|
52
59
|
}
|
|
53
|
-
|
|
60
|
+
|
|
61
|
+
##|(markdown)
|
|
54
62
|
## 3. What it cost
|
|
55
63
|
|
|
56
64
|
The claim a semantic model makes is not only that the agent gets the right
|
|
57
65
|
answer, but that it gets there quickly: fewer turns, fewer tokens, less money.
|
|
58
66
|
An arm that wins on accuracy while costing three times as much has not obviously
|
|
59
67
|
won.
|
|
60
|
-
|
|
68
|
+
|##
|
|
69
|
+
|
|
61
70
|
run: cost
|
|
62
|
-
|
|
71
|
+
|
|
63
72
|
run: effort
|
|
64
|
-
|
|
73
|
+
|
|
74
|
+
##|(markdown)
|
|
65
75
|
## 4. Where the failures are
|
|
66
76
|
|
|
67
77
|
Every failure carries exactly one **where to fix**, so these counts sum to the
|
|
@@ -82,9 +92,11 @@ tables below disagree with each other.
|
|
|
82
92
|
- **coverage not measured** -- a miss on a case with no measured coverage label.
|
|
83
93
|
Retrieval's or the model's, and nothing says which, so it is charged to
|
|
84
94
|
neither.
|
|
85
|
-
|
|
95
|
+
|##
|
|
96
|
+
|
|
86
97
|
run: failures_by_where_to_fix
|
|
87
|
-
|
|
98
|
+
|
|
99
|
+
##|(markdown)
|
|
88
100
|
### Pass rate by coverage
|
|
89
101
|
|
|
90
102
|
Coverage is the ceiling on what any agent can do. A model-shaped problem shows up
|
|
@@ -93,9 +105,11 @@ only by assembling it, and the agent could not.
|
|
|
93
105
|
|
|
94
106
|
`absent` cases are scored on the **refusal** -- passing means the agent correctly
|
|
95
107
|
declined.
|
|
96
|
-
|
|
108
|
+
|##
|
|
109
|
+
|
|
97
110
|
run: pass_by_coverage
|
|
98
|
-
|
|
111
|
+
|
|
112
|
+
##|(markdown)
|
|
99
113
|
## 5. Retrieval
|
|
100
114
|
|
|
101
115
|
**Per-question entity recall**: of the entities each golden answer depends on, how
|
|
@@ -103,16 +117,20 @@ many the agent's own `get_context` calls delivered -- as a ranked entity, under
|
|
|
103
117
|
sibling source, or named in a returned source's documentation. It uses the search
|
|
104
118
|
terms the answerer chose, so it moves when the answerer changes; use it to
|
|
105
119
|
attribute a failure *within* one arm, not to compare retrieval across arms.
|
|
106
|
-
|
|
120
|
+
|##
|
|
121
|
+
|
|
107
122
|
run: retrieval_as_used
|
|
108
|
-
|
|
123
|
+
|
|
124
|
+
##|(markdown)
|
|
109
125
|
### Entities most often needed and not returned
|
|
110
126
|
|
|
111
127
|
The ranked retrieval backlog. An entity high on this list is one fix that several
|
|
112
128
|
cases are waiting on.
|
|
113
|
-
|
|
129
|
+
|##
|
|
130
|
+
|
|
114
131
|
run: most_missed_entities
|
|
115
|
-
|
|
132
|
+
|
|
133
|
+
##|(markdown)
|
|
116
134
|
## 6. What to do next
|
|
117
135
|
|
|
118
136
|
Failures grouped by shared root cause, ordered by how many cases one edit would
|
|
@@ -122,17 +140,21 @@ clear. Eight failures are rarely eight problems.
|
|
|
122
140
|
proposal is an open question, and that is an honest state -- a generated proposal
|
|
123
141
|
that nobody stands behind is worse than a blank, because a reviewer has to
|
|
124
142
|
disprove it.
|
|
125
|
-
|
|
143
|
+
|##
|
|
144
|
+
|
|
126
145
|
run: backlog
|
|
127
|
-
|
|
146
|
+
|
|
147
|
+
##|(markdown)
|
|
128
148
|
### Coverage gaps in the set itself
|
|
129
149
|
|
|
130
150
|
Independent of any run: cases the model cannot answer outright, with the entity
|
|
131
151
|
that would close the gap. This is the model-improvement backlog even before an
|
|
132
152
|
agent has attempted anything, and it is the part a customer can act on directly.
|
|
133
|
-
|
|
153
|
+
|##
|
|
154
|
+
|
|
134
155
|
run: coverage_gaps
|
|
135
|
-
|
|
156
|
+
|
|
157
|
+
##|(markdown)
|
|
136
158
|
## 7. Refusal behaviour
|
|
137
159
|
|
|
138
160
|
Refusal is scored in both directions, and getting one right while getting the
|
|
@@ -143,9 +165,11 @@ other wrong is a common and misleading result.
|
|
|
143
165
|
- A case tagged **answerable-sounds-unanswerable** passes by *answering*. An agent
|
|
144
166
|
that has learned to refuse whenever a question sounds hard will fail these while
|
|
145
167
|
looking well-calibrated on the ones above.
|
|
146
|
-
|
|
168
|
+
|##
|
|
169
|
+
|
|
147
170
|
run: refusal_behaviour
|
|
148
|
-
|
|
171
|
+
|
|
172
|
+
##|(markdown)
|
|
149
173
|
## 8. Comparing arms
|
|
150
174
|
|
|
151
175
|
Only meaningful with two or more arms loaded.
|
|
@@ -156,12 +180,15 @@ one case. What matters is which cases actually moved, and in which direction.
|
|
|
156
180
|
|
|
157
181
|
`arms_passed` between 1 and `arms` means the arms disagree. Read the nested
|
|
158
182
|
verdicts to see who won each one.
|
|
159
|
-
|
|
183
|
+
|##
|
|
184
|
+
|
|
160
185
|
run: flips
|
|
161
|
-
|
|
186
|
+
|
|
187
|
+
##|(markdown)
|
|
162
188
|
### Cases every arm failed
|
|
163
189
|
|
|
164
190
|
Changing the answerer will not move these. They are where model and retrieval
|
|
165
191
|
work pays off, and they are the most durable finding in the whole run.
|
|
166
|
-
|
|
192
|
+
|##
|
|
193
|
+
|
|
167
194
|
run: failed_everywhere
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* query was wrong or a right query was read wrongly; the sequence can, and
|
|
12
12
|
* every earlier viewer flattened it into one blob of prose.
|
|
13
13
|
*
|
|
14
|
-
* Aggregate tables live in eval_run.
|
|
14
|
+
* Aggregate tables live in notebooks/eval_run.malloy, which Publisher renders. This
|
|
15
15
|
* file holds no scoring logic; everything comes from eval_run.malloy. That
|
|
16
16
|
* includes classifying a verdict: the `outcome` column is decided once, in
|
|
17
17
|
* flip_table.outcome, and travels through the CSV. This file used to keep its
|
|
@@ -330,7 +330,7 @@
|
|
|
330
330
|
<div class="wrap">
|
|
331
331
|
<div class="top">
|
|
332
332
|
<div class="eyebrow" id="eyebrow">eval run</div>
|
|
333
|
-
<div class="eyebrow"><a id="nblink" href="../eval_run
|
|
333
|
+
<div class="eyebrow"><a id="nblink" href="../notebooks/eval_run">analytical tables →</a></div>
|
|
334
334
|
</div>
|
|
335
335
|
<div id="heroes"><div class="empty">Loading the run…</div></div>
|
|
336
336
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: eval-report
|
|
3
3
|
description: 'Write the summary a person reads after an evaluation run: what ran, what it scored, which failures were the MODEL and which were the EVAL itself (out of turns, contaminated, unestablished golden, unreadable judge), what was learned, and what to do next. Every claim links to the artifact behind it. Use at the end of any run, before quoting a number to anyone. Never scores an answer (eval-answer) or decides who owns a failure (eval-diagnose).'
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Report a Run
|
|
@@ -66,7 +66,7 @@ restart. It gives you two things to link:
|
|
|
66
66
|
| Artifact | What it is | URL |
|
|
67
67
|
|---|---|---|
|
|
68
68
|
| The case matrix app | Every question, its verdict, which needed entities retrieval delivered, and a drawer per case holding the reference answer, the judge's reasoning, the re-executed rows and every query the answerer ran | `<publisher>/environments/<env>/packages/eval-<label>/` |
|
|
69
|
-
| `eval_run.
|
|
69
|
+
| `notebooks/eval_run.malloy` | The aggregate tables: pass rate, effort, cost, most-missed entities, the backlog | `<publisher>/<env>/eval-<label>/notebooks/eval_run` |
|
|
70
70
|
|
|
71
71
|
**Those two URLs are in different path spaces, and guessing costs a 404.** The
|
|
72
72
|
app is served by the in-package `public/` handler, which owns
|
package/skills/malloy/SKILL.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: malloy
|
|
3
3
|
description: Index of all Malloy skills. Use when user asks "malloy help", "what malloy skills are available", "how do I use malloy", or needs guidance on which Malloy skill to use.
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
<!-- Copyright (c) Credible Data Inc. SPDX-License-Identifier: MIT -->
|
|
7
7
|
|
|
@@ -44,7 +44,7 @@ This table is a catalogue of what exists, not of what is loaded. A host that ins
|
|
|
44
44
|
| `skill:malloy-model-as-you-go` | After answering a question, writing down what it assumed: a `#(doc)`'d field in the model, an `extend` in the notebook, or a stated assumption plus snippet, depending on what the session can write |
|
|
45
45
|
| `skill:malloy-analyze` | Open-ended exploration with no intent to keep anything: profiling, hypotheses, views |
|
|
46
46
|
| `skill:malloy-charts` | Chart selection and renderer reference for Malloy visualizations |
|
|
47
|
-
| `skill:malloy-notebooks` | Building Malloy notebooks (
|
|
47
|
+
| `skill:malloy-notebooks` | Building Malloy notebooks (`notebooks/<slug>.malloy`; an existing `.malloynb` still reads) |
|
|
48
48
|
| `skill:malloy-analysis-report` | Combining validated queries into a notebook report or dashboard |
|
|
49
49
|
| `skill:malloy-analysis-pitfalls` | Checking a query and its results before presenting an answer |
|
|
50
50
|
| `skill:malloy-source-unreachable` | A source is missing from discovery, or a query was refused with a 404 or 403 |
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: malloy-analysis
|
|
3
3
|
description: "Read before your first get_context call on any business or data question. The required workflow, not background: which source, which rows, how to verify. Skipping it causes confident wrong answers."
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
<!-- Copyright (c) Credible Data Inc. SPDX-License-Identifier: MIT -->
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: malloy-analysis-pitfalls
|
|
3
3
|
description: Checks to run on a query and its numbers before a result is presented - wrong grain, fan-out, invented field names, mismatched filters, count vs distinct. The common data-analysis pitfalls.
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
<!-- Copyright (c) Credible Data Inc. SPDX-License-Identifier: MIT -->
|
|
7
7
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: malloy-analysis-report
|
|
3
3
|
description: Combine validated Malloy queries into a notebook report or dashboard. Use when the user asks to "create a report", "build a dashboard", "combine these into a report", or wants a persistent multi-query artifact.
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.30
|
|
5
5
|
---
|
|
6
6
|
<!--
|
|
7
7
|
Copyright (c) Credible Data Inc.
|
|
@@ -10,7 +10,7 @@ SPDX-License-Identifier: MIT
|
|
|
10
10
|
|
|
11
11
|
# Creating Reports
|
|
12
12
|
|
|
13
|
-
An ad-hoc report is a `.
|
|
13
|
+
An ad-hoc report is a `.malloy` notebook, `notebooks/<slug>.malloy`, that combines markdown narrative with live Malloy query cells. There is no dedicated report tool: you author the notebook directly. Load `skill:malloy-notebooks` for the full format and authoring rules; this skill covers when to build one and how to design good report content (cells, chart annotations, narrative structure). Never write a new `.malloynb`.
|
|
14
14
|
|
|
15
15
|
> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
|
|
16
16
|
|
|
@@ -24,31 +24,65 @@ Do NOT build the notebook in the same turn as `execute_query`. Explain first, th
|
|
|
24
24
|
|
|
25
25
|
## Filters are inherited from the model, don't declare them in the report
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
Do not declare filters in an ad-hoc report. If the source declares `given:` parameters (or legacy `#(filter)` annotations), Publisher renders the controls, parses caller parameters, and applies them server-side automatically: the report inherits and displays them with no extra work. If the analysis needs a knob the source doesn't expose, the right move is to add a `given:` to the source itself, not to wedge a filter widget into the report. `#(filter)` is deprecated in favour of native Malloy `given:` parameters. Never add a `#(filter)` annotation: every use, including `required`, `implicit`, and date/number ranges, has a `given:` form. The `malloy-model` skill covers this under § Legacy: Parameterizable Filters. For curated notebooks with their own per-notebook filter UI on top of the model, see `skill:malloy-notebooks` instead.
|
|
28
28
|
|
|
29
29
|
## What goes in the report
|
|
30
30
|
|
|
31
|
-
Do NOT add an H1 heading in any
|
|
31
|
+
Do NOT add an H1 heading in any markdown (use H2 and below for sections); the `title` in the `## artifact` tag serves as the title. To redo the structure rather than tweak one cell, rewrite the notebook file end-to-end.
|
|
32
32
|
|
|
33
33
|
Markdown cells own narrative; query cells own a single Malloy query whose chart annotation tells the renderer how to display the result. Markdown supports H2 headings, lists, bold, and inline code. Keep narrative cells short, one idea per cell, so the rendered output reads as a story instead of a wall of text.
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
The file starts with `## artifact { kind=notebook title="..." }`, then the `import` for the model file. Definitions (`import`, `source:`, `query:`, `given:`) come before the first markdown or `run:`. Prose is `##|(markdown)` ... `|##` for a block (body on the lines between) or `##(markdown) text` for one line. Each `run:` is a query cell, and its tags sit directly above it with nothing between. A `#"` directly above the `run:` is its caption. Trailing prose is `##(markdown)`, never `#(markdown)` or `#"`.
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
>>>markdown
|
|
39
|
-
## Section heading
|
|
40
|
-
Narrative text here.
|
|
41
|
-
```
|
|
37
|
+
Each `run:` must be a standalone query (for example `run: source -> { ... }`). The `import` is file-wide: query cells never repeat it. Compile the file with `/compile` (`"scope": "file"`, at the path `notebooks/<slug>.malloy`) before saving; a `.malloy` notebook compiles as a model, so its errors come back there. A complete report:
|
|
42
38
|
|
|
43
|
-
|
|
39
|
+
```malloy
|
|
40
|
+
## artifact { kind=notebook title="Sales report" }
|
|
41
|
+
import "../order_analysis.malloy"
|
|
42
|
+
|
|
43
|
+
##|(markdown)
|
|
44
|
+
## Overview
|
|
45
|
+
What is driving sales, and which categories carry it? The queries below cover every order in the model.
|
|
46
|
+
|##
|
|
47
|
+
|
|
48
|
+
# big_value
|
|
49
|
+
run: order_analysis -> {
|
|
50
|
+
aggregate:
|
|
51
|
+
# label="Revenue"
|
|
52
|
+
# currency
|
|
53
|
+
total_revenue
|
|
54
|
+
|
|
55
|
+
# label="Orders"
|
|
56
|
+
# number=auto
|
|
57
|
+
order_count
|
|
58
|
+
}
|
|
44
59
|
|
|
45
|
-
|
|
46
|
-
|
|
60
|
+
##(markdown) ## Trend: how does revenue move over time?
|
|
61
|
+
|
|
62
|
+
#" Revenue by month
|
|
63
|
+
# line_chart
|
|
64
|
+
run: order_analysis -> {
|
|
65
|
+
group_by: order_date.month
|
|
66
|
+
aggregate: total_revenue
|
|
67
|
+
order_by: 1
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
##|(markdown)
|
|
71
|
+
## Breakdown
|
|
72
|
+
Which categories account for the most revenue?
|
|
73
|
+
|##
|
|
74
|
+
|
|
75
|
+
#" Top ten categories by revenue
|
|
47
76
|
# bar_chart
|
|
48
|
-
run:
|
|
49
|
-
|
|
77
|
+
run: order_analysis -> {
|
|
78
|
+
group_by: category
|
|
79
|
+
aggregate: total_revenue
|
|
80
|
+
order_by: total_revenue desc
|
|
81
|
+
limit: 10
|
|
82
|
+
}
|
|
50
83
|
|
|
51
|
-
|
|
84
|
+
##(markdown) ## Key takeaways: what to look at next.
|
|
85
|
+
```
|
|
52
86
|
|
|
53
87
|
A well-structured report typically follows this pattern:
|
|
54
88
|
|
|
@@ -124,7 +158,7 @@ Key rendering rules to keep in mind when shaping a cell:
|
|
|
124
158
|
|
|
125
159
|
## Editing an existing report
|
|
126
160
|
|
|
127
|
-
For small targeted changes (fix one cell, insert one new cell), edit
|
|
161
|
+
For small targeted changes (fix one cell, insert one new cell), edit the cell's statement or markdown in place rather than recreating the whole notebook. For structural rewrites (reordering many cells, changing the narrative arc), rewrite the notebook file. An existing `.malloynb` is read-only in a Credible draft (it can be deleted): to edit its story, write a `.malloy` notebook.
|
|
128
162
|
|
|
129
163
|
## IMPORTANT
|
|
130
164
|
|