@malloy-publisher/skills 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +68 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +47 -0
- package/package.json +54 -0
- package/skills/malloy/SKILL.md +45 -0
- package/skills/malloy-analysis/SKILL.md +76 -0
- package/skills/malloy-analysis-pitfalls/SKILL.md +68 -0
- package/skills/malloy-analysis-report/SKILL.md +125 -0
- package/skills/malloy-analyze/SKILL.md +262 -0
- package/skills/malloy-charts/SKILL.md +417 -0
- package/skills/malloy-debug/SKILL.md +121 -0
- package/skills/malloy-define/SKILL.md +194 -0
- package/skills/malloy-discover/SKILL.md +182 -0
- package/skills/malloy-document/SKILL.md +165 -0
- package/skills/malloy-getting-started/SKILL.md +57 -0
- package/skills/malloy-gotchas-modeling/SKILL.md +277 -0
- package/skills/malloy-gotchas-queries/SKILL.md +132 -0
- package/skills/malloy-gotchas-rendering/SKILL.md +124 -0
- package/skills/malloy-html-data-app-embedding/SKILL.md +39 -0
- package/skills/malloy-html-data-app-runtime/SKILL.md +158 -0
- package/skills/malloy-html-data-apps/SKILL.md +165 -0
- package/skills/malloy-html-data-apps/reference/lazy-load.md +84 -0
- package/skills/malloy-html-data-apps/reference/verification-harness.md +121 -0
- package/skills/malloy-lookml-review/SKILL.md +79 -0
- package/skills/malloy-lookml-review/reference/_concepts.md +42 -0
- package/skills/malloy-lookml-review/reference/build-derived-tables.md +110 -0
- package/skills/malloy-lookml-review/reference/build-unnest.md +47 -0
- package/skills/malloy-lookml-review/reference/curate-visibility.md +66 -0
- package/skills/malloy-lookml-review/reference/discover.md +102 -0
- package/skills/malloy-lookml-review/reference/document.md +28 -0
- package/skills/malloy-lookml-review/reference/propose-fields.md +75 -0
- package/skills/malloy-lookml-review/reference/review-coverage.md +102 -0
- package/skills/malloy-materialization-tuning/SKILL.md +103 -0
- package/skills/malloy-model/SKILL.md +320 -0
- package/skills/malloy-model/reference/access-modifiers.md +21 -0
- package/skills/malloy-model/reference/analysis-to-model.md +82 -0
- package/skills/malloy-model/reference/bridge-tables.md +51 -0
- package/skills/malloy-model/reference/normalized-schemas.md +66 -0
- package/skills/malloy-model/reference/query-sources.md +57 -0
- package/skills/malloy-modeling/SKILL.md +174 -0
- package/skills/malloy-notebook-chat/SKILL.md +19 -0
- package/skills/malloy-notebooks/SKILL.md +225 -0
- package/skills/malloy-patterns/SKILL.md +117 -0
- package/skills/malloy-phrase-detection/SKILL.md +75 -0
- package/skills/malloy-publish/SKILL.md +111 -0
- package/skills/malloy-queries/SKILL.md +270 -0
- package/skills/malloy-review/SKILL.md +160 -0
- package/skills/malloy-review/reference/output-template.md +164 -0
- package/skills/malloy-review/reference/rubric-correctness.md +161 -0
- package/skills/malloy-review/reference/rubric-documentation.md +69 -0
- package/skills/malloy-review/reference/rubric-governance.md +37 -0
- package/skills/malloy-review/reference/rubric-queries.md +78 -0
- package/skills/malloy-review/reference/rubric-rendering.md +127 -0
- package/skills/malloy-review/reference/rubric-structure.md +68 -0
- package/skills/malloy-review/reference/rubric-style.md +59 -0
- package/skills/malloy-review/reference/scope-resolution.md +100 -0
- package/skills/malloy-review/reference/severity-taxonomy.md +139 -0
- package/skills/malloy-scope/SKILL.md +100 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Query-Based Sources (Derived/Pre-Aggregated)
|
|
2
|
+
|
|
3
|
+
When you need a pre-aggregated or windowed source, use a Malloy query as the source definition. **Do NOT use `conn.sql()` unless there is no Malloy equivalent.**
|
|
4
|
+
|
|
5
|
+
## Pre-aggregation (group_by + aggregate)
|
|
6
|
+
```malloy
|
|
7
|
+
source: user_facts is conn.table('orders') -> {
|
|
8
|
+
where: status != 'Cancelled'
|
|
9
|
+
group_by: user_id
|
|
10
|
+
aggregate:
|
|
11
|
+
lifetime_revenue is sum(sale_price)
|
|
12
|
+
order_count is count(order_id)
|
|
13
|
+
} extend {
|
|
14
|
+
primary_key: user_id
|
|
15
|
+
dimension: is_vip is lifetime_revenue > 500
|
|
16
|
+
}
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Window functions (calculate)
|
|
20
|
+
```malloy
|
|
21
|
+
source: order_sequence is conn.table('orders') -> {
|
|
22
|
+
group_by: order_id, user_id
|
|
23
|
+
aggregate: order_time is min(created_at)
|
|
24
|
+
calculate: sequence is row_number() {
|
|
25
|
+
partition_by: user_id
|
|
26
|
+
order_by: order_time
|
|
27
|
+
}
|
|
28
|
+
} extend {
|
|
29
|
+
primary_key: order_id
|
|
30
|
+
dimension: is_first_order is sequence = 1
|
|
31
|
+
}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Pipeline (window + filter)
|
|
35
|
+
```malloy
|
|
36
|
+
source: first_touch is conn.table('events') -> {
|
|
37
|
+
where: user_id is not null
|
|
38
|
+
group_by: user_id, channel
|
|
39
|
+
aggregate: first_at is min(created_at)
|
|
40
|
+
calculate: rank is row_number() {
|
|
41
|
+
partition_by: user_id
|
|
42
|
+
order_by: first_at
|
|
43
|
+
}
|
|
44
|
+
} -> {
|
|
45
|
+
where: rank = 1
|
|
46
|
+
select: user_id, first_channel is channel
|
|
47
|
+
} extend {
|
|
48
|
+
primary_key: user_id
|
|
49
|
+
}
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Key Rules
|
|
53
|
+
|
|
54
|
+
- **Cannot redefine** columns from query-based sources, they already exist as fields. Add only NEW derived dimensions in `extend {}`.
|
|
55
|
+
- To add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.
|
|
56
|
+
- **Use the RAW TABLE** in query-based sources, not a modeled source, when the modeled source would create a circular dependency.
|
|
57
|
+
- **Never use `conn.sql()`** when Malloy has a native pattern. `conn.sql()` is a last resort for UNNEST, PIVOT, or dialect-specific functions only. Call `search_malloy_docs` first.
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: malloy-modeling
|
|
3
|
+
description: Build semantic models with Malloy for the Malloy Publisher. Read this skill whenever the user asks about modeling data or specifically mentions Malloy.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# STOP - READ BEFORE WRITING ANY MALLOY CODE
|
|
7
|
+
|
|
8
|
+
> **AI AGENTS: You MUST review this file before writing Malloy code.** Cross-skill references below use logical `skill:` names; load the referenced skill before acting. Before writing code, also read the gotcha skills: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering`.
|
|
9
|
+
|
|
10
|
+
## Pre-Flight Checklist
|
|
11
|
+
|
|
12
|
+
1. **Discover first**: ground yourself with `malloy_getContext` before writing ANY code. It returns the package's sources, views, and fields (with their docs), so you build on what actually exists. Never guess field names.
|
|
13
|
+
2. **Search docs proactively**: call `malloy_searchDocs` BEFORE writing unfamiliar patterns (window functions, query-based sources, pipelines). Don't guess. Malloy syntax is specific and SQL intuition is often wrong.
|
|
14
|
+
3. **Use `skill:malloy-patterns`** to discover available doc topics (YoY, cohorts, rendering, window functions).
|
|
15
|
+
4. **Check diagnostics** after writing: fix the FIRST error first, errors cascade.
|
|
16
|
+
5. **Read the gotcha skills**: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering` prevent the most common mistakes.
|
|
17
|
+
|
|
18
|
+
**Quick syntax reminders:**
|
|
19
|
+
1. **Backtick reserved words:** `` `Date` ``, `` `Hour` ``, `` `Timestamp` ``, `` `Type` ``, `` `number` ``, `` `source` ``
|
|
20
|
+
2. **Use `having:` for aggregate filters**: not `where:` on measures
|
|
21
|
+
3. **Alias joined fields in `group_by`** if using them in `order_by`
|
|
22
|
+
4. **Use `count(x)` not `count(distinct x)`**: Malloy's count() is always distinct
|
|
23
|
+
5. **One tag per line**: `# label="Revenue"` and `# currency` on separate lines
|
|
24
|
+
6. **No fixed scale on measures**: use `# currency` not `# currency=usd0m`
|
|
25
|
+
7. **Cast strings for aggregates:** `avg(score::number)` not `avg(score)`
|
|
26
|
+
8. **Boolean columns:** use `= true` not `= 'true'` (no quotes!)
|
|
27
|
+
|
|
28
|
+
## Planning and `modeling-notes.md`
|
|
29
|
+
|
|
30
|
+
If the IDE has a native plan mode, use it for the high-level approach: do data exploration during planning, then present a concrete plan for user approval before writing any files. Once approved, you can write a `modeling-notes.md` during execution to record decisions (scope, sources, key choices, prior art, gaps). This file persists alongside the model. Otherwise, keep the proposal and decisions in the conversation; Publisher has no separate workspace document store to write them to.
|
|
31
|
+
|
|
32
|
+
## 8-Step Modeling Workflow
|
|
33
|
+
|
|
34
|
+
The agent orchestrates all steps. Steps marked **(user)** pause for input. Each step has a dedicated skill with full instructions; load the relevant skill when needed.
|
|
35
|
+
|
|
36
|
+
**A field is not complete until it has its definition, `#(doc)` tag, and rendering tags.** Documentation is part of defining a field, not a separate activity. Read `skill:malloy-document` for full documentation standards (doc string writing, tag ordering).
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
DISCOVER → SCOPE → SOURCES → DEFINITIONS → BUILD BASE → BUILD JOINED → REVIEW → CURATE
|
|
40
|
+
(silent) (user) (user) (user) (agent) (agent) (user) (user)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
| Step | Skill | What Happens |
|
|
44
|
+
|------|-------|-------------|
|
|
45
|
+
| 1. Discover | `skill:malloy-discover` | Read the model and data; scan sources, fields, distributions; detect prior art |
|
|
46
|
+
| 2. Propose Scope | `skill:malloy-scope` | Present findings, user selects focus |
|
|
47
|
+
| 3. Propose Sources | `skill:malloy-define` | Propose source plan, user confirms architecture |
|
|
48
|
+
| 4. Propose Definitions | `skill:malloy-define` | Propose fields per base source, user confirms logic |
|
|
49
|
+
| 5. Build Base Sources | `skill:malloy-model` | Write fully documented base source files (one per table), check diagnostics. Read `skill:malloy-document` for doc standards. |
|
|
50
|
+
| 6. Build Joined Sources | `skill:malloy-model` | Write fully documented joined source files, validate. Read `skill:malloy-document` for doc standards. |
|
|
51
|
+
| 7. Review | (none) | Present structure, assumptions, and doc coverage; user confirms |
|
|
52
|
+
| 8. Curate | `skill:malloy-model` | Propose access controls, user approves: optional, ask user |
|
|
53
|
+
|
|
54
|
+
Publishing is out of scope for open-source v1. Self-hosters move a finished model into a served package via git and the host's publish path; see `skill:malloy-publish` for the local-to-served handoff.
|
|
55
|
+
|
|
56
|
+
**Two paths to a model: both produce the same fully documented result:**
|
|
57
|
+
- **Schema-first:** "Model my data" → 8-step workflow above using the relevant skills
|
|
58
|
+
- **Analysis-first:** "Explore this data" → `skill:malloy-analyze` → formalize via `skill:malloy-model` (`reference/analysis-to-model.md`)
|
|
59
|
+
|
|
60
|
+
After analysis completes, **always recommend formalizing into a model.**
|
|
61
|
+
|
|
62
|
+
## Agent Behavior
|
|
63
|
+
|
|
64
|
+
**Research before asking.** Present proposals with evidence. Never ask open-ended questions: propose with data and let the user confirm.
|
|
65
|
+
|
|
66
|
+
**Use business language.** Say "I simplified the column name" not "reserved word replaced." Don't expose Malloy internals unless the user asks.
|
|
67
|
+
|
|
68
|
+
**Describe what you're doing, not which step you're on.** The user doesn't have the skill files open. Say "I'll propose which tables to include and how they relate" not "Steps 3 and 4." Say "Now I'll write the source files" not "Moving to Step 5." Explain the purpose of each phase in plain language before doing it.
|
|
69
|
+
|
|
70
|
+
**Present choices as A/B/C.** When asking the user to choose, use lettered options with one-line descriptions. Mark your recommendation.
|
|
71
|
+
|
|
72
|
+
**Complete all workflow steps.** Once modeling begins, complete through review. A field without documentation is not finished. If you lose track, re-read the model and your notes. Suggest notebooks at the end.
|
|
73
|
+
|
|
74
|
+
## Route by Intent
|
|
75
|
+
|
|
76
|
+
| User says... | Route to |
|
|
77
|
+
|-------------|----------|
|
|
78
|
+
| "Model my data", "create a model" | 8-step workflow (`skill:malloy-discover`) |
|
|
79
|
+
| "Model from LookML" | 8-step with prior art via `skill:malloy-lookml-review` |
|
|
80
|
+
| "Explore this data", "what's interesting?", "show me the top X" | `skill:malloy-analyze` (EDA) |
|
|
81
|
+
| "Build a dashboard", "create views" on existing model | `skill:malloy-analyze` (views), plus `skill:malloy-charts` or `skill:malloy-notebooks` as needed |
|
|
82
|
+
| "Build a model but not sure what metrics" | `skill:malloy-analyze` first, then formalize via `skill:malloy-model` |
|
|
83
|
+
|
|
84
|
+
**If the user's first message is a data question** (not "build me a model"), route to `skill:malloy-analyze`. After analysis completes, **always recommend formalizing via the analysis-to-model workflow** (`skill:malloy-model` → `reference/analysis-to-model.md`).
|
|
85
|
+
|
|
86
|
+
## Additional Support Skills
|
|
87
|
+
|
|
88
|
+
These supplemental skills may also be loaded as needed:
|
|
89
|
+
|
|
90
|
+
- **`skill:malloy`**: Index of Malloy skills and routing guide
|
|
91
|
+
- **`skill:malloy-debug`**: Fix compile errors and interpret diagnostics
|
|
92
|
+
|
|
93
|
+
## Publisher MCP Tools
|
|
94
|
+
|
|
95
|
+
Ensure the Publisher MCP tools are configured before modeling.
|
|
96
|
+
|
|
97
|
+
| Tool | Purpose |
|
|
98
|
+
|------|---------|
|
|
99
|
+
| `malloy_getContext` | Ground yourself in a package: its sources, views, and fields |
|
|
100
|
+
| `malloy_executeQuery` | Run ad-hoc queries for validation |
|
|
101
|
+
| `malloy_compile` | Compile-check a change and get diagnostics back without running a query |
|
|
102
|
+
| `malloy_reloadPackage` | Recompile a package from disk so a saved edit becomes queryable by name |
|
|
103
|
+
| `malloy_searchDocs` | Search Malloy docs (call BEFORE unfamiliar patterns) |
|
|
104
|
+
|
|
105
|
+
Never guess field names. Ground yourself with `malloy_getContext` to see the sources and fields a package defines.
|
|
106
|
+
|
|
107
|
+
### The edit-and-run loop
|
|
108
|
+
|
|
109
|
+
Publisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until you reload the package. The loop is:
|
|
110
|
+
|
|
111
|
+
1. **Validate** the change with `malloy_compile`, which reads the model fresh from disk and returns diagnostics without running anything.
|
|
112
|
+
2. **Save** it to the package's model file.
|
|
113
|
+
3. **Reload** with `malloy_reloadPackage`.
|
|
114
|
+
4. **Run** the new view with `malloy_executeQuery`.
|
|
115
|
+
|
|
116
|
+
A reload that fails to compile is safe: your files are left alone and the previously compiled model keeps serving, with the compile errors returned to you. Compile first anyway for faster feedback. Keep the source of truth outside `publisher_data/`, which is not version-controlled and is wiped by a `--init` restart. If these two tools are missing, the Publisher you are connected to predates them; fall back to validating with a throwaway `malloy_executeQuery`.
|
|
117
|
+
|
|
118
|
+
## SQL-to-Malloy Quick Reference
|
|
119
|
+
|
|
120
|
+
| SQL | Malloy |
|
|
121
|
+
|-----|--------|
|
|
122
|
+
| `COUNT(*)` | `count()` |
|
|
123
|
+
| `COUNT(DISTINCT x)` | `count(x)` |
|
|
124
|
+
| `NOW()` | `now` |
|
|
125
|
+
| `CASE WHEN...END` | `pick...when...else` |
|
|
126
|
+
| `col IN ('a','b')` | `col ? 'a' \| 'b'` |
|
|
127
|
+
| `COALESCE(a,b)` | `a ?? b` |
|
|
128
|
+
| `CAST(x AS type)` | `x::type` |
|
|
129
|
+
| `DATEDIFF(day, a, b)` | `days(a to b)` |
|
|
130
|
+
| `CONCAT(a, b)` or `a \|\| b` | `concat(a, b)` |
|
|
131
|
+
| `TIMESTAMP_DIFF(a, b, SECOND)` | `seconds(b to a)` |
|
|
132
|
+
|
|
133
|
+
## Critical Rules
|
|
134
|
+
|
|
135
|
+
1. **All keywords require colons**: `source:`, `dimension:`, `measure:`, `view:`
|
|
136
|
+
2. **Use `is` not `as`**: `dimension: name is expression`
|
|
137
|
+
3. **Arrow operator required**: `run: source -> { operations }`
|
|
138
|
+
4. **Specify join type**: `join_one:`, `join_many:`, `join_cross:`
|
|
139
|
+
5. **Safe division**: `revenue / nullif(count, 0)`
|
|
140
|
+
6. **Group definitions under one keyword**: `measure:` then indent fields beneath
|
|
141
|
+
|
|
142
|
+
## Common Anti-Patterns
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
WRONG: source flights is ... RIGHT: source: flights is ...
|
|
146
|
+
WRONG: dimension: x as y RIGHT: dimension: y is x
|
|
147
|
+
WRONG: count(*) RIGHT: count()
|
|
148
|
+
WRONG: count(distinct x) RIGHT: count(x)
|
|
149
|
+
WRONG: revenue / order_count RIGHT: revenue / nullif(order_count, 0)
|
|
150
|
+
WRONG: run: src { ... } RIGHT: run: src -> { ... }
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Reserved Words: Scan Schema First
|
|
154
|
+
|
|
155
|
+
**Malloy has many reserved words. When in doubt, backtick it.** Most likely to appear as column names:
|
|
156
|
+
|
|
157
|
+
```
|
|
158
|
+
date, time, day, month, year, quarter, week, hour, minute, second,
|
|
159
|
+
number, string, boolean, type, table, source, index, count, sum, avg, min, max,
|
|
160
|
+
true, false, null, is, on, with, all, from, by, in, to, for, select, order_by,
|
|
161
|
+
top, bottom, desc, asc, row, range, current, window, rank
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
- `number`: only the bare word needs backticking; `account_number` is fine
|
|
165
|
+
- `source`: reserved; use a different alias like `traffic_source`
|
|
166
|
+
- `string`, `boolean`, `true`, `false`: backtick any column with these exact names
|
|
167
|
+
|
|
168
|
+
## Gotcha Skills: Read Before Writing Code
|
|
169
|
+
|
|
170
|
+
The following skills contain detailed WRONG/RIGHT patterns that prevent the most common Malloy errors. **Read them before writing code:**
|
|
171
|
+
|
|
172
|
+
- **`skill:malloy-gotchas-modeling`**: Reserved words, NULL checks, date functions, type casts, rename pitfalls, query-based source gotchas, `conn.sql()` anti-pattern
|
|
173
|
+
- **`skill:malloy-gotchas-queries`**: Chart constraints, aggregate filters, joined field aliasing, time truncation vs extraction
|
|
174
|
+
- **`skill:malloy-gotchas-rendering`**: Tag syntax, scale rules, sparkline setup, big_value patterns
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: malloy-notebook-chat
|
|
3
|
+
description: Steps to follow when the chat is bound to a notebook or saved report. The notebook's cells are the agent's primary context, answer from it, run its queries, and only reach for get_context when the user asks about something outside it.
|
|
4
|
+
---
|
|
5
|
+
# Notebook/Report Chat Workflow
|
|
6
|
+
|
|
7
|
+
Steps to follow when the user asks a question:
|
|
8
|
+
|
|
9
|
+
> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
|
|
10
|
+
|
|
11
|
+
1. Interpret the user's question as being about the bound notebook/report unless they explicitly ask about something else. Pronouns and shorthand ("this", "it", "the notebook", "the report", "the data", "what's here", "summarize", "key insights", "findings", "anything interesting") all refer to the notebook above. Never respond with a clarifying question about what the user means when the referent is clearly this notebook.
|
|
12
|
+
2. Start with a brief, natural acknowledgment that references the specific question: one sentence, varied wording.
|
|
13
|
+
3. The notebook above IS your context. Its code cells define the queries the user cares about. For any question:
|
|
14
|
+
- For broad requests like "summarize", "what are the key insights", or "tell me about this notebook", run the notebook's queries via `execute_query` and synthesize the findings across them. Do NOT ask the user to be more specific.
|
|
15
|
+
- If the question can be answered by a query already in the notebook, run that cell's query via `execute_query` (exact code, or a minor variation like adding a filter or changing a group_by).
|
|
16
|
+
- If the question asks for an analysis that is clearly NOT in the notebook (new source, different package, different domain), then, and only then, call `get_context` to explore.
|
|
17
|
+
- Do NOT call `get_context` as a default first step. The notebook already tells you what's available.
|
|
18
|
+
4. Before writing or modifying a query, read the `malloy-queries` skill for syntax patterns. When you tweak a query (add a `where:` clause, change a `group_by`, etc.), do NOT add `#(filter)` annotations: filters live on the source's model file and are inherited by this notebook automatically. Query-level `where:` filtering inside a cell is fine; declaring new filter UI is a model change, not a chat-time change.
|
|
19
|
+
5. Summarize insights from query results. Do not echo raw rows: the user sees them rendered.
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: malloy-notebooks
|
|
3
|
+
description: Create Malloy notebooks (.malloynb) for interactive dashboards and data stories. Use when user asks to "create a notebook", "make a dashboard notebook", "write a malloynb", "data story", or needs to build reports/visualizations.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Malloy Notebooks
|
|
7
|
+
|
|
8
|
+
Files: `.malloynb`
|
|
9
|
+
|
|
10
|
+
## Scope: when to use this skill vs `skill:malloy-analysis-report`
|
|
11
|
+
|
|
12
|
+
This skill is for **curated `.malloynb` notebooks**. Interactive filter widgets render automatically from `#(filter)` annotations on the model's source. The notebook itself never declares filters.
|
|
13
|
+
|
|
14
|
+
For **ad-hoc reports** generated by an analysis agent, combining queries into a quick narrative artifact, use `skill:malloy-analysis-report` instead. Ad-hoc reports also inherit whatever `#(filter)` annotations are on the source.
|
|
15
|
+
|
|
16
|
+
## Important Limitations
|
|
17
|
+
|
|
18
|
+
**Compile errors in `.malloynb` files are NOT shown in the IDE linter.** You will only see errors when cells are executed. Be extra careful with syntax and test queries in the model first if unsure.
|
|
19
|
+
|
|
20
|
+
## Multiple Models & Cross-Model Joins
|
|
21
|
+
|
|
22
|
+
A published `.malloynb` runs each `run:` cell against the whole notebook, which compiles with **all** its `import`s in scope. So a notebook can import several `.malloy` models, and a single cell can reference, and **join**, sources from different imported files. Import each model you need in a setup cell at the top; no "re-export" wrapper model is required.
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
>>>malloy
|
|
26
|
+
import "flights.malloy"
|
|
27
|
+
import "carriers.malloy"
|
|
28
|
+
|
|
29
|
+
>>>malloy
|
|
30
|
+
// joins `flights` (from flights.malloy) to `carriers` (from carriers.malloy)
|
|
31
|
+
run: flights extend {
|
|
32
|
+
join_one: carriers on carrier = carriers.code
|
|
33
|
+
} -> {
|
|
34
|
+
group_by: carriers.nickname
|
|
35
|
+
aggregate: flight_count
|
|
36
|
+
}
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Two things still hold:
|
|
40
|
+
|
|
41
|
+
- **Imports are notebook-wide, declare them at the top, never inside a `run:` cell.** A cell's query runs as a *restricted query* against the already-compiled notebook, so its imports must already be in place. You can't `import` a model inside a query cell to scope it there; an in-query import is rejected (`file imports are not permitted in a restricted query`).
|
|
42
|
+
- **Compile errors surface only at publish/render, not in the IDE linter** (see above). Verify a multi-model notebook by publishing to a scratch environment and loading it, rather than relying only on single-model query checks.
|
|
43
|
+
|
|
44
|
+
## No Data-Specific Insights in Markdown
|
|
45
|
+
|
|
46
|
+
**Notebooks must NOT contain findings about current data values.** Data refreshes will make these stale. Use markdown for framing questions and structural narrative, not for stating results.
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
>>>markdown
|
|
50
|
+
## WRONG: will become stale when data refreshes
|
|
51
|
+
Revenue spiked 23% in March. Electronics drove 60% of the spike.
|
|
52
|
+
|
|
53
|
+
>>>markdown
|
|
54
|
+
## RIGHT: frames the question, lets the query answer it
|
|
55
|
+
How is revenue trending? Which categories are driving changes?
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Cell Syntax
|
|
59
|
+
|
|
60
|
+
Cells delimited by `>>>markdown` or `>>>malloy`:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
>>>markdown
|
|
64
|
+
# Sales Analysis
|
|
65
|
+
Overview of sales trends.
|
|
66
|
+
|
|
67
|
+
>>>malloy
|
|
68
|
+
import "sales_model.malloy"
|
|
69
|
+
|
|
70
|
+
>>>markdown
|
|
71
|
+
## Summary
|
|
72
|
+
|
|
73
|
+
>>>malloy
|
|
74
|
+
run: orders -> summary
|
|
75
|
+
|
|
76
|
+
>>>markdown
|
|
77
|
+
## Monthly Trend
|
|
78
|
+
|
|
79
|
+
>>>malloy
|
|
80
|
+
# line_chart
|
|
81
|
+
run: orders -> by_month
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
**NEVER create `>>>malloysql` cells.** Only use `>>>malloy` and `>>>markdown`.
|
|
85
|
+
|
|
86
|
+
## Interactive Filters
|
|
87
|
+
|
|
88
|
+
Interactive filters are common and important. Add them to most notebooks. Filters are declared **once, on the model's source**, using `#(filter)` annotations. The notebook itself does not declare or list filters. Publisher reads the source's filter metadata and renders the widgets above the notebook automatically, then injects `where:` clauses server-side on every cell.
|
|
89
|
+
|
|
90
|
+
Filter declarations live in the `.malloy` model file, above the source, using key=value parameters. The full reference (syntax, filter types, `required` / `implicit` flags) lives with the `#(filter)` guidance in the `malloy-model` skill's Parameterizable Filters section. Short example:
|
|
91
|
+
|
|
92
|
+
```malloy
|
|
93
|
+
#(filter) name=Manufacturer dimension=Manufacturer type=in
|
|
94
|
+
#(filter) name=OrderStatus dimension=order_status type=in
|
|
95
|
+
#(filter) name=Recall_After dimension="Report Received Date" type=greater_than
|
|
96
|
+
#(filter) name=Recall_Before dimension="Report Received Date" type=less_than
|
|
97
|
+
source: recalls is duckdb.table('data/auto_recalls.csv') extend {
|
|
98
|
+
measure:
|
|
99
|
+
recall_count is count()
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`#(filter)` lines go **above** the `source:` declaration and reference dimensions by name (use `dimension=`). `name` defaults to the dimension name and only needs to be set when two filters target the same dimension (e.g. a date range). `type` is the comparator: `equal`, `in`, `like`, `greater_than`, `less_than`.
|
|
104
|
+
|
|
105
|
+
> **Deprecated.** `#(filter)` is server-side `where:` injection and is deprecated in favor of native Malloy `given:`. New notebooks should declare runtime parameters as givens (below) rather than adding new `#(filter)` annotations; see `docs/givens.md` for the migration recipes.
|
|
106
|
+
|
|
107
|
+
## Runtime Parameters (`given:`)
|
|
108
|
+
|
|
109
|
+
A notebook's parameter surface comes from the `given:` declarations on the models it imports, not from anything declared in the `.malloynb` itself. When an imported model declares givens, Publisher's notebook UI automatically renders a Parameters panel above the notebook: declared givens become parameter inputs, the values a user sets are forwarded to Malloy's runtime, and every cell re-executes against the model with those values applied.
|
|
110
|
+
|
|
111
|
+
The widget for each parameter follows the given's declared Malloy type (`string`, `string[]`, `number`, `boolean`, `date`/`timestamp`, `filter<T>`); see `docs/givens.md` for the full type table. A `#(description="...")` annotation on the given renders as helper text under its input.
|
|
112
|
+
|
|
113
|
+
```malloy
|
|
114
|
+
##! experimental.givens
|
|
115
|
+
|
|
116
|
+
#(description="Two-letter IATA carrier code to spotlight")
|
|
117
|
+
given: carrier :: string is 'WN'
|
|
118
|
+
|
|
119
|
+
source: spotlight_carrier is duckdb.table('data/carriers.parquet') extend {
|
|
120
|
+
view: by_name is {
|
|
121
|
+
where: code = $carrier
|
|
122
|
+
select: code, name, nickname
|
|
123
|
+
limit: 1
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Declare givens at the **top of the model file**, before the source that uses them, not in the notebook. `malloy-model` § Access Control covers the syntax and the `#(authorize)` gating story built on top of givens.
|
|
129
|
+
|
|
130
|
+
## Notebook Style: Iterative Analysis (Default)
|
|
131
|
+
|
|
132
|
+
**Build a story.** Each query should reveal something that motivates the next query. Start broad, then drill into what's interesting. Frame questions in markdown, let queries answer them.
|
|
133
|
+
|
|
134
|
+
**Pattern:** Question, Query, Next Question, Drill
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
>>>markdown
|
|
138
|
+
## How is revenue trending?
|
|
139
|
+
|
|
140
|
+
>>>malloy
|
|
141
|
+
# line_chart
|
|
142
|
+
run: orders -> { group_by: order_month; aggregate: revenue }
|
|
143
|
+
|
|
144
|
+
>>>markdown
|
|
145
|
+
## What's driving the biggest changes?
|
|
146
|
+
|
|
147
|
+
>>>malloy
|
|
148
|
+
# bar_chart
|
|
149
|
+
run: orders -> { group_by: category; aggregate: revenue; order_by: revenue desc; limit: 10 }
|
|
150
|
+
|
|
151
|
+
>>>markdown
|
|
152
|
+
## How does the top category break down?
|
|
153
|
+
|
|
154
|
+
>>>malloy
|
|
155
|
+
run: orders -> { aggregate: order_count, avg_order_value; where: category = 'Electronics' }
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
**When to use other styles:**
|
|
159
|
+
- **Dashboard with filters:** User asks for "filterable dashboard". Show key KPIs and breakdowns with interactive filters.
|
|
160
|
+
- **EDA/summary:** User asks for "overview". Run views like `summary`, `by_month`, `by_category` in sequence.
|
|
161
|
+
|
|
162
|
+
## View Refinement
|
|
163
|
+
|
|
164
|
+
To add `limit`, `where`, or other options to an existing view, use `+`:
|
|
165
|
+
|
|
166
|
+
```malloy
|
|
167
|
+
run: source -> my_view + { limit: 15 }
|
|
168
|
+
run: source -> my_view + { where: status = 'active', limit: 10 }
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
## Template
|
|
172
|
+
|
|
173
|
+
```
|
|
174
|
+
>>>markdown
|
|
175
|
+
# [Title]
|
|
176
|
+
[Framing question: what are we trying to understand?]
|
|
177
|
+
|
|
178
|
+
>>>malloy
|
|
179
|
+
import "model.malloy"
|
|
180
|
+
|
|
181
|
+
>>>markdown
|
|
182
|
+
## [First question: start broad]
|
|
183
|
+
|
|
184
|
+
>>>malloy
|
|
185
|
+
run: main_source -> [broad query]
|
|
186
|
+
|
|
187
|
+
>>>markdown
|
|
188
|
+
## [Next question: motivated by what the first query reveals]
|
|
189
|
+
|
|
190
|
+
>>>malloy
|
|
191
|
+
run: main_source -> [drill into finding]
|
|
192
|
+
|
|
193
|
+
>>>markdown
|
|
194
|
+
## [Deeper question]
|
|
195
|
+
|
|
196
|
+
>>>malloy
|
|
197
|
+
run: main_source -> [further breakdown]
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## Common Mistakes
|
|
201
|
+
|
|
202
|
+
| Mistake | Fix |
|
|
203
|
+
|---------|-----|
|
|
204
|
+
| `view_name { limit: 10 }` | Use `+`: `view_name + { limit: 10 }` |
|
|
205
|
+
| `# currency` on non-money | Only use `# currency` for monetary values |
|
|
206
|
+
| `#(filter) {"type": "Star"}` (JSON-blob form, on a dimension) | Unsupported legacy syntax. `#(filter)` goes **above the source** with key=value parameters (`name=`, `dimension=`, `type=`). See the `malloy-model` skill's Parameterizable Filters section. |
|
|
207
|
+
| `##(filters) [...]` annotation in the notebook | Unsupported. The notebook does not declare or list filters. Publisher renders widgets automatically from the source's `#(filter)` annotations. |
|
|
208
|
+
| Creating MalloySQL cells | **NEVER use `>>>malloysql`**, only `>>>malloy` and `>>>markdown` |
|
|
209
|
+
| Data-specific insights in markdown | Don't write "Revenue grew 23%." Frame questions instead. Data refreshes will make findings stale. |
|
|
210
|
+
| `import` inside a `run:` cell (to scope a model to that cell) | Imports are notebook-wide, put them in a setup cell at the top. An in-query import is rejected: `file imports are not permitted in a restricted query` (see Multiple Models & Cross-Model Joins). |
|
|
211
|
+
| Cross-model join inside a single ad-hoc report cell | A `skill:malloy-analysis-report` cell renders against a single model, so a cross-model join there won't render. Build a published `.malloynb` (which runs against the whole notebook) for a cross-model join. |
|
|
212
|
+
|
|
213
|
+
## Best Practices
|
|
214
|
+
|
|
215
|
+
1. Start with markdown framing the question
|
|
216
|
+
2. Import each model the notebook needs in a setup cell at the top, cells can reference and join sources across all imports (see Multiple Models & Cross-Model Joins); verify a multi-model notebook by publishing to a scratch environment
|
|
217
|
+
3. Each query should follow from what the previous one could reveal
|
|
218
|
+
4. One query per cell
|
|
219
|
+
5. Charts render only the FIRST aggregate
|
|
220
|
+
6. Add interactive filters to most notebooks by declaring `#(filter)` on the model's source (see Interactive Filters), or prefer `given:` runtime parameters for new models (see Runtime Parameters)
|
|
221
|
+
7. Never include data-specific findings in markdown. Frame questions, let queries answer
|
|
222
|
+
|
|
223
|
+
## Done
|
|
224
|
+
|
|
225
|
+
Step complete. Output: `.malloynb` notebook file.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: malloy-patterns
|
|
3
|
+
description: Index of Malloy documentation topics. Use to discover what's available in search_malloy_docs. Covers language reference (sources, queries, views, fields, aggregates, joins, filters, expressions, functions), common patterns (YoY, cohorts, percent of total), rendering, and experimental features.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Malloy Documentation Topics
|
|
7
|
+
|
|
8
|
+
Call `search_malloy_docs` with these topics.
|
|
9
|
+
|
|
10
|
+
> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
|
|
11
|
+
|
|
12
|
+
## Language Reference
|
|
13
|
+
|
|
14
|
+
| Topic | Search for... |
|
|
15
|
+
|-------|---------------|
|
|
16
|
+
| Models | `"models"` or `"statement"` |
|
|
17
|
+
| Sources | `"sources"` or `"source definition"` |
|
|
18
|
+
| Queries | `"queries"` |
|
|
19
|
+
| Views | `"views"` |
|
|
20
|
+
| Data Types | `"data types"` |
|
|
21
|
+
| Fields | `"fields"` |
|
|
22
|
+
| Aggregates | `"aggregates"` |
|
|
23
|
+
| Expressions | `"expressions"` |
|
|
24
|
+
| Functions | `"functions"` |
|
|
25
|
+
| Filters | `"filters"` or `"where"` |
|
|
26
|
+
| Calculations (window functions) | `"calculations"` or `"window functions"` |
|
|
27
|
+
| Ordering and Limiting | `"order_by"` or `"limit"` |
|
|
28
|
+
| Joins | `"joins"` or `"join_one"` or `"join_many"` |
|
|
29
|
+
| Nested Views | `"nesting"` or `"nest"` |
|
|
30
|
+
| SQL Sources | `"sql sources"` |
|
|
31
|
+
| Imports | `"imports"` |
|
|
32
|
+
| Connections | `"connections"` |
|
|
33
|
+
| Tags | `"tags"` or `"#(doc)"` |
|
|
34
|
+
|
|
35
|
+
## Common Patterns
|
|
36
|
+
|
|
37
|
+
| Pattern | Search for... |
|
|
38
|
+
|---------|---------------|
|
|
39
|
+
| Comparing Timeframes (YoY) | `"comparing timeframes"` or `"year over year"` |
|
|
40
|
+
| Foreign Sums and Averages | `"foreign sums"` |
|
|
41
|
+
| Reading Nested Data | `"reading nested"` |
|
|
42
|
+
| Percent of Total | `"percent of total"` |
|
|
43
|
+
| Cohort Analysis | `"cohort analysis"` |
|
|
44
|
+
| Nested Subtotals | `"nested subtotals"` |
|
|
45
|
+
| Bucketing with 'Other' | `"bucketing other"` |
|
|
46
|
+
| Auto-binning Histograms | `"autobin"` or `"histogram"` |
|
|
47
|
+
| Moving Average | `"moving average"` |
|
|
48
|
+
| Transform Data | `"transform"` |
|
|
49
|
+
| Sessionize - Map/Reduce | `"sessionize"` |
|
|
50
|
+
| Dimensional Indexes | `"dimensional indexes"` |
|
|
51
|
+
|
|
52
|
+
## Rendering & Visualization
|
|
53
|
+
|
|
54
|
+
| Topic | Search for... |
|
|
55
|
+
|-------|---------------|
|
|
56
|
+
| Overview | `"rendering"` |
|
|
57
|
+
| Number/Currency Scaling | `"number formatting"` or `"currency formatting"` |
|
|
58
|
+
| Big Value / KPI Cards | `"big_value"` or `"big value"` |
|
|
59
|
+
| Bar Charts | `"bar charts"` or `"# bar_chart"` |
|
|
60
|
+
| Line Charts | `"line charts"` or `"# line_chart"` |
|
|
61
|
+
| Scatter Charts | `"scatter charts"` |
|
|
62
|
+
| Dashboards | `"dashboards"` |
|
|
63
|
+
| Transposed Tables | `"transpose"` |
|
|
64
|
+
| Pivoted Tables | `"pivots"` |
|
|
65
|
+
| Shape Maps | `"shape maps"` |
|
|
66
|
+
|
|
67
|
+
## Database Dialects
|
|
68
|
+
|
|
69
|
+
These search terms find docs on dialect-specific SQL *functions* available inside expressions. Malloy models themselves are dialect-portable - the connection supplies the dialect, so you don't write "BigQuery-specific" (or any vendor-specific) Malloy.
|
|
70
|
+
|
|
71
|
+
| Dialect | Search for... |
|
|
72
|
+
|---------|---------------|
|
|
73
|
+
| BigQuery | `"bigquery"` |
|
|
74
|
+
| DuckDB | `"duckdb"` |
|
|
75
|
+
| PostgreSQL | `"postgres"` |
|
|
76
|
+
| Snowflake | `"snowflake"` |
|
|
77
|
+
| Presto/Trino | `"presto"` or `"trino"` |
|
|
78
|
+
| MySQL | `"mysql"` |
|
|
79
|
+
|
|
80
|
+
## Experimental Features
|
|
81
|
+
|
|
82
|
+
| Feature | Search for... |
|
|
83
|
+
|---------|---------------|
|
|
84
|
+
| Join INNER/RIGHT/FULL | `"inner join"` or `"full join"` |
|
|
85
|
+
| SQL Expressions | `"sql expressions"` or `"sql_number"` |
|
|
86
|
+
| Window Partitions | `"window partitions"` |
|
|
87
|
+
| Parameters | `"parameters"` |
|
|
88
|
+
| Composite "Cube" Sources | `"composite sources"` |
|
|
89
|
+
| Access Modifiers | `"access modifiers"` or `"public"` or `"private"` |
|
|
90
|
+
|
|
91
|
+
## Multi-File & Computed Sources
|
|
92
|
+
|
|
93
|
+
| Topic | Search for... |
|
|
94
|
+
|-------|---------------|
|
|
95
|
+
| Imports | `"imports"` |
|
|
96
|
+
| Computed sources (from queries) | `"from"` or `"sql sources"` |
|
|
97
|
+
| Access Modifiers (include/public/private) | `"access modifiers"` or `"public"` or `"private"` |
|
|
98
|
+
|
|
99
|
+
## Other Topics
|
|
100
|
+
|
|
101
|
+
| Topic | Search for... |
|
|
102
|
+
|-------|---------------|
|
|
103
|
+
| Notebooks | `"notebooks"` |
|
|
104
|
+
| MalloySQL | `"malloysql"` |
|
|
105
|
+
| Python Package | `"python"` |
|
|
106
|
+
| Jupyter | `"jupyter"` |
|
|
107
|
+
| Publishing | `"publishing"` |
|
|
108
|
+
| REST API | `"rest api"` |
|
|
109
|
+
| MCP for AI Agents | `"mcp agents"` |
|
|
110
|
+
| Troubleshooting | `"troubleshooting"` |
|
|
111
|
+
|
|
112
|
+
## Example
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
Call: search_malloy_docs
|
|
116
|
+
Parameters: { question: "How do I use window functions in Malloy?" }
|
|
117
|
+
```
|