@malloydata/malloyyo 0.2.23 → 0.2.24

package/dist/index.js

@@ -99,8 +99,8 @@ function resolveTarget(dir, name) {
 }
 function resolveInstance(dir, arg) {
   if (arg && /^https?:\/\//i.test(arg)) {
-    const url4 = normalizeUrl(arg);
-    return { name: url4, url: url4 };
+    const url5 = normalizeUrl(arg);
+    return { name: url5, url: url5 };
   }
   const targets = readTargetMap(dir);
   const entries = Object.entries(targets);
@@ -2374,24 +2374,24 @@ function readAll() {
     return {};
   }
 }
-function loadCreds(url4) {
-  return readAll()[url4];
+function loadCreds(url5) {
+  return readAll()[url5];
 }
-function saveCreds(url4, creds) {
+function saveCreds(url5, creds) {
   const p = credsPath();
   mkdirSync(dirname(p), { recursive: true });
   const all = readAll();
-  all[url4] = creds;
+  all[url5] = creds;
   writeFileSync(p, JSON.stringify(all, null, 2) + "\n", { mode: 384 });
   try {
     chmodSync(p, 384);
   } catch {
   }
 }
-function clearCreds(url4) {
+function clearCreds(url5) {
   const all = readAll();
-  if (!(url4 in all)) return false;
-  delete all[url4];
+  if (!(url5 in all)) return false;
+  delete all[url5];
   writeFileSync(credsPath(), JSON.stringify(all, null, 2) + "\n", { mode: 384 });
   return true;
 }
@@ -2424,8 +2424,8 @@ async function registerClient(registrationEndpoint, redirectUri) {
   if (!res.ok) throw new Error(`client registration failed: ${res.status} ${await res.text()}`);
   return (await res.json()).client_id;
 }
-function openBrowser(url4) {
-  const [cmd, args] = process.platform === "darwin" ? ["open", [url4]] : process.platform === "win32" ? ["cmd", ["/c", "start", "", url4]] : ["xdg-open", [url4]];
+function openBrowser(url5) {
+  const [cmd, args] = process.platform === "darwin" ? ["open", [url5]] : process.platform === "win32" ? ["cmd", ["/c", "start", "", url5]] : ["xdg-open", [url5]];
   try {
     spawn(cmd, args, { stdio: "ignore", detached: true }).unref();
   } catch {
@@ -3015,8 +3015,8 @@ window.addEventListener('message',async(e)=>{
     dash.title
   );
 }
-function givensFromUrl(url4) {
-  return givensFromSearch(url4.search);
+function givensFromUrl(url5) {
+  return givensFromSearch(url5.search);
 }
 function frameDoc(dash, givenSpecs, initialGivens, tileSpecs) {
   const info = {
@@ -3061,7 +3061,7 @@ async function serveDashboard(opts) {
   let byName = new Map(dashboards.map((d) => [d.name, d]));
   const bundle = makeBundler();
   const inPageBundle = makeInPageBundler();
-  const pick = (url4) => byName.get(url4.searchParams.get("d") ?? dashboards[0].name) ?? dashboards[0];
+  const pick = (url5) => byName.get(url5.searchParams.get("d") ?? dashboards[0].name) ?? dashboards[0];
   async function resolveGivens(dash) {
     if (dash.tiles && dash.entryFile) {
       const t = await runner.dashboardTiles(dash.entryFile, dash.tiles);
@@ -3095,15 +3095,15 @@ async function serveDashboard(opts) {
   }
   const handler = async (req, res) => {
     const onFramePort = (req.socket.localPort ?? port) === framePort;
-    const url4 = new URL(req.url ?? "/", `http://localhost:${onFramePort ? framePort : port}`);
+    const url5 = new URL(req.url ?? "/", `http://localhost:${onFramePort ? framePort : port}`);
     const send = (code, type, body, extra = {}) => {
       res.writeHead(code, { "content-type": type, ...extra });
       res.end(body);
     };
     try {
       if (onFramePort) {
-        if (url4.pathname === "/frame") {
-          const dash = pick(url4);
+        if (url5.pathname === "/frame") {
+          const dash = pick(url5);
           const g = await resolveGivens(dash);
           if (!g.ok) {
             return send(
@@ -3112,22 +3112,22 @@ async function serveDashboard(opts) {
               html(`<pre style="color:crimson;padding:16px">model error: ${esc2(g.error)}</pre>`, dash.title)
             );
           }
-          return send(200, "text/html; charset=utf-8", frameDoc(dash, g.union, givensFromUrl(url4), g.tiles));
+          return send(200, "text/html; charset=utf-8", frameDoc(dash, g.union, givensFromUrl(url5), g.tiles));
         }
-        if (url4.pathname === "/bundle.js") {
-          return send(200, "application/javascript; charset=utf-8", await bundle(pick(url4)));
+        if (url5.pathname === "/bundle.js") {
+          return send(200, "application/javascript; charset=utf-8", await bundle(pick(url5)));
         }
         return send(404, "text/plain", "not found");
       }
-      if (url4.pathname === "/events") {
+      if (url5.pathname === "/events") {
         res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" });
         res.write("retry: 1000\n\n");
         sseClients.add(res);
         req.on("close", () => sseClients.delete(res));
         return;
       }
-      if (url4.pathname === "/") {
-        const dash = pick(url4);
+      if (url5.pathname === "/") {
+        const dash = pick(url5);
         if (!dash.tsxPath) {
           const g = await resolveGivens(dash);
           if (!g.ok) {
@@ -3137,14 +3137,14 @@ async function serveDashboard(opts) {
               html(`<pre style="color:crimson;padding:16px">model error: ${esc2(g.error)}</pre>`, dash.title)
             );
           }
-          return send(200, "text/html; charset=utf-8", inPageShell(dash, dashboards, g.union, givensFromUrl(url4), g.tiles));
+          return send(200, "text/html; charset=utf-8", inPageShell(dash, dashboards, g.union, givensFromUrl(url5), g.tiles));
         }
-        return send(200, "text/html; charset=utf-8", parentShell(dash, frameBase, dashboards, givensFromUrl(url4)));
+        return send(200, "text/html; charset=utf-8", parentShell(dash, frameBase, dashboards, givensFromUrl(url5)));
       }
-      if (url4.pathname === "/inpage.js") {
+      if (url5.pathname === "/inpage.js") {
         return send(200, "application/javascript; charset=utf-8", await inPageBundle());
       }
-      if (url4.pathname === "/api/run" && req.method === "POST") {
+      if (url5.pathname === "/api/run" && req.method === "POST") {
         const { d, query, malloy, givens } = JSON.parse(await readBody(req));
         const dash = byName.get(d);
         if (!dash) return send(404, "application/json", JSON.stringify({ ok: false, problems: [{ message: `no dashboard '${d}'` }] }));
@@ -3624,6 +3624,7 @@ Publish: deploy ${path7.basename(outDir)}/ to Vercel (vercel.json written: clean
 // src/init.ts
 import fs7 from "node:fs";
 import path8 from "node:path";
+import { fileURLToPath as fileURLToPath2 } from "node:url";
 var AUTHOR_MCP = {
   mcpServers: {
     // No -C: the server roots at the launch cwd (the project dir), so this file
@@ -3677,6 +3678,31 @@ function scaffoldIndex(root) {
     note: anyNames ? `wrote index.malloy re-exporting ${models.length} model file(s) \u2014 REVIEW it` : "wrote index.malloy skeleton \u2014 no exports detected, fill them in by hand"
   };
 }
+function installSkills(root) {
+  const distDir = path8.dirname(fileURLToPath2(import.meta.url));
+  const candidates = [
+    path8.join(distDir, "templates", "skills"),
+    path8.join(distDir, "..", "src", "templates", "skills")
+  ];
+  const srcSkills = candidates.find((p) => fs7.existsSync(p));
+  if (!srcSkills) return { wrote: [], skipped: [], note: "no skill templates found \u2014 skipped" };
+  const destSkills = path8.join(root, ".claude", "skills");
+  fs7.mkdirSync(destSkills, { recursive: true });
+  const wrote = [];
+  const skipped = [];
+  for (const name of fs7.readdirSync(srcSkills)) {
+    const from = path8.join(srcSkills, name);
+    if (!fs7.statSync(from).isDirectory()) continue;
+    const to = path8.join(destSkills, name);
+    if (fs7.existsSync(to)) {
+      skipped.push(name);
+      continue;
+    }
+    fs7.cpSync(from, to, { recursive: true });
+    wrote.push(name);
+  }
+  return { wrote, skipped };
+}
 async function initCmd(dir) {
   const root = path8.resolve(dir);
   if (!fs7.existsSync(root) || !fs7.statSync(root).isDirectory()) {
@@ -3693,6 +3719,17 @@ async function initCmd(dir) {
   }
   const idx = scaffoldIndex(root);
   console.log(`${idx.wrote ? "\u2713" : "\u2022"} ${idx.note}`);
+  const sk = installSkills(root);
+  if (sk.note) {
+    console.log(`\u2022 ${sk.note}`);
+  } else {
+    if (sk.wrote.length) {
+      console.log(`\u2713 installed skill(s) into .claude/skills/: ${sk.wrote.join(", ")}`);
+    }
+    if (sk.skipped.length) {
+      console.log(`\u2022 skill(s) already present \u2014 left as-is: ${sk.skipped.join(", ")}`);
+    }
+  }
   console.log("");
   console.log("Next:");
   console.log("  claude              # author mode (mcp__malloyyo_author__* tools)");
@@ -3700,24 +3737,81 @@ async function initCmd(dir) {
   console.log("  malloyyo dashboard dev   # see dashboards render in a browser");
 }
 
+// src/sql.ts
+import fs8 from "node:fs";
+import path9 from "node:path";
+import url4 from "node:url";
+import { MalloyConfig as MalloyConfig3, discoverConfig as discoverConfig3 } from "@malloydata/malloy";
+function fileReader() {
+  return {
+    readURL: async (u) => {
+      if (u.protocol !== "file:") {
+        throw new Error(`unsupported URL scheme for import: ${u.href}`);
+      }
+      return fs8.promises.readFile(u, "utf8");
+    }
+  };
+}
+async function loadConfig3(rootDir) {
+  const rootUrl = url4.pathToFileURL(rootDir.endsWith(path9.sep) ? rootDir : rootDir + path9.sep);
+  const discovered = await discoverConfig3(rootUrl, rootUrl, fileReader()).catch(() => null);
+  return discovered ?? new MalloyConfig3({ includeDefaultConnections: true }, {
+    rootDirectory: rootUrl.toString()
+  });
+}
+async function readStdin() {
+  const chunks = [];
+  for await (const chunk of process.stdin) chunks.push(chunk);
+  return Buffer.concat(chunks).toString("utf8");
+}
+async function resolveSql(opts) {
+  if (opts.execute != null) return opts.execute;
+  if (opts.file) return fs8.promises.readFile(opts.file, "utf8");
+  return readStdin();
+}
+async function sqlCmd(connection, opts) {
+  const name = connection ?? "duckdb";
+  const rootDir = path9.resolve(opts.root ?? ".");
+  const sql = (await resolveSql(opts)).trim();
+  if (!sql) {
+    throw new Error("no SQL provided \u2014 pass -e <sql>, -f <file>, or pipe it via stdin");
+  }
+  await import("@malloydata/malloy-connections");
+  const cfg = await loadConfig3(rootDir);
+  try {
+    const conn = await cfg.connections.lookupConnection(name);
+    const result = await conn.runSQL(sql);
+    const rows = result?.rows ?? [];
+    if (opts.json) {
+      process.stdout.write(JSON.stringify(rows, null, 2) + "\n");
+    } else if (rows.length === 0) {
+      console.log(`ok \u2014 statement ran on connection "${name}" (no result rows)`);
+    } else {
+      console.table(rows);
+    }
+  } finally {
+    await cfg.shutdown?.();
+  }
+}
+
 // src/launch.ts
 import { spawn as spawn2 } from "node:child_process";
-import fs8 from "node:fs";
+import fs9 from "node:fs";
 import os from "node:os";
-import path9 from "node:path";
+import path10 from "node:path";
 var SURFACE_FLAG = { author: "--develop", test: "--explore" };
 var SERVER_KEY = { author: "malloyyo_author", test: "malloyyo_test" };
 async function launchCmd(mode, opts) {
-  const root = path9.resolve(opts.root ?? process.cwd());
-  const tmpDir = fs8.mkdtempSync(path9.join(os.tmpdir(), "malloyyo-launch-"));
-  const cfgPath = path9.join(tmpDir, "mcp.json");
+  const root = path10.resolve(opts.root ?? process.cwd());
+  const tmpDir = fs9.mkdtempSync(path10.join(os.tmpdir(), "malloyyo-launch-"));
+  const cfgPath = path10.join(tmpDir, "mcp.json");
   const cfg = {
     mcpServers: {
       // Absolute -C: an ephemeral config, so pinning the root is robust.
       [SERVER_KEY[mode]]: { command: "malloyyo", args: ["mcp", SURFACE_FLAG[mode], "-C", root] }
     }
   };
-  fs8.writeFileSync(cfgPath, JSON.stringify(cfg, null, 2));
+  fs9.writeFileSync(cfgPath, JSON.stringify(cfg, null, 2));
   const label = mode === "author" ? "AUTHOR (compile/edit)" : "TEST (claude.ai web preview)";
   process.stderr.write(`\u25B6 launching Claude in ${label} mode over ${root}
 `);
@@ -3736,11 +3830,11 @@ async function launchCmd(mode, opts) {
     });
     child.on("exit", () => resolve3());
   });
-  fs8.rmSync(tmpDir, { recursive: true, force: true });
+  fs9.rmSync(tmpDir, { recursive: true, force: true });
 }
 
 // package.json
-var version = "0.2.23";
+var version = "0.2.24";
 
 // src/index.ts
 function shortSha(sha) {
@@ -3841,6 +3935,13 @@ program.command("mcp").option("-C, --root <dir>", "project root (default: curren
 program.command("init").argument("[dir]", "model repo to set up", ".").description(
   "set up a model repo: write .mcp.json so `cd <repo> && claude` opens in author mode, and scaffold index.malloy if missing"
 ).action(initCmd);
+program.command("sql").argument("[connection]", "connection name from malloy-config.json", "duckdb").option("-e, --execute <sql>", "SQL to run (else read from -f <file> or stdin)").option("-f, --file <path>", "read SQL from a file").option("-C, --root <dir>", "project root for malloy-config.json discovery (default: current directory)").option("-j, --json", "print result rows as JSON").description(
+  "run raw SQL against a configured connection using the embedded DuckDB \u2014 e.g. COPY a web CSV into docs/*.parquet, no standalone duckdb needed"
+).action(
+  async (connection, opts) => {
+    await sqlCmd(connection, opts);
+  }
+);
 program.command("author").option("-C, --root <dir>", "project root (default: current directory)").description("launch Claude wired ONLY to the author surface (compile/edit the model)").action(async (opts) => {
   await launchCmd("author", opts);
 });

package/dist/templates/skills/malloyyo-auto-update/SKILL.md

@@ -0,0 +1,117 @@
+---
+name: malloyyo-auto-update
+description: Keep a malloyyo GitHub-Pages data site current automatically with a weekly GitHub Actions job — download from the source URL, transform, export parquet, commit. Use after a site is built (see malloyyo-data-site) when the data comes from a public URL that refreshes over time and you want the published site to track it with no manual steps. Worked example: malloydata/malloyyo-imdb.
+---
+
+# Auto-update a data site weekly
+
+Add a scheduled GitHub Actions job that rebuilds the data and commits it, so the
+published Pages site stays current on its own. Use this **after** the site works
+(built with `malloyyo-data-site`). If the data never changes, skip this.
+
+The worked example is `malloydata/malloyyo-imdb` — fetch and read its
+`scripts/build_data.sh` and `.github/workflows/refresh-data.yml`; adapt, don't
+copy blindly.
+
+## Precondition: one command that rebuilds the data
+
+The whole thing rests on a single script that turns the source URL(s) into the
+committed parquet — the same script you'd run by hand. Keep it linear so it
+reads as a recipe:
+
+```bash
+# scripts/build_data.sh
+set -euo pipefail
+cd "$(dirname "$0")/.."
+bash data/get.sh                                   # 1. download source URLs -> data/
+rm -f data/build.duckdb
+malloy-cli -c malloy-build.json build transform.malloy   # 2. transform
+echo "ATTACH 'data/build.duckdb' AS b (READ_ONLY);      -- 3. export -> docs/*.parquet
+      COPY b.thing TO 'docs/thing.parquet' (FORMAT parquet)" | malloyyo sql
+```
+
+(If you used Path A — direct — the whole script is just the one
+`echo "COPY (FROM read_..('https://…')) TO 'docs/…'" | malloyyo sql` command.
+No `data/get.sh`, `malloy-build.json`, `malloy-cli`, or transform needed — and
+no standalone `duckdb` either, since `malloyyo sql` uses the embedded one.)
+
+Gitignore the working files: `data/*.gz`, `data/*.duckdb`, `MANIFESTS/`.
+
+## The workflow
+
+`.github/workflows/refresh-data.yml`:
+
+```yaml
+name: refresh-data
+on:
+  schedule:
+    - cron: "0 6 * * 0"    # weekly, Sunday 06:00 UTC
+  workflow_dispatch:        # ...and on demand from the Actions tab
+jobs:
+  refresh:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write       # so the job can commit + push
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with: { node-version: "20" }
+      # malloyyo carries its own DuckDB (`malloyyo sql`), so no duckdb CLI to install.
+      # Add `@malloydata/cli` only for a `#@ persist` transform (malloy-cli build).
+      - run: npm install -g @malloydata/malloyyo @malloydata/cli
+      - uses: actions/setup-python@v5              # only if an enrichment step needs it
+        with: { python-version: "3.12" }
+      - name: Build data
+        run: bash scripts/build_data.sh
+      - name: Commit refreshed data
+        run: |
+          git config user.name  github-actions
+          git config user.email github-actions@github.com
+          git add docs/*.parquet
+          git commit -m "refresh data $(date -u +%F)" || exit 0   # unchanged = clean no-op
+          git push
+```
+
+## Hard-won details (these came from a real first run)
+
+- **`git commit … || exit 0`** — an unchanged week means nothing new upstream.
+  That is success, not a failed job.
+- **Secondary/enrichment steps must be best-effort.** If the build has a step
+  that can fail independently (an external API lookup, an optional metadata
+  enrichment) and it isn't essential to the data, mark it
+  `continue-on-error: true`. Otherwise one flaky API call throws away an
+  otherwise-good data refresh — the commit step never runs and the fresh parquet
+  is lost. (In `malloyyo-imdb` this bit us: a poster-image lookup with a missing
+  secret failed the whole job until it was made non-blocking.)
+- **Secrets** go in repo settings (Settings → Secrets and variables → Actions,
+  or `gh secret set NAME`), referenced as `${{ secrets.NAME }}`. A step reading
+  a secret that isn't set gets an empty string — pair that with best-effort.
+- **No re-bundle needed for data changes.** The dashboards fetch parquet at
+  runtime, so committing fresh parquet updates the live site. Only install
+  `@malloydata/malloyyo` and run `malloyyo dashboard bundle` in CI if the job
+  also needs to regenerate the HTML (i.e. dashboard code changed — rare for a
+  data refresh).
+- **Test before trusting the schedule.** Push, then Actions tab → the workflow →
+  **Run workflow** (that's `workflow_dispatch`). Or `gh workflow run
+  refresh-data.yml`, then `gh run watch <id> --exit-status`.
+
+## Git growth, and the flatten
+
+Each committed refresh adds the full parquet to history (~its file size per
+run). That is intentional simplicity, not a leak. When history gets too big,
+flatten it to a single commit by hand — **not** as part of the weekly job:
+
+```bash
+git checkout --orphan flat && git add -A && git commit -m "flatten history"
+git branch -M flat main && git push -f origin main
+```
+
+Only do this on a repo you solely own (it force-pushes; other clones must
+`git reset --hard origin/main`). Run it as often or as rarely as you like.
+Document this in the repo's README so future-you remembers it's available.
+
+## Done when
+
+- The workflow file is on the default branch (so `workflow_dispatch` shows up)
+- A manual run goes green end-to-end and commits fresh `docs/*.parquet`
+- The Pages site shows the new data

package/dist/templates/skills/malloyyo-data-site/SKILL.md

@@ -0,0 +1,174 @@
+---
+name: malloyyo-data-site
+description: Turn a public data URL into a browsable, interactive dashboard site hosted on GitHub Pages, using Malloy (malloyyo). Use when someone points at data on the web (CSV/TSV/Parquet/JSON at an https URL) and wants a public web interface to explore it — scaffold the repo with `malloyyo init`, transform the data into parquet under docs/, write the Malloy model + dashboards, preview with `malloyyo dashboard dev`, build with `malloyyo dashboard bundle`, and publish on GitHub Pages. Worked examples: malloydata/malloyyo-babynames (base) and malloydata/malloyyo-imdb (adds a transform + weekly auto-update).
+---
+
+# Build a public data site from a web data pointer
+
+Goal: given one or more **public data URLs**, produce a **GitHub Pages site** of
+interactive Malloy dashboards that anyone can open in a browser. The data ends
+up as parquet in `docs/`; the dashboards are static HTML that query it
+client-side with DuckDB-WASM — so there is no server, no database to run.
+
+Two reference repos — read them, they are the ground truth for structure:
+
+- **`malloydata/malloyyo-babynames`** — the clean base pattern (one model file,
+  a few dashboards, `docs/`). Data is static.
+- **`malloydata/malloyyo-imdb`** — the same, plus a `transform.malloy` that
+  cleans raw source files into parquet, plus a **weekly auto-refresh** (that
+  part is the separate `malloyyo-auto-update` skill).
+
+Fetch either with `gh api repos/<repo>/git/trees/HEAD?recursive=1` and read the
+files you need. Match their layout rather than inventing your own.
+
+## The shape of a finished repo
+
+```
+malloy-config.json     connection(s): duckdb (local), optionally md/gs mirrors
+index.malloy           the EXPORT SURFACE — only what this file exports is live
+<model>.malloy         sources, measures, joins, givens (parameters)
+storage.malloy         sources point at the parquet — docs-local OR an https URL
+  (or gs.malloy / md.malloy)   same source names, swappable hosting (step 3)
+dashboards/*.malloy    the query behind each dashboard
+dashboards/*.jsx       the dashboard layout (grid of charts/tables)
+docs/                  PUBLISHED SITE — bundled HTML (+ the *.parquet if docs-local)
+  *.parquet            the data, when hosted docs-local (committed; served by Pages)
+  *.html .nojekyll     written by `malloyyo dashboard bundle`
+.mcp.json              written by `malloyyo init` (author-mode Claude)
+```
+
+## Prerequisites (install once)
+
+```bash
+npm install -g @malloydata/malloyyo      # the `malloyyo` command
+```
+
+That's the whole toolchain for the common case. `malloyyo` has **DuckDB built
+in** — `malloyyo sql` runs SQL (read a URL, `COPY` to parquet) through it, so you
+do **not** need a standalone `duckdb` CLI.
+
+Add `@malloydata/cli` (`malloy-cli`) **only** if you do a heavier transform with
+Malloy `#@ persist` (see Path B in `reference/data-to-parquet.md`):
+
+```bash
+npm install -g @malloydata/cli           # only for #@ persist transforms
+```
+
+## Recipe
+
+Work top to bottom. After each step, prove it before moving on.
+
+### 1. Scaffold
+
+```bash
+mkdir my-site && cd my-site && git init
+malloyyo init            # writes .mcp.json (author-mode Claude) + index.malloy stub
+```
+
+Create `malloy-config.json` with a local DuckDB connection (this is what reads
+the parquet):
+
+```json
+{ "connections": { "duckdb": { "is": "duckdb" } } }
+```
+
+### 2. Get the web data into `docs/*.parquet`
+
+This is the one dataset-specific step. Two paths — pick the simpler one that
+works. Full detail and copy-paste commands: **`reference/data-to-parquet.md`**.
+
+- **Direct** (default): `malloyyo sql` reads the URL and writes parquet in one
+  shot, through the embedded DuckDB:
+  ```bash
+  echo "COPY (FROM read_csv_auto('https://…/thing.csv')) TO 'docs/thing.parquet'" | malloyyo sql
+  ```
+  Use when the web data is already close to what you want to show.
+- **Transform** (when you need to clean / join / rank / reshape): write a
+  `transform.malloy` with `#@ persist` sources and build it with `malloy-cli`,
+  then export the tables to `docs/*.parquet`. This is the `malloyyo-imdb`
+  pattern.
+
+Verify: `malloyyo sql -e "DESCRIBE SELECT * FROM 'docs/thing.parquet'"` and a
+`SELECT count(*)`.
+
+### 3. Write the model
+
+A storage file — one source per parquet file — that the rest of the model builds
+on. **Where the parquet lives is a choice** (the two examples differ here):
+
+- **docs-local** (the `malloyyo-imdb` way) — parquet committed in `docs/`,
+  served same-origin by Pages. Self-contained; git carries the data. Address it
+  by project-relative path so the same spelling works locally and published:
+  ```malloy
+  // storage.malloy
+  source: thing_table is duckdb.table('docs/thing.parquet') extend {}
+  ```
+- **External URL** (the `malloyyo-babynames` way — it uses `import "gs.malloy"`)
+  — parquet hosted on GCS / a CDN / any https URL; `docs/` holds only the HTML,
+  so git stays small. DuckDB-WASM fetches the URL at runtime:
+  ```malloy
+  // gs.malloy
+  source: thing_table is duckdb.table('https://storage.googleapis.com/…/thing.parquet') extend {}
+  ```
+
+Keep the **source names identical** across storage files (as babynames does with
+`gs.malloy` / `md.malloy`) so switching hosting is a one-line import change in
+the model. Start docs-local (simplest); move to a URL if git growth bites.
+
+`<model>.malloy` — build real sources on top: primary keys, measures, joins,
+and **givens** (the parameters that become the dashboard's filter controls).
+`index.malloy` — re-export exactly the sources/queries/givens the dashboards
+use. **Only what `index.malloy` exports is visible** to dashboards, `dashboard
+dev`, and the hosted app.
+
+For Malloy modeling and givens specifics, lean on the author MCP rather than
+guessing: the repo's `.mcp.json` wires `mcp__malloyyo_author__*`. Call
+`mcp__malloyyo_author__compile` to check files and
+`mcp__malloyyo_author__yo_help` for topics (`develop/working-with-models`).
+
+### 4. Author dashboards and preview live
+
+Each dashboard is a `.malloy` (the query/view) + a `.jsx` (the layout) under
+`dashboards/`. Author them with the `malloyyo_author` MCP and its `yo_help`
+topics — **read these, don't guess the JSX/grid API**:
+`dashboards/authoring`, `dashboards/grid-layout`, `dashboards/vega-charts`.
+
+Preview in a browser with live reload:
+
+```bash
+malloyyo dashboard dev          # serves at http://localhost:4173
+```
+
+Iterate here until the dashboards look right. `malloyyo lint` validates the
+dashboards against the model; `malloyyo test` previews what the hosted claude.ai
+app would see.
+
+### 5. Build the static site
+
+```bash
+malloyyo dashboard bundle --out docs
+# optional: add analytics + a title
+malloyyo dashboard bundle --out docs --title "My Site" --analytics G-XXXXXXXXXX
+```
+
+`bundle` writes the HTML, `.nojekyll`, and assets into `docs/`, next to the
+parquet from step 2. The pages fetch `./*.parquet` relative to the site root, so
+the data must already be in `docs/` — always run the data step before bundling.
+
+### 6. Publish on GitHub Pages
+
+Commit `docs/` and turn on Pages (Settings → Pages → Deploy from a branch →
+`main` / `docs`). Full steps + gotchas: **`reference/publish-to-github-pages.md`**.
+
+### 7. (Optional) Keep it fresh automatically
+
+If the data comes from a URL that updates over time and you want the site to
+track it, add a weekly GitHub Actions refresh — see the **`malloyyo-auto-update`**
+skill (worked example: `malloydata/malloyyo-imdb`).
+
+## Done when
+
+- `malloyyo sql -e "SELECT count(*) FROM 'docs/<file>.parquet'"` returns real rows
+- `malloyyo dashboard dev` renders every dashboard with no errors
+- `docs/` has the bundled `*.html` + `.nojekyll` alongside the parquet
+- the Pages URL loads and the dashboards populate (data fetch succeeds)

package/dist/templates/skills/malloyyo-data-site/reference/data-to-parquet.md

@@ -0,0 +1,109 @@
+# Getting web data into `docs/*.parquet`
+
+The site queries parquet files (in `docs/`, or at an https URL — see the model
+step). This is how you turn a public data URL into that parquet. Pick the
+simplest path that produces clean data.
+
+`malloyyo sql` runs SQL through malloyyo's **embedded DuckDB** — no standalone
+`duckdb` binary. It takes SQL from `-e`, `-f <file>`, or stdin, and runs against
+a connection from `malloy-config.json` (default `duckdb`).
+
+## Path A — Direct (default)
+
+One command reads the URL and writes parquet. Best when the web data is already
+close to what you want to display.
+
+```bash
+# CSV / TSV
+echo "COPY (SELECT * FROM read_csv_auto('https://example.com/data.csv'))
+      TO 'docs/data.parquet' (FORMAT parquet)" | malloyyo sql
+
+# TSV, gzipped, tab-delimited, header row (IMDb-style)
+echo "COPY (SELECT * FROM read_csv_auto('https://example.com/data.tsv.gz',
+            delim='\t', header=true, all_varchar=true))
+      TO 'docs/data.parquet' (FORMAT parquet)" | malloyyo sql
+
+# JSON / NDJSON
+echo "COPY (SELECT * FROM read_json_auto('https://example.com/data.json'))
+      TO 'docs/data.parquet' (FORMAT parquet)" | malloyyo sql
+
+# Already parquet on the web — just fetch it
+curl -fsSL -o docs/data.parquet 'https://example.com/data.parquet'
+```
+
+For anything longer than a line or two, put the SQL in a file and run it with
+`-f`, so it's re-runnable and diffable:
+
+```bash
+malloyyo sql -f scripts/build.sql        # a file of ;-separated statements
+```
+
+Notes:
+- DuckDB autoloads `httpfs` for `https://` reads.
+- Keep only the columns/rows the dashboards need — `SELECT` the columns and add a
+  `WHERE` to drop noise. Smaller parquet = faster page loads.
+- Cast types here if the source is all-strings: `col::INT`, `col::DOUBLE`, etc.
+
+## Path B — Transform with Malloy (when you need to reshape)
+
+Use when the data needs cleaning, joining across files, ranking, or nesting —
+the `malloydata/malloyyo-imdb` case. You write the transform once in Malloy;
+`malloy-cli build` materializes it; you export the tables to parquet. This path
+also needs `@malloydata/cli` (`npm install -g @malloydata/cli`).
+
+**1. A build connection** — `malloy-build.json` (a DuckDB db just for building):
+
+```json
+{ "connections": { "build": { "is": "duckdb", "databasePath": "data/build.duckdb" } } }
+```
+
+**2. `transform.malloy`** — read the raw source(s) through the `build`
+connection and mark each output source with `#@ persist name="…"`:
+
+```malloy
+##! experimental.persistence experimental.virtual_source
+
+source: raw is build.sql("""
+  SELECT * FROM read_csv_auto('data/raw.csv.gz', delim='\t', all_varchar=true, header=true)
+""")
+
+#@ persist name="thing"
+source: thing_base is raw -> {
+  where: some_count::number > 100
+  select: id, name, value is value::number
+  calculate: rank is rank() { order_by: value::number desc }
+}
+```
+
+Download the raw files first (a `data/get.sh` that `wget`s the URLs into `data/`,
+gitignored). Gitignore `data/`, `data/*.duckdb`, and `MANIFESTS/`.
+
+**3. Build, then export the persisted tables to `docs/`** — the export still
+goes through `malloyyo sql` (no standalone duckdb needed):
+
+```bash
+rm -f data/build.duckdb
+malloy-cli -c malloy-build.json build transform.malloy   # -> tables in build.duckdb
+echo "ATTACH 'data/build.duckdb' AS b (READ_ONLY);
+      COPY b.thing TO 'docs/thing.parquet' (FORMAT parquet)" | malloyyo sql
+```
+
+The persist `name=` is the table name inside `build.duckdb`; the `COPY` names the
+served file (they can differ — e.g. `thing` → `docs/mysite_thing.parquet`).
+
+Wrap steps 1–3 in a single `scripts/build_data.sh` so it is one command, by hand
+and in CI. That's exactly what `malloyyo-auto-update` automates weekly.
+
+## Either way, verify
+
+```bash
+malloyyo sql -e "DESCRIBE SELECT * FROM 'docs/thing.parquet'"
+malloyyo sql -e "SELECT count(*) FROM 'docs/thing.parquet'"
+```
+
+Then point a storage source at it (docs-local shown; an https URL works too —
+see the model step in SKILL.md):
+
+```malloy
+source: thing_table is duckdb.table('docs/thing.parquet') extend {}
+```

package/dist/templates/skills/malloyyo-data-site/reference/publish-to-github-pages.md

@@ -0,0 +1,61 @@
+# Publishing the site on GitHub Pages
+
+The published site is just the `docs/` directory: bundled HTML plus the parquet
+it queries. GitHub Pages serves it directly — no build step on GitHub's side.
+
+## One-time setup
+
+```bash
+# from the repo root, after `malloyyo dashboard bundle --out docs`
+gh repo create <owner>/<name> --public --source=. --remote=origin   # or an existing repo
+git add -A
+git commit -m "initial data site"
+git push -u origin main
+```
+
+Turn on Pages, pointing at the `docs/` folder on the default branch:
+
+```bash
+gh api -X POST repos/<owner>/<name>/pages \
+  -f 'source[branch]=main' -f 'source[path]=/docs'
+```
+
+(Or in the UI: Settings → Pages → Source: **Deploy from a branch** → Branch
+`main`, folder `/docs`.)
+
+The site appears at `https://<owner>.github.io/<name>/` within a minute or two.
+
+## Why `docs/` and why `.nojekyll`
+
+- Serving from `/docs` on the main branch keeps the data committed **once**
+  (it's the same directory you build into), not duplicated on a separate branch.
+- `malloyyo dashboard bundle` writes a `.nojekyll` file so Pages serves the
+  bundled assets as-is instead of running them through Jekyll (which would drop
+  files beginning with `_`). Keep it committed.
+
+## Updating the site
+
+Re-run the data step and the bundle, then commit `docs/`:
+
+```bash
+bash scripts/build_data.sh        # or your Path-A duckdb command → docs/*.parquet
+malloyyo dashboard bundle --out docs
+git add docs && git commit -m "update" && git push
+```
+
+Because the dashboards fetch the parquet client-side, **committing fresh parquet
+is enough to update the live data** — you only need to re-`bundle` when you
+change dashboard code. To automate the data refresh weekly, see the
+`malloyyo-auto-update` skill.
+
+## Gotchas
+
+- **Data must be in `docs/` before you bundle** — the pages fetch `./*.parquet`
+  relative to the site root; `bundle` also reads the parquet to get schemas.
+- **Big parquet grows git history.** Each committed refresh adds the full file.
+  Fine for occasional updates; for frequent auto-refresh, see the flatten note
+  in `malloyyo-auto-update`.
+- **Git LFS does not work with Pages "deploy from a branch"** — Pages serves the
+  LFS pointer text, not the file. Commit parquet as normal git objects.
+- **Private repos**: Pages on private repos needs a paid plan. Use a public repo
+  for a public site.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloyyo",
-  "version": "0.2.23",
+  "version": "0.2.24",
   "description": "Publish Malloy models to a Malloyyo instance",
   "license": "MIT",
   "repository": {