@malloydata/malloyyo 0.2.20 → 0.2.22

package/dist/frame-wasm-entry.tsx

@@ -46,24 +46,30 @@ class StaticWasmConnection extends DuckDBWASMConnection {
   }
 }
 
-// Whole-file fetch, not ranged reads. For a columnar file whose dominant column
-// every query touches, ranged reads trade one bulk download for dozens of WAN
-// round trips and lose badly.
+// Whole-file fetch, not ranged reads.
 //
-// This CANNOT go through registerRemoteTableCallback: db-duckdb's findTables
-// skips anything matching ^https?:// ("handled by duckdb-wasm") so the callback
-// never fires for a remote URL. Instead we pre-register the bytes under the
-// exact URL string before any query runs — duckdb-wasm resolves its registered
-// files by name first, so the table reference finds the buffer and never issues
-// an HTTP request. The URL list is extracted from the model at build time.
-const REMOTE_TABLES: string[] = (window as any).__REMOTE_TABLES__ || [];
-
-async function preloadRemoteTables(db: any) {
+// __TABLE_FILES__ maps each table reference the model makes to the URL this
+// page fetches for it. The KEY is what the model wrote and what DuckDB will
+// look up, so registering the bytes under that exact name means the model needs
+// no rewriting between `dashboard dev` (reads the file off disk) and here.
+//
+// Project-relative refs like `data/x.parquet` resolve against the page, so they
+// are SAME-ORIGIN — no CORS involved at all. Absolute URLs are fetched
+// cross-origin and do need CORS on that host.
+//
+// This cannot go through db-duckdb's registerRemoteTableCallback: findTables
+// skips anything matching ^https?:// ("handled by duckdb-wasm"), so the callback
+// never fires for a remote URL. Pre-registering is also strictly better — one
+// bulk GET instead of dozens of ranged round trips.
+const TABLE_FILES: Record<string, string> = (window as any).__TABLE_FILES__ || {};
+
+async function preloadTables(db: any) {
   await Promise.all(
-    REMOTE_TABLES.map(async (url) => {
+    Object.entries(TABLE_FILES).map(async ([name, href]) => {
+      const url = new URL(href, document.baseURI).href;
       const r = await fetch(url);
       if (!r.ok) throw new Error(`fetch ${url} failed: ${r.status} ${r.statusText}`);
-      await db.registerFileBuffer(url, new Uint8Array(await r.arrayBuffer()));
+      await db.registerFileBuffer(name, new Uint8Array(await r.arrayBuffer()));
     }),
   );
 }
@@ -89,7 +95,7 @@ function getRuntime() {
     runtimeP = (async () => {
       const connection = new StaticWasmConnection({ name: "duckdb" });
       await connection.connecting;
-      await preloadRemoteTables((connection as any).database);
+      await preloadTables((connection as any).database);
       return new SingleConnectionRuntime({ connection, urlReader });
     })();
   }

package/dist/index.js

@@ -3202,13 +3202,54 @@ function inlineModelFiles(root) {
   walk(root);
   return files;
 }
-function findRemoteTables(modelFiles) {
-  const urls = /* @__PURE__ */ new Set();
-  const re = /\btable\(\s*(['"])(https?:\/\/[^'"]+)\1/g;
+function reachableModelFiles(modelFiles, entries) {
+  const IMPORT = /\bimport\s+(?:\{[^}]*\}\s+from\s+)?['"]([^'"]+)['"]/g;
+  const out = {};
+  const queue = entries.map((e) => `file:///${e.replace(/^\.?\//, "")}`);
+  while (queue.length) {
+    const key = queue.shift();
+    if (key in out) continue;
+    const src = modelFiles[key];
+    if (src == null) continue;
+    out[key] = src;
+    const dir = key.slice(0, key.lastIndexOf("/"));
+    for (const m of src.matchAll(IMPORT)) {
+      queue.push(new URL(m[1], dir + "/").href);
+    }
+  }
+  return out;
+}
+function findTableRefs(modelFiles) {
+  const refs = /* @__PURE__ */ new Set();
+  const re = /\btable\(\s*(['"])([^'"]+)\1/g;
   for (const src of Object.values(modelFiles)) {
-    for (const m of src.matchAll(re)) urls.add(m[2]);
+    for (const m of src.matchAll(re)) refs.add(m[2]);
+  }
+  return [...refs].sort();
+}
+function isDataFile(ref) {
+  if (/^https?:\/\//i.test(ref)) return true;
+  return /\.(parquet|csv|tsv|json|ndjson)$/i.test(ref);
+}
+function tableFilePlan(modelFiles, outRel) {
+  const map = {};
+  const copies = [];
+  const out = outRel.replace(/^\.?\//, "").replace(/\/$/, "");
+  for (const ref of findTableRefs(modelFiles)) {
+    if (!isDataFile(ref)) continue;
+    if (/^https?:\/\//i.test(ref)) {
+      map[ref] = ref;
+      continue;
+    }
+    const rel = ref.replace(/^\.?\//, "");
+    if (out && (rel === out || rel.startsWith(out + "/"))) {
+      map[ref] = `./${rel.slice(out.length + 1)}`;
+    } else {
+      map[ref] = `./${rel}`;
+      copies.push(rel);
+    }
   }
-  return [...urls].sort();
+  return { map, copies };
 }
 function copyDuckDBAssets(outDir) {
   const names = [
@@ -3234,7 +3275,14 @@ function navFor(dash, all, cleanUrls) {
     (n) => cleanUrls ? `./${encodeURIComponent(n)}` : `./${encodeURIComponent(n)}.html`
   );
 }
-function page(dash, all, title, givenSpecs, tileSpecs, cleanUrls) {
+function analyticsSnippet(id) {
+  if (!id) return "";
+  const j = JSON.stringify(id);
+  return `<script async src="https://www.googletagmanager.com/gtag/js?id=${encodeURIComponent(id)}"></script>
+<script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}
+gtag('js',new Date());gtag('config',${j});</script>`;
+}
+function page(dash, all, title, givenSpecs, tileSpecs, cleanUrls, analytics) {
   const info = {
     name: dash.name,
     query: dash.query,
@@ -3254,6 +3302,7 @@ function page(dash, all, title, givenSpecs, tileSpecs, cleanUrls) {
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>${esc3(dash.title || dash.name)}${title ? ` \xB7 ${esc3(title)}` : ""}</title>
 <link rel="stylesheet" href="./assets/site.css">
+${analyticsSnippet(analytics)}
 </head>
 <body>
 ${navFor(dash, all, cleanUrls)}
@@ -3276,7 +3325,7 @@ window.__GIVENS__ = ${JSON.stringify(givenSpecs)};
 </html>
 `;
 }
-function indexPage(dashboards, title, custom, cleanUrls) {
+function indexPage(dashboards, title, custom, cleanUrls, analytics) {
   const link = (n) => cleanUrls ? `./${encodeURIComponent(n)}` : `./${encodeURIComponent(n)}.html`;
   const body = custom ? `<div id="root"></div>
 <script>window.__DASHBOARDS__ = ${JSON.stringify(
@@ -3292,6 +3341,7 @@ function indexPage(dashboards, title, custom, cleanUrls) {
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>${esc3(title)}</title>
 <link rel="stylesheet" href="./assets/site.css">
+${analyticsSnippet(analytics)}
 </head>
 <body>
 ${navHtml("", dashboards, link)}
@@ -3321,6 +3371,12 @@ async function bundleDashboards(opts = {}) {
   const runner = await makeRunner(root);
   const dashboards = await discoverDashboards(root, runner);
   if (dashboards.length === 0) throw new Error(`no dashboards found in ${path7.join(root, "dashboards")}`);
+  const manifestPath = path7.join(outDir, ".bundle-manifest.json");
+  let priorData = [];
+  try {
+    priorData = JSON.parse(fs6.readFileSync(manifestPath, "utf8")).dataFiles ?? [];
+  } catch {
+  }
   for (const sub of ["assets", "duckdb"]) {
     fs6.rmSync(path7.join(outDir, sub), { recursive: true, force: true });
   }
@@ -3357,11 +3413,39 @@ async function bundleDashboards(opts = {}) {
     );
   }
   const modelFiles = inlineModelFiles(root);
-  const remoteTables = findRemoteTables(modelFiles);
+  const outRel = path7.relative(root, outDir).split(path7.sep).join("/");
+  const usedFiles = reachableModelFiles(
+    modelFiles,
+    dashboards.map((d) => d.entryFile).filter((f) => !!f)
+  );
+  const { map: tableFiles, copies } = tableFilePlan(usedFiles, outRel);
+  for (const rel of copies) {
+    const from = path7.join(root, rel);
+    if (!fs6.existsSync(from)) {
+      throw new Error(
+        `model reads '${rel}' but ${from} does not exist.
+Data files are referenced by a path relative to the project root.`
+      );
+    }
+    const to = path7.join(outDir, rel);
+    fs6.mkdirSync(path7.dirname(to), { recursive: true });
+    fs6.copyFileSync(from, to);
+  }
+  const copiedData = copies;
+  for (const stale of priorData) {
+    if (copies.includes(stale)) continue;
+    fs6.rmSync(path7.join(outDir, stale), { force: true });
+    try {
+      fs6.rmdirSync(path7.dirname(path7.join(outDir, stale)));
+    } catch {
+    }
+    console.log(`  removed stale ${stale}`);
+  }
+  fs6.writeFileSync(manifestPath, JSON.stringify({ dataFiles: copies }, null, 2) + "\n");
   fs6.writeFileSync(
     path7.join(outDir, "assets", "model-files.js"),
     `window.__MODEL_FILES__ = ${JSON.stringify(modelFiles)};
-window.__REMOTE_TABLES__ = ${JSON.stringify(remoteTables)};
+window.__TABLE_FILES__ = ${JSON.stringify(tableFiles)};
 ` + (selfHostDuckdb ? `window.__DUCKDB_BASE__ = "./duckdb/";
 ` : "")
   );
@@ -3418,9 +3502,9 @@ boot(Dashboard);
       if (!got.ok) throw new Error(`dashboard ${d.name}: ${got.error}`);
       specs = got.givens;
     }
-    fs6.writeFileSync(path7.join(outDir, `${d.name}.html`), page(d, dashboards, title, specs, tileSpecs, cleanUrls));
+    fs6.writeFileSync(path7.join(outDir, `${d.name}.html`), page(d, dashboards, title, specs, tileSpecs, cleanUrls, opts.analytics));
   }
-  fs6.writeFileSync(path7.join(outDir, "index.html"), indexPage(dashboards, title, !!landing, cleanUrls));
+  fs6.writeFileSync(path7.join(outDir, "index.html"), indexPage(dashboards, title, !!landing, cleanUrls, opts.analytics));
   if (landing) {
     await esbuild3.build({
       stdin: {
@@ -3458,6 +3542,10 @@ bundled ${dashboards.length} dashboard(s) \u2192 ${path7.relative(process.cwd(),
     selfHostDuckdb ? `  duckdb     ${duck.length} assets (self-hosted)` : `  duckdb     jsDelivr CDN (nothing copied)`
   );
   console.log(`  model      ${Object.keys(modelFiles).length} .malloy files inlined`);
+  if (copiedData.length) {
+    const mb = copiedData.reduce((a, r) => a + fs6.statSync(path7.join(root, r)).size, 0) / 1048576;
+    console.log(`  data       ${copiedData.length} file(s) copied, ${mb.toFixed(1)} MB (same-origin)`);
+  }
   console.log(
     target === "pages" ? `
 Publish: commit ${path7.basename(outDir)}/ and point GitHub Pages at it.` : `
@@ -3589,7 +3677,7 @@ async function launchCmd(mode, opts) {
 }
 
 // package.json
-var version = "0.2.20";
+var version = "0.2.22";
 
 // src/index.ts
 function shortSha(sha) {
@@ -3696,7 +3784,7 @@ program.command("author").option("-C, --root <dir>", "project root (default: cur
 program.command("test").option("-C, --root <dir>", "project root (default: current directory)").description("launch Claude wired ONLY to the explore surface \u2014 the claude.ai web preview").action(async (opts) => {
   await launchCmd("test", opts);
 });
-program.command("dashboard").argument("<action>", "action to run (dev | bundle)").option("-C, --root <dir>", "project root (default: current directory)").option("-p, --port <port>", "port to serve on (dev)", "4173").option("-o, --out <dir>", "output directory (bundle)", "docs").option("--title <title>", "site title (bundle; default: project directory name)").option("--target <target>", "deploy target: pages | vercel (bundle)", "pages").option("--duckdb <source>", "DuckDB binaries: cdn | bundled (bundle)", "cdn").option("--no-serve", "bundle only; don't serve the result (bundle)").description("preview dashboards locally (dev), or build a static site from them (bundle)").action(
+program.command("dashboard").argument("<action>", "action to run (dev | bundle)").option("-C, --root <dir>", "project root (default: current directory)").option("-p, --port <port>", "port to serve on (dev)", "4173").option("-o, --out <dir>", "output directory (bundle)", "docs").option("--title <title>", "site title (bundle; default: project directory name)").option("--target <target>", "deploy target: pages | vercel (bundle)", "pages").option("--duckdb <source>", "DuckDB binaries: cdn | bundled (bundle)", "cdn").option("--analytics <id>", "Google Analytics 4 Measurement ID, e.g. G-XXXXXXXXXX (bundle)").option("--no-serve", "bundle only; don't serve the result (bundle)").description("preview dashboards locally (dev), or build a static site from them (bundle)").action(
   async (action, opts) => {
     if (action === "dev") {
       await serveDashboard({ root: opts.root, port: Number(opts.port) });
@@ -3706,6 +3794,11 @@ program.command("dashboard").argument("<action>", "action to run (dev | bundle)"
       if (opts.target !== "pages" && opts.target !== "vercel") {
         throw new Error(`unknown --target '${opts.target}' (expected: pages | vercel)`);
       }
+      if (opts.analytics && !/^G-[A-Z0-9]+$/i.test(opts.analytics)) {
+        throw new Error(
+          `--analytics expects a GA4 Measurement ID like G-XXXXXXXXXX, got '${opts.analytics}'`
+        );
+      }
       if (opts.duckdb !== "cdn" && opts.duckdb !== "bundled") {
         throw new Error(`unknown --duckdb '${opts.duckdb}' (expected: cdn | bundled)`);
       }
@@ -3716,6 +3809,7 @@ program.command("dashboard").argument("<action>", "action to run (dev | bundle)"
         serve: opts.serve,
         target: opts.target,
         duckdb: opts.duckdb,
+        analytics: opts.analytics,
         // `dashboard dev` owns 4173/4174; default the bundle preview clear of
         // both so you can run the two side by side.
         port: opts.port === "4173" ? 4180 : Number(opts.port)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloyyo",
-  "version": "0.2.20",
+  "version": "0.2.22",
   "description": "Publish Malloy models to a Malloyyo instance",
   "license": "MIT",
   "repository": {