@malloydata/malloyyo 0.2.20 → 0.2.21

package/dist/frame-wasm-entry.tsx

@@ -46,24 +46,30 @@ class StaticWasmConnection extends DuckDBWASMConnection {
   }
 }
 
-// Whole-file fetch, not ranged reads. For a columnar file whose dominant column
-// every query touches, ranged reads trade one bulk download for dozens of WAN
-// round trips and lose badly.
+// Whole-file fetch, not ranged reads.
 //
-// This CANNOT go through registerRemoteTableCallback: db-duckdb's findTables
-// skips anything matching ^https?:// ("handled by duckdb-wasm") so the callback
-// never fires for a remote URL. Instead we pre-register the bytes under the
-// exact URL string before any query runs — duckdb-wasm resolves its registered
-// files by name first, so the table reference finds the buffer and never issues
-// an HTTP request. The URL list is extracted from the model at build time.
-const REMOTE_TABLES: string[] = (window as any).__REMOTE_TABLES__ || [];
-
-async function preloadRemoteTables(db: any) {
+// __TABLE_FILES__ maps each table reference the model makes to the URL this
+// page fetches for it. The KEY is what the model wrote and what DuckDB will
+// look up, so registering the bytes under that exact name means the model needs
+// no rewriting between `dashboard dev` (reads the file off disk) and here.
+//
+// Project-relative refs like `data/x.parquet` resolve against the page, so they
+// are SAME-ORIGIN — no CORS involved at all. Absolute URLs are fetched
+// cross-origin and do need CORS on that host.
+//
+// This cannot go through db-duckdb's registerRemoteTableCallback: findTables
+// skips anything matching ^https?:// ("handled by duckdb-wasm"), so the callback
+// never fires for a remote URL. Pre-registering is also strictly better — one
+// bulk GET instead of dozens of ranged round trips.
+const TABLE_FILES: Record<string, string> = (window as any).__TABLE_FILES__ || {};
+
+async function preloadTables(db: any) {
   await Promise.all(
-    REMOTE_TABLES.map(async (url) => {
+    Object.entries(TABLE_FILES).map(async ([name, href]) => {
+      const url = new URL(href, document.baseURI).href;
       const r = await fetch(url);
       if (!r.ok) throw new Error(`fetch ${url} failed: ${r.status} ${r.statusText}`);
-      await db.registerFileBuffer(url, new Uint8Array(await r.arrayBuffer()));
+      await db.registerFileBuffer(name, new Uint8Array(await r.arrayBuffer()));
     }),
   );
 }
@@ -89,7 +95,7 @@ function getRuntime() {
     runtimeP = (async () => {
       const connection = new StaticWasmConnection({ name: "duckdb" });
       await connection.connecting;
-      await preloadRemoteTables((connection as any).database);
+      await preloadTables((connection as any).database);
       return new SingleConnectionRuntime({ connection, urlReader });
     })();
   }

package/dist/index.js

@@ -3202,13 +3202,54 @@ function inlineModelFiles(root) {
   walk(root);
   return files;
 }
-function findRemoteTables(modelFiles) {
-  const urls = /* @__PURE__ */ new Set();
-  const re = /\btable\(\s*(['"])(https?:\/\/[^'"]+)\1/g;
+function reachableModelFiles(modelFiles, entries) {
+  const IMPORT = /\bimport\s+(?:\{[^}]*\}\s+from\s+)?['"]([^'"]+)['"]/g;
+  const out = {};
+  const queue = entries.map((e) => `file:///${e.replace(/^\.?\//, "")}`);
+  while (queue.length) {
+    const key = queue.shift();
+    if (key in out) continue;
+    const src = modelFiles[key];
+    if (src == null) continue;
+    out[key] = src;
+    const dir = key.slice(0, key.lastIndexOf("/"));
+    for (const m of src.matchAll(IMPORT)) {
+      queue.push(new URL(m[1], dir + "/").href);
+    }
+  }
+  return out;
+}
+function findTableRefs(modelFiles) {
+  const refs = /* @__PURE__ */ new Set();
+  const re = /\btable\(\s*(['"])([^'"]+)\1/g;
   for (const src of Object.values(modelFiles)) {
-    for (const m of src.matchAll(re)) urls.add(m[2]);
+    for (const m of src.matchAll(re)) refs.add(m[2]);
+  }
+  return [...refs].sort();
+}
+function isDataFile(ref) {
+  if (/^https?:\/\//i.test(ref)) return true;
+  return /\.(parquet|csv|tsv|json|ndjson)$/i.test(ref);
+}
+function tableFilePlan(modelFiles, outRel) {
+  const map = {};
+  const copies = [];
+  const out = outRel.replace(/^\.?\//, "").replace(/\/$/, "");
+  for (const ref of findTableRefs(modelFiles)) {
+    if (!isDataFile(ref)) continue;
+    if (/^https?:\/\//i.test(ref)) {
+      map[ref] = ref;
+      continue;
+    }
+    const rel = ref.replace(/^\.?\//, "");
+    if (out && (rel === out || rel.startsWith(out + "/"))) {
+      map[ref] = `./${rel.slice(out.length + 1)}`;
+    } else {
+      map[ref] = `./${rel}`;
+      copies.push(rel);
+    }
   }
-  return [...urls].sort();
+  return { map, copies };
 }
 function copyDuckDBAssets(outDir) {
   const names = [
@@ -3321,6 +3362,12 @@ async function bundleDashboards(opts = {}) {
   const runner = await makeRunner(root);
   const dashboards = await discoverDashboards(root, runner);
   if (dashboards.length === 0) throw new Error(`no dashboards found in ${path7.join(root, "dashboards")}`);
+  const manifestPath = path7.join(outDir, ".bundle-manifest.json");
+  let priorData = [];
+  try {
+    priorData = JSON.parse(fs6.readFileSync(manifestPath, "utf8")).dataFiles ?? [];
+  } catch {
+  }
   for (const sub of ["assets", "duckdb"]) {
     fs6.rmSync(path7.join(outDir, sub), { recursive: true, force: true });
   }
@@ -3357,11 +3404,39 @@ async function bundleDashboards(opts = {}) {
     );
   }
   const modelFiles = inlineModelFiles(root);
-  const remoteTables = findRemoteTables(modelFiles);
+  const outRel = path7.relative(root, outDir).split(path7.sep).join("/");
+  const usedFiles = reachableModelFiles(
+    modelFiles,
+    dashboards.map((d) => d.entryFile).filter((f) => !!f)
+  );
+  const { map: tableFiles, copies } = tableFilePlan(usedFiles, outRel);
+  for (const rel of copies) {
+    const from = path7.join(root, rel);
+    if (!fs6.existsSync(from)) {
+      throw new Error(
+        `model reads '${rel}' but ${from} does not exist.
+Data files are referenced by a path relative to the project root.`
+      );
+    }
+    const to = path7.join(outDir, rel);
+    fs6.mkdirSync(path7.dirname(to), { recursive: true });
+    fs6.copyFileSync(from, to);
+  }
+  const copiedData = copies;
+  for (const stale of priorData) {
+    if (copies.includes(stale)) continue;
+    fs6.rmSync(path7.join(outDir, stale), { force: true });
+    try {
+      fs6.rmdirSync(path7.dirname(path7.join(outDir, stale)));
+    } catch {
+    }
+    console.log(`  removed stale ${stale}`);
+  }
+  fs6.writeFileSync(manifestPath, JSON.stringify({ dataFiles: copies }, null, 2) + "\n");
   fs6.writeFileSync(
     path7.join(outDir, "assets", "model-files.js"),
     `window.__MODEL_FILES__ = ${JSON.stringify(modelFiles)};
-window.__REMOTE_TABLES__ = ${JSON.stringify(remoteTables)};
+window.__TABLE_FILES__ = ${JSON.stringify(tableFiles)};
 ` + (selfHostDuckdb ? `window.__DUCKDB_BASE__ = "./duckdb/";
 ` : "")
   );
@@ -3458,6 +3533,10 @@ bundled ${dashboards.length} dashboard(s) \u2192 ${path7.relative(process.cwd(),
     selfHostDuckdb ? `  duckdb     ${duck.length} assets (self-hosted)` : `  duckdb     jsDelivr CDN (nothing copied)`
   );
   console.log(`  model      ${Object.keys(modelFiles).length} .malloy files inlined`);
+  if (copiedData.length) {
+    const mb = copiedData.reduce((a, r) => a + fs6.statSync(path7.join(root, r)).size, 0) / 1048576;
+    console.log(`  data       ${copiedData.length} file(s) copied, ${mb.toFixed(1)} MB (same-origin)`);
+  }
   console.log(
     target === "pages" ? `
 Publish: commit ${path7.basename(outDir)}/ and point GitHub Pages at it.` : `
@@ -3589,7 +3668,7 @@ async function launchCmd(mode, opts) {
 }
 
 // package.json
-var version = "0.2.20";
+var version = "0.2.21";
 
 // src/index.ts
 function shortSha(sha) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloyyo",
-  "version": "0.2.20",
+  "version": "0.2.21",
   "description": "Publish Malloy models to a Malloyyo instance",
   "license": "MIT",
   "repository": {