facetful 0.3.0 → 0.4.0

package/README.md

@@ -17,6 +17,9 @@ browser** — facet counts, pivots, top-k, filters — at interaction speed.
 - **Filter-mask cache**: WHERE conjuncts are cached as per-row-group bitmaps,
   so facet bursts sharing a filter evaluate it once, and a `LIKE '%needle%'`
   extending a cached needle verifies only the rows the shorter one matched.
+- **Joins, CTEs and subqueries as cached materializations**: `JOIN`, `WITH`,
+  `FROM (select …)`, `IN (select …)`, `EXISTS` all build a derived table once
+  and serve every later facet from it — no per-query join cost.
 - Dates, medians, stddev, group_concat, `select *`, LIKE fast paths — the
   boring things work.
 
@@ -57,6 +60,50 @@ Persistence is always explicit (`storeOpfs`), never write-behind. OPFS needs a
 secure context (https or localhost); everything degrades to memory-only
 without one.
 
+## Derived tables: `materialize`
+
+```js
+// a grouped result becomes a table of its own — queryable like any other,
+// with the same typed columns, nulls and (here) sort metadata
+await db.materialize("fuel_state", `
+  select fuel, state, count(*) as n, round(sum(net_generation_mwh)/1e6, 3) as twh
+  from t group by fuel, state order by twh desc`);
+const r = await db.query("select fuel, sum(twh) as twh from t group by fuel", { table: "fuel_state" });
+
+// persist the image to OPFS so a later visit can loadOpfs() it instead
+await db.materialize("fuel_state", sql, { persist: "facetful/fuel_state.facetful" });
+```
+
+Pre-aggregate a large detail table once per session and run the facets
+against the rollup; every SELECT item needs a distinct name. Row order is the
+query's output order, so a materialized `ORDER BY` is recorded as the table's
+sort. The CLI has the same verb: `facetful materialize in.facetful "select …" out.facetful`.
+
+## Joins, CTEs, subqueries
+
+Every table the worker holds — loaded, opened from Parquet, or materialized —
+is registered under its name, and any query can name it:
+
+```js
+await db.materialize("plants", "select plant_id, state, count(*) as n_gens from t group by plant_id, state");
+await db.query(`
+  select t.fuel, p.state, sum(t.mwh) as mwh
+  from t left join plants p on t.plant_id = p.plant_id
+  where p.n_gens >= 3 group by t.fuel, p.state order by mwh desc`);
+```
+
+`[INNER|LEFT] JOIN … ON a.k = b.k [AND …]` and `USING (k)`; the right side's
+join key must be unique (a dimension). `WITH name AS (…)`, `FROM (select …) s`,
+`x IN (select …)`, `(a, b) IN (select …)`, `EXISTS (select … where d.k = t.k)`.
+All of these are **materializations, not query-time operators**: the first
+query builds the joined or derived table (milliseconds for a fact table
+against a dimension), and it lives in a per-table cache (64 MB default) that
+every later query with the same shape reuses — a facet burst over a join pays
+for the join once. Correlated subqueries beyond `inner.k = outer.k` equalities,
+`FULL`/`RIGHT` joins and non-unique right keys are refused with a clear error.
+`NOT IN` follows SQL's NULL rule (a NULL in the set makes it select nothing);
+`NOT EXISTS` doesn't, and is usually what you mean.
+
 ## Parquet support
 
 Reading uses [hyparquet](https://github.com/hyparam/hyparquet) (~20 KB gz),
@@ -72,10 +119,10 @@ browser transcoder (the native CLI has no such limit).
 
 ## SQL dialect, briefly
 
-SELECT-only, single table, SQLite semantics (3-valued logic, null-skipping
-aggregates, truncating integer division, NULL-first ascending sorts).
-Idioms: `IN`, `BETWEEN`, `IS [NOT] NULL`, `[NOT] LIKE`, `CASE WHEN`, `CAST`,
-`COUNT(DISTINCT x)`, `||`, `select *`. Aggregates: count, sum, avg, min, max,
+SELECT-only, SQLite semantics (3-valued logic, null-skipping aggregates,
+truncating integer division, NULL-first ascending sorts). Idioms: `IN`,
+`BETWEEN`, `IS [NOT] NULL`, `[NOT] LIKE`, `CASE WHEN`, `CAST`,
+`COUNT(DISTINCT x)`, `||`, `select *`, `JOIN`/`WITH`/subqueries as above. Aggregates: count, sum, avg, min, max,
 count(distinct), median, stddev, group_concat. Scalars: math (abs, round,
 floor, ceil, sqrt, pow, exp, ln, sign), text (lower, upper, length, substr,
 trim/ltrim/rtrim, replace, instr, concat), null handling (coalesce, ifnull,

package/core.js

@@ -121,6 +121,30 @@ export class Engine {
     return img;
   }
 
+  /**
+   * Materialize a query's result as a compiled image: the derived-table
+   * primitive. Returns an image handle (pass to openImage / imageBytes), or
+   * throws QueryError with the engine's diagnostic.
+   */
+  materialize(tableHandle, sql, { groupTarget = 65536 } = {}) {
+    const sqlBytes = this.enc.encode(sql);
+    const sqlPtr = u32(this.w.alloc(sqlBytes.byteLength));
+    new Uint8Array(this.mem(), sqlPtr, sqlBytes.byteLength).set(sqlBytes);
+    const h = this.w.table_materialize(tableHandle, sqlPtr, sqlBytes.byteLength, groupTarget);
+    try {
+      if (this.w.outcome_is_err(h)) {
+        const n = this.w.outcome_error(h, this.scratch, 4096);
+        throw new QueryError(this.dec.decode(new Uint8Array(this.mem(), this.scratch, n)));
+      }
+      const img = this.w.outcome_image(h);
+      if (!img) throw new Error("materialize produced no image");
+      return img;
+    } finally {
+      this.w.dealloc(sqlPtr, sqlBytes.byteLength);
+      this.w.outcome_free(h);
+    }
+  }
+
   /** Copy a compiled image's bytes out (for OPFS persistence). */
   imageBytes(img) {
     return new Uint8Array(this.mem(), u32(this.w.image_ptr(img)), u32(this.w.image_len(img))).slice();
@@ -133,6 +157,15 @@ export class Engine {
     return { handle, rows: this.w.table_total_rows(handle) };
   }
 
+  /** Make `handle` reachable by `name` from other tables' queries (FROM / JOIN). */
+  catalogRegister(name, handle) {
+    const b = this.enc.encode(name);
+    const p = u32(this.w.alloc(b.byteLength));
+    new Uint8Array(this.mem(), p, b.byteLength).set(b);
+    this.w.catalog_register(p, b.byteLength, handle);
+    this.w.dealloc(p, b.byteLength);
+  }
+
   /** Run SQL; returns { columns, rowCount, stats } with copied-out buffers. */
   query(tableHandle, sql) {
     const sqlBytes = this.enc.encode(sql);

package/index.d.ts

@@ -92,6 +92,17 @@ export declare class Facetful {
   loadParquet(name: string, buffer: ArrayBuffer): Promise<{ rows: number; transcodeMs: number }>;
 
   /** Persist a .facetful image into OPFS at `path` (buffer transferred). */
+  /**
+   * Materialize a query's result as a new table `name`, queryable via
+   * `{ table: name }`. With `persist`, the image is also written to OPFS at
+   * that path for a later `loadOpfs`. `bytes` is the persisted image size (0
+   * when not persisted).
+   */
+  materialize(
+    name: string,
+    sql: string,
+    options?: { table?: string; persist?: string },
+  ): Promise<{ rows: number; bytes: number; elapsedMs: number }>;
   storeOpfs(path: string, buffer: ArrayBuffer): Promise<{ bytes: number }>;
 
   /**

package/index.js

@@ -71,6 +71,20 @@ export class Facetful {
     return this._call({ cmd: "loadParquet", name, buffer }, [buffer]);
   }
 
+  /**
+   * Materialize a query's result as a new table named `name`: a derived,
+   * immutable table you then query like any other (`{ table: name }`).
+   * Runs against `table` (default: the last loaded). With `persist`, the
+   * compiled image is also written to OPFS at that path, so a later visit
+   * can `loadOpfs(name, path)` instead of recomputing. Returns { rows, bytes }.
+   *
+   * The SQL may name other loaded tables in FROM and JOIN, so a joined,
+   * persisted table is `materialize("x", "select … from t join dims d using (k)")`.
+   */
+  async materialize(name, sql, { table, persist } = {}) {
+    return this._call({ cmd: "materialize", name, sql, table, persist });
+  }
+
   /** Persist a .facetful image into OPFS at `path` (e.g. "facetful/plants.facetful"). */
   async storeOpfs(path, buffer) {
     return this._call({ cmd: "storeOpfs", path, buffer }, [buffer]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "facetful",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Tiny columnar SQL engine for browser faceting — Rust/wasm, ~170KB gz, zero dependencies. Open Parquet instantly, facet 1M rows in milliseconds, persist to OPFS.",
   "license": "MIT",
   "author": "David Raznick",

package/worker.js

@@ -8,6 +8,8 @@ import { parquetToColumns } from "./parquet.js";
 
 let engine = null;
 const tables = new Map(); // name -> handle
+// register under a name for FROM / JOIN from other tables' queries
+const setTable = (name, handle) => { setTable(name, handle); engine.catalogRegister(name, handle); };
 let lastTable = null;
 
 // OPFS file registry: the wasm's opfs_read import addresses files by these ids.
@@ -77,7 +79,7 @@ async function opfsOpenTable(name, path, cacheBytes) {
   }
   try {
     const { handle, rows } = engine.openOpfsTable(fileId, h.getSize(), cacheBytes);
-    tables.set(name, handle);
+    setTable(name, handle);
     lastTable = handle;
     return { rows, fileLen: h.getSize() };
   } catch (err) {
@@ -127,7 +129,7 @@ self.onmessage = async (e) => {
       reply({ ok: true });
     } else if (cmd === "load") {
       const { handle, rows } = engine.openTable(e.data.buffer);
-      tables.set(e.data.name, handle);
+      setTable(e.data.name, handle);
       lastTable = handle;
       reply({ ok: true, rows });
     } else if (cmd === "loadParquet") {
@@ -135,7 +137,7 @@ self.onmessage = async (e) => {
       const t0 = performance.now();
       const { rows, img } = await transcodeParquet(e.data.buffer);
       const { handle } = engine.openImage(img);
-      tables.set(e.data.name, handle);
+      setTable(e.data.name, handle);
       lastTable = handle;
       reply({ ok: true, rows, transcodeMs: performance.now() - t0 });
     } else if (cmd === "openParquet") {
@@ -159,7 +161,7 @@ self.onmessage = async (e) => {
         imageBytes = engine.imageBytes(img);
       } catch { /* copy-out failed: open uncached */ }
       const { handle } = engine.openImage(img);
-      tables.set(e.data.name, handle);
+      setTable(e.data.name, handle);
       lastTable = handle;
       if (imageBytes) {
         try {
@@ -168,6 +170,22 @@ self.onmessage = async (e) => {
         } catch { /* non-secure context or quota: stay memory-only */ }
       }
       reply({ ok: true, rows, source: "transcode", cached, transcodeMs });
+    } else if (cmd === "materialize") {
+      // derived table: run the query on the source table, compile its result
+      // into a new in-memory table under `name`; optionally persist the image
+      const src = e.data.table ? tables.get(e.data.table) : lastTable;
+      if (!src) throw new Error(`no table loaded${e.data.table ? `: '${e.data.table}'` : ""}`);
+      const t0 = performance.now();
+      const img = engine.materialize(src, e.data.sql);
+      let bytes = 0;
+      if (e.data.persist) {
+        const image = engine.imageBytes(img);
+        bytes = image.byteLength;
+        await opfsWrite(e.data.persist, image);
+      }
+      const { handle, rows } = engine.openImage(img);
+      setTable(e.data.name, handle);
+      reply({ ok: true, rows, bytes, elapsedMs: performance.now() - t0 });
     } else if (cmd === "storeOpfs") {
       await opfsWrite(e.data.path, new Uint8Array(e.data.buffer));
       reply({ ok: true, bytes: e.data.buffer.byteLength });