facetful 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 David Raznick
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,90 @@
+# facetful
+
+A tiny columnar SQL engine for browser faceting. Rust compiled to ~150 KB gz
+of WebAssembly, zero runtime dependencies, running in a dedicated worker.
+
+Built for one job and fast at it: **exploring 100K–5M row datasets in the
+browser** — facet counts, pivots, top-k, filters — at interaction speed.
+
+- **Open Parquet instantly.** First visit transcodes once (seconds) and caches
+  the compiled image in OPFS; every visit after reopens in ~20–40 ms with zero
+  decode. Measured on a real 183K×52 dataset: facet queries 1–4 ms.
+- **SELECT-only SQL**, SQLite-flavored semantics, verified cell-for-cell
+  against SQLite by a differential test suite. Friendly errors with carets and
+  did-you-mean hints.
+- **Larger-than-memory**: tables can live in OPFS and load column segments
+  lazily through a byte-budgeted LRU cache.
+- Dates, medians, stddev, group_concat, `select *`, LIKE fast paths — the
+  boring things work.
+
+## Quickstart
+
+```js
+import { Facetful } from "facetful";
+
+const db = await Facetful.open();
+const buf = await (await fetch("plants.parquet")).arrayBuffer();
+await db.openParquet("plants", buf); // transcode once, OPFS-cached thereafter
+
+const r = await db.query(`
+  select "Country/area", count(*) as n, round(sum("Capacity (MW)"), 1) as mw
+  from t
+  where "Status" = 'operating'
+  group by "Country/area"
+  order by mw desc limit 15
+`);
+
+for (const row of r.rows()) console.log(row);
+r.columnRaw("mw"); // Float64Array + validity bitmap, near-zero copy (charts)
+console.log(r.elapsedMs, r.stats); // ms in worker, row groups pruned
+```
+
+The SQL table is always named `t`. Multiple tables coexist:
+`db.query(sql, { table: "plants" })`.
+
+## The three ways in
+
+| method | use when |
+|---|---|
+| `openParquet(name, buf)` | the default: Parquet in, OPFS-cached compiled image, instant repeat visits |
+| `load(name, buf)` | you already have a `.facetful` image (built by the native CLI) |
+| `loadOpfs(name, path, {cacheBytes})` | the file is in OPFS; open lazily, spill-over for big data |
+
+Persistence is always explicit (`storeOpfs`), never write-behind. OPFS needs a
+secure context (https or localhost); everything degrades to memory-only
+without one.
+
+## Parquet support
+
+Reading uses [hyparquet](https://github.com/hyparam/hyparquet) (~20 KB gz),
+declared as an optional peer dependency and loaded dynamically only when a
+Parquet method is called. Under a bundler, `npm install hyparquet` is enough;
+without one, pass `hyparquetUrl` to `Facetful.open`.
+
+Flat schemas only (no nested/repeated columns). BOOLEAN/INT32/INT64 →
+integers (narrowed), FLOAT/DOUBLE → float64, strings → dictionary-encoded when
+it pays, DATE/TIMESTAMP → real date/timestamp columns (days / ms since epoch,
+ISO strings on output). Int64 values beyond 2^53 lose precision in the
+browser transcoder (the native CLI has no such limit).
+
+## SQL dialect, briefly
+
+SELECT-only, single table, SQLite semantics (3-valued logic, null-skipping
+aggregates, truncating integer division, NULL-first ascending sorts).
+Idioms: `IN`, `BETWEEN`, `IS [NOT] NULL`, `[NOT] LIKE`, `CASE WHEN`, `CAST`,
+`COUNT(DISTINCT x)`, `||`, `select *`. Aggregates: count, sum, avg, min, max,
+count(distinct), median, stddev, group_concat. Scalars: math (abs, round,
+floor, ceil, sqrt, pow, exp, ln, sign), text (lower, upper, length, substr,
+trim/ltrim/rtrim, replace, instr, concat), null handling (coalesce, ifnull,
+nullif), temporal (year, month, day, hour, minute, second, date, timestamp,
+strftime). Quote column names with spaces: `"Capacity (MW)"`.
+
+## Building the wasm from source
+
+The engine lives in the same repository (Rust workspace, zero dependencies).
+`scripts/build-package.sh` builds the wasm (with wasm-opt when available),
+copies it next to this package, and runs `npm pack`.
+
+## License
+
+MIT

package/core.js

@@ -0,0 +1,183 @@
+// Environment-agnostic marshalling over the facetful wasm exports.
+// Used by worker.js in the browser and driven directly by the Node smoke test.
+
+const KINDS = { 1: "int", 2: "float", 3: "bool", 4: "text", 5: "date", 6: "timestamp" };
+
+// The wasm module declares one import: env.opfs_read(fileId, offset, len, destPtr)
+// -> bytes read. The browser worker supplies a real implementation over OPFS
+// sync access handles; environments without OPFS (Node smoke test) get a stub
+// that fails any read (memory-backed tables never call it).
+export async function instantiate(wasmBytes, opfsRead) {
+  let engine = null;
+  const env = {
+    opfs_read: (fileId, offset, len, destPtr) => {
+      if (!opfsRead || !engine) return -1;
+      // the view must be built per call: memory.buffer detaches on growth
+      return opfsRead(fileId, offset, new Uint8Array(engine.mem(), destPtr, len));
+    },
+  };
+  const { instance } = await WebAssembly.instantiate(wasmBytes, { env });
+  engine = new Engine(instance.exports);
+  return engine;
+}
+
+export class Engine {
+  constructor(exports) {
+    this.w = exports;
+    this.scratch = this.w.alloc(4096);
+    this.enc = new TextEncoder();
+    this.dec = new TextDecoder();
+  }
+  mem() {
+    return this.w.memory.buffer;
+  }
+
+  /** Open a .facetful image from bytes; returns a table handle. */
+  openTable(bytes) {
+    const src = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+    const ptr = this.w.alloc(src.byteLength);
+    new Uint8Array(this.mem(), ptr, src.byteLength).set(src);
+    const handle = this.w.table_open(ptr, src.byteLength);
+    if (!handle) throw new Error("not a valid .facetful image");
+    return { handle, rows: this.w.table_total_rows(handle) };
+  }
+
+  /** Open a table backed by a registered OPFS file (reads go through opfs_read). */
+  openOpfsTable(fileId, fileLen, cacheBytes) {
+    const handle = this.w.table_open_opfs(fileId, fileLen, cacheBytes ?? 0);
+    if (!handle) throw new Error("not a valid .facetful image (OPFS)");
+    return { handle, rows: this.w.table_total_rows(handle) };
+  }
+
+  colByName(tableHandle, name) {
+    const b = this.enc.encode(name);
+    new Uint8Array(this.mem(), this.scratch, b.byteLength).set(b);
+    return this.w.table_col_by_name(tableHandle, this.scratch, b.byteLength);
+  }
+
+  /** Pre-touch one column's segments into the cache; returns bytes read. */
+  warmColumn(tableHandle, colIdx) {
+    return this.w.table_warm(tableHandle, colIdx);
+  }
+
+  /** { segments, bytes } currently cached. */
+  cacheStats(tableHandle) {
+    const packed = this.w.table_cache_stats(tableHandle);
+    return { segments: Number(packed >> 32n), bytes: Number(packed & 0xffffffffn) * 1024 };
+  }
+
+  /**
+   * Compile typed columns into a .facetful image (the baseline compiler —
+   * same Rust code path as the native CLI). Columns:
+   *   { name, kind: "num", data: Float64Array, isInt?, temporal?: "date"|"timestamp",
+   *     validity?: Uint8Array }   // temporal: data = days / ms since epoch
+   *   { name, kind: "text", offsets: Uint32Array, bytes: Uint8Array, validity? }
+   * validity is one byte per row, 0 = null. Returns an image handle; pass it
+   * to imageBytes() (copy out, e.g. for OPFS) and/or openImage() (consumes).
+   */
+  compileTable(rows, columns, { groupTarget = 65536 } = {}) {
+    const put = (src) => {
+      const p = this.w.alloc(src.byteLength);
+      new Uint8Array(this.mem(), p, src.byteLength).set(
+        new Uint8Array(src.buffer, src.byteOffset, src.byteLength),
+      );
+      return p;
+    };
+    const b = this.w.compile_begin(rows);
+    for (const c of columns) {
+      const nameB = this.enc.encode(c.name);
+      const nameP = put(nameB);
+      const validP = c.validity ? put(c.validity) : 0;
+      if (c.kind === "num") {
+        const dataP = put(c.data);
+        const numKind =
+          c.temporal === "date" ? 2 : c.temporal === "timestamp" ? 3 : c.isInt ? 1 : 0;
+        this.w.compile_add_num(b, nameP, nameB.byteLength, dataP, validP, numKind);
+        this.w.dealloc(dataP, c.data.byteLength);
+      } else {
+        const offP = put(c.offsets);
+        const bytesP = put(c.bytes);
+        this.w.compile_add_text(b, nameP, nameB.byteLength, offP, bytesP, c.bytes.byteLength, validP);
+        this.w.dealloc(offP, c.offsets.byteLength);
+        this.w.dealloc(bytesP, c.bytes.byteLength);
+      }
+      this.w.dealloc(nameP, nameB.byteLength);
+      if (validP) this.w.dealloc(validP, c.validity.byteLength);
+    }
+    const img = this.w.compile_finish(b, groupTarget);
+    if (!img) throw new Error("compile failed (mismatched column lengths?)");
+    return img;
+  }
+
+  /** Copy a compiled image's bytes out (for OPFS persistence). */
+  imageBytes(img) {
+    return new Uint8Array(this.mem(), this.w.image_ptr(img), this.w.image_len(img)).slice();
+  }
+
+  /** Open a table over a compiled image; consumes the image handle (no copy). */
+  openImage(img) {
+    const handle = this.w.image_open_table(img);
+    if (!handle) throw new Error("compiled image failed to open");
+    return { handle, rows: this.w.table_total_rows(handle) };
+  }
+
+  /** Run SQL; returns { columns, rowCount, stats } with copied-out buffers. */
+  query(tableHandle, sql) {
+    const sqlBytes = this.enc.encode(sql);
+    const sqlPtr = this.w.alloc(sqlBytes.byteLength);
+    new Uint8Array(this.mem(), sqlPtr, sqlBytes.byteLength).set(sqlBytes);
+    const h = this.w.query_run(tableHandle, sqlPtr, sqlBytes.byteLength);
+    try {
+      if (this.w.outcome_is_err(h)) {
+        const n = this.w.outcome_error(h, this.scratch, 4096);
+        throw new QueryError(this.dec.decode(new Uint8Array(this.mem(), this.scratch, n)));
+      }
+      const rowCount = this.w.outcome_rows(h);
+      const nCols = this.w.outcome_cols(h);
+      const stats = this.w.outcome_scan_stats(h);
+      const columns = [];
+      for (let i = 0; i < nCols; i++) {
+        const kind = KINDS[this.w.col_kind(h, i)];
+        const nameLen = this.w.col_name(h, i, this.scratch, 4096);
+        const name = this.dec.decode(new Uint8Array(this.mem(), this.scratch, nameLen));
+        const validity = new Uint8Array(
+          this.mem(), this.w.col_validity_ptr(h, i), Math.ceil(rowCount / 8),
+        ).slice();
+        const col = { name, kind, validity };
+        if (kind === "int" || kind === "float" || kind === "date" || kind === "timestamp") {
+          col.values = new Float64Array(this.mem(), this.w.col_f64_ptr(h, i), rowCount).slice();
+        } else if (kind === "bool") {
+          col.values = new Uint8Array(this.mem(), this.w.col_bools_ptr(h, i), rowCount).slice();
+        } else {
+          col.offsets = new Uint32Array(this.mem(), this.w.col_offsets_ptr(h, i), rowCount + 1).slice();
+          col.bytes = new Uint8Array(
+            this.mem(), this.w.col_bytes_ptr(h, i), this.w.col_bytes_len(h, i),
+          ).slice();
+        }
+        columns.push(col);
+      }
+      return {
+        columns,
+        rowCount,
+        stats: { scannedGroups: Number(stats & 0xffffffffn), totalGroups: Number(stats >> 32n) },
+      };
+    } finally {
+      this.w.outcome_free(h);
+    }
+  }
+}
+
+export class QueryError extends Error {}
+
+/** Transferable list for a query payload (zero-copy postMessage). */
+export function transferables(result) {
+  const t = [];
+  for (const c of result.columns) {
+    t.push(c.validity.buffer);
+    if (c.values) t.push(c.values.buffer);
+    if (c.offsets) {
+      t.push(c.offsets.buffer, c.bytes.buffer);
+    }
+  }
+  return t;
+}

package/index.d.ts

@@ -0,0 +1,125 @@
+// Type definitions for facetful — the main-thread API (index.js).
+// The engine runs in a dedicated worker; all methods are asynchronous.
+
+export type ColumnKind = "int" | "float" | "bool" | "text" | "date" | "timestamp";
+
+export interface OpenOptions {
+  /** URL of the engine wasm. Default: facetful_wasm.wasm next to the package. */
+  wasmUrl?: string | URL;
+  /** URL of the worker module. Default: worker.js next to the package. */
+  workerUrl?: string | URL;
+  /**
+   * Module specifier or URL for hyparquet (needed only by the Parquet
+   * methods). Default "hyparquet" — resolved by your bundler; pass an
+   * explicit URL when running without one.
+   */
+  hyparquetUrl?: string;
+}
+
+export interface QueryStats {
+  /** Row groups actually scanned (min/max pruning and early exit skip the rest). */
+  scannedGroups: number;
+  totalGroups: number;
+}
+
+export interface RawColumn {
+  name: string;
+  kind: ColumnKind;
+  /** Bitmap, bit i set = row i is non-null. */
+  validity: Uint8Array;
+  /** int/float/date/timestamp: f64 lanes (date = days since epoch, timestamp = ms). bool: 0/1 bytes. */
+  values?: Float64Array | Uint8Array;
+  /** text only: rowCount+1 byte offsets into `bytes`. */
+  offsets?: Uint32Array;
+  /** text only: UTF-8 blob. */
+  bytes?: Uint8Array;
+}
+
+export type CellValue = number | string | boolean | null;
+
+export declare class Result {
+  columns: { name: string; kind: ColumnKind }[];
+  rowCount: number;
+  stats: QueryStats;
+  /** Execution time inside the worker, ms (excludes the message hop). */
+  elapsedMs: number;
+  /** Raw transferred buffers for a column — near-zero copy, ideal for charts. */
+  columnRaw(name: string): RawColumn;
+  /** Materialized values with nulls; date/timestamp as ISO strings. */
+  column(name: string): CellValue[];
+  /** Row objects, materialized lazily. */
+  rows(): Generator<Record<string, CellValue>>;
+}
+
+export interface LoadResult {
+  name: string;
+  rows: number;
+}
+
+export interface OpenParquetResult {
+  rows: number;
+  /** "cache" = compiled image reopened from OPFS (no transcode); "transcode" = first visit. */
+  source: "cache" | "transcode";
+  /** Present when source = "cache". */
+  openMs?: number;
+  /** Present when source = "transcode". */
+  transcodeMs?: number;
+  /** Whether the compiled image was persisted to OPFS for next time. */
+  cached?: boolean;
+}
+
+export declare class Facetful {
+  /** Start the engine (spawns the worker, instantiates wasm). */
+  static open(options?: OpenOptions): Promise<Facetful>;
+
+  /** Load a .facetful image from an ArrayBuffer (transferred). */
+  load(name: string, buffer: ArrayBuffer): Promise<LoadResult>;
+
+  /**
+   * Open a Parquet file (ArrayBuffer, transferred). The compiled image is
+   * cached in OPFS keyed by content hash + format version: repeat visits
+   * reopen in ~tens of ms with no transcode. Requires hyparquet (see
+   * OpenOptions.hyparquetUrl) and a secure context for the OPFS cache
+   * (degrades to memory-only otherwise).
+   */
+  openParquet(
+    name: string,
+    buffer: ArrayBuffer,
+    options?: { cacheBytes?: number },
+  ): Promise<OpenParquetResult>;
+
+  /** Transcode-only variant: Parquet -> in-memory table, no OPFS. */
+  loadParquet(name: string, buffer: ArrayBuffer): Promise<{ rows: number; transcodeMs: number }>;
+
+  /** Persist a .facetful image into OPFS at `path` (buffer transferred). */
+  storeOpfs(path: string, buffer: ArrayBuffer): Promise<{ bytes: number }>;
+
+  /**
+   * Open a table over an OPFS file: metadata reads now, column segments load
+   * lazily into an LRU bounded by cacheBytes (default min(deviceMemory/4, 1GB)).
+   */
+  loadOpfs(
+    name: string,
+    path: string,
+    options?: { cacheBytes?: number },
+  ): Promise<{ name: string; rows: number; fileLen: number; cacheBytes: number }>;
+
+  /** Delete an OPFS file (closes any open handles on it first). */
+  removeOpfs(path: string): Promise<void>;
+
+  /** Pre-touch columns into the segment cache; queries interleave with warming. */
+  warm(cols: string[], options?: { table?: string }): Promise<{ bytes: number }>;
+
+  /** Current segment-cache occupancy for a table. */
+  cacheStats(options?: { table?: string }): Promise<{ segments: number; bytes: number }>;
+
+  /**
+   * Run SQL (SELECT-only; the table is always `t`). `table` picks a loaded
+   * table by name, defaulting to the most recently loaded. Rejects with an
+   * Error whose message is a rendered diagnostic (caret + hint) on SQL errors.
+   */
+  query(sql: string, options?: { table?: string }): Promise<Result>;
+
+  /** Terminate the worker. */
+  close(): void;
+}

package/index.js

@@ -0,0 +1,181 @@
+// facetful — main-thread API. The engine runs in a dedicated worker; results
+// arrive as transferable column buffers and are wrapped for convenience here.
+//
+//   const db = await Facetful.open({ wasmUrl });
+//   await db.load("plants", await (await fetch("plants.facetful")).arrayBuffer());
+//   const r = await db.query("select country, count(*) n from t group by country order by n desc");
+//   r.column("country")        // Array<string|null> (materialized on demand)
+//   r.columnRaw("n")           // { values: Float64Array, validity } — near-zero copy
+//   [...r.rows()]              // row objects, materialized lazily
+
+export class Facetful {
+  static async open({ wasmUrl, workerUrl, hyparquetUrl } = {}) {
+    // the no-argument form must stay a literal `new Worker(new URL(...))`
+    // expression: bundlers (Vite, webpack) statically analyze exactly that
+    // pattern to compile the worker graph
+    const worker = workerUrl
+      ? new Worker(workerUrl, { type: "module" })
+      : new Worker(new URL("./worker.js", import.meta.url), { type: "module" });
+    const db = new Facetful(worker);
+    await db._call({
+      cmd: "init",
+      wasmUrl: String(wasmUrl ?? new URL("./facetful_wasm.wasm", import.meta.url)),
+      hyparquetUrl,
+    });
+    return db;
+  }
+
+  constructor(worker) {
+    this._worker = worker;
+    this._pending = new Map();
+    this._nextId = 1;
+    worker.onmessage = (e) => {
+      const { id, ...msg } = e.data;
+      const p = this._pending.get(id);
+      if (!p) return;
+      this._pending.delete(id);
+      if (msg.error) p.reject(new Error(msg.error));
+      else p.resolve(msg);
+    };
+  }
+
+  _call(msg, transfer = []) {
+    const id = this._nextId++;
+    return new Promise((resolve, reject) => {
+      this._pending.set(id, { resolve, reject });
+      this._worker.postMessage({ id, ...msg }, transfer);
+    });
+  }
+
+  /** Load a .facetful image (ArrayBuffer). Registered under `name`. */
+  async load(name, buffer) {
+    const { rows } = await this._call({ cmd: "load", name, buffer }, [buffer]);
+    return { name, rows };
+  }
+
+  /**
+   * Open a Parquet file (ArrayBuffer) — the headline path. The image compiled
+   * from it is cached in OPFS keyed by content hash + format version, so a
+   * repeat visit with the same file reopens zero-decode without transcoding.
+   * Returns { rows, source: "cache" | "transcode", ... timings }.
+   */
+  async openParquet(name, buffer, { cacheBytes } = {}) {
+    return this._call(
+      { cmd: "openParquet", name, buffer, cacheBytes: cacheBytes ?? defaultCacheBudget() },
+      [buffer],
+    );
+  }
+
+  /** Transcode-only variant: Parquet -> in-memory table, no OPFS involved. */
+  async loadParquet(name, buffer) {
+    return this._call({ cmd: "loadParquet", name, buffer }, [buffer]);
+  }
+
+  /** Persist a .facetful image into OPFS at `path` (e.g. "facetful/plants.facetful"). */
+  async storeOpfs(path, buffer) {
+    return this._call({ cmd: "storeOpfs", path, buffer }, [buffer]);
+  }
+
+  /**
+   * Open a table over an OPFS file — the spill-over path. Only metadata is
+   * read up front; column segments load on demand into an LRU cache bounded
+   * by `cacheBytes` (default: min(deviceMemory/4, 1GB)).
+   */
+  async loadOpfs(name, path, { cacheBytes } = {}) {
+    const budget = cacheBytes ?? defaultCacheBudget();
+    const { rows, fileLen } = await this._call({ cmd: "loadOpfs", name, path, cacheBytes: budget });
+    return { name, rows, fileLen, cacheBytes: budget };
+  }
+
+  async removeOpfs(path) {
+    return this._call({ cmd: "removeOpfs", path });
+  }
+
+  /** Pre-touch columns into the cache (background; queries interleave). */
+  async warm(cols, { table } = {}) {
+    const { bytes } = await this._call({ cmd: "warm", cols, table });
+    return { bytes };
+  }
+
+  /** { segments, bytes } currently held by a table's segment cache. */
+  async cacheStats({ table } = {}) {
+    const { segments, bytes } = await this._call({ cmd: "cacheStats", table });
+    return { segments, bytes };
+  }
+
+  /** Run SQL. `table` selects a loaded table (defaults to the last loaded). */
+  async query(sql, { table } = {}) {
+    const { result } = await this._call({ cmd: "query", sql, table });
+    return new Result(result);
+  }
+
+  close() {
+    this._worker.terminate();
+  }
+}
+
+// Cache budget when the caller doesn't set one: a quarter of device memory,
+// capped at 1GB. navigator.deviceMemory is Chromium-only; assume 4GB elsewhere.
+function defaultCacheBudget() {
+  const gb = (typeof navigator !== "undefined" && navigator.deviceMemory) || 4;
+  return Math.min((gb / 4) * 1024 ** 3, 1024 ** 3);
+}
+
+const dec = new TextDecoder();
+
+export class Result {
+  constructor(r) {
+    this.columns = r.columns.map((c) => ({ name: c.name, kind: c.kind }));
+    this.rowCount = r.rowCount;
+    this.stats = r.stats;
+    this.elapsedMs = r.elapsedMs;
+    this._cols = r.columns;
+  }
+
+  _find(name) {
+    const c = this._cols.find((c) => c.name === name);
+    if (!c) throw new Error(`no result column '${name}'`);
+    return c;
+  }
+
+  /** Raw buffers: { kind, values | offsets+bytes, validity }. */
+  columnRaw(name) {
+    return this._find(name);
+  }
+
+  /** Materialized values with nulls, in row order. */
+  column(name) {
+    const c = this._find(name);
+    const out = new Array(this.rowCount);
+    for (let i = 0; i < this.rowCount; i++) {
+      out[i] = cellValue(c, i);
+    }
+    return out;
+  }
+
+  *rows() {
+    for (let i = 0; i < this.rowCount; i++) {
+      const o = {};
+      for (const c of this._cols) o[c.name] = cellValue(c, i);
+      yield o;
+    }
+  }
+}
+
+function cellValue(c, i) {
+  if ((c.validity[i >> 3] & (1 << (i & 7))) === 0) return null;
+  switch (c.kind) {
+    case "int":
+      return c.values[i];
+    case "float":
+      return c.values[i];
+    case "bool":
+      return c.values[i] !== 0;
+    case "date": // days since epoch -> "YYYY-MM-DD"
+      return new Date(c.values[i] * 86400000).toISOString().slice(0, 10);
+    case "timestamp": // ms since epoch -> ISO, UTC
+      return new Date(c.values[i]).toISOString().replace("T", " ").slice(0, 19);
+    default:
+      return dec.decode(c.bytes.subarray(c.offsets[i], c.offsets[i + 1]));
+  }
+}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "facetful",
+  "version": "0.1.0",
+  "description": "Tiny columnar SQL engine for browser faceting — Rust/wasm, ~170KB gz, zero dependencies. Open Parquet instantly, facet 1M rows in milliseconds, persist to OPFS.",
+  "license": "MIT",
+  "author": "David Raznick",
+  "type": "module",
+  "main": "./index.js",
+  "types": "./index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./index.js"
+    },
+    "./core": "./core.js",
+    "./worker": "./worker.js",
+    "./parquet": "./parquet.js",
+    "./facetful_wasm.wasm": "./facetful_wasm.wasm"
+  },
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "core.js",
+    "worker.js",
+    "parquet.js",
+    "facetful_wasm.wasm",
+    "README.md",
+    "LICENSE"
+  ],
+  "peerDependencies": {
+    "hyparquet": ">=1.0.0"
+  },
+  "peerDependenciesMeta": {
+    "hyparquet": {
+      "optional": true
+    }
+  },
+  "keywords": [
+    "wasm",
+    "sql",
+    "columnar",
+    "parquet",
+    "opfs",
+    "facets",
+    "analytics",
+    "browser"
+  ],
+  "sideEffects": false,
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kindly/facetful.git",
+    "directory": "js/facetful"
+  },
+  "homepage": "https://github.com/kindly/facetful#readme",
+  "bugs": {
+    "url": "https://github.com/kindly/facetful/issues"
+  }
+}

package/parquet.js

@@ -0,0 +1,104 @@
+// Parquet -> compiler input columns, via a caller-supplied hyparquet module.
+// Shared by the worker (browser) and the node differential test; keeping it
+// environment-free is what makes the "parquet path === image path" check
+// runnable headlessly.
+
+const enc = new TextEncoder();
+
+/** Read a Parquet buffer into compiler input columns. */
+export async function parquetToColumns(h, buffer) {
+  const meta = await h.parquetMetadataAsync(buffer);
+  const rows = Number(meta.num_rows);
+  const root = meta.schema[0];
+  const elems = meta.schema.slice(1, 1 + Number(root.num_children));
+  for (const e of elems) {
+    if (e.num_children) {
+      throw new Error(`nested parquet column '${e.name}' is not supported`);
+    }
+  }
+  const isText = (e) => e.type === "BYTE_ARRAY";
+  const isInt = (e) => e.type === "INT32" || e.type === "INT64" || e.type === "BOOLEAN";
+  const temporalOf = (e) => {
+    const lt = e.logical_type?.type;
+    const ct = e.converted_type;
+    if (lt === "DATE" || ct === "DATE") return "date";
+    if (lt === "TIMESTAMP" || ct === "TIMESTAMP_MILLIS" || ct === "TIMESTAMP_MICROS") {
+      return "timestamp";
+    }
+    return undefined;
+  };
+
+  // builders per column
+  const byName = new Map();
+  for (const e of elems) {
+    const b = { elem: e, validity: new Uint8Array(rows).fill(1), anyNull: false };
+    if (isText(e)) b.strs = new Array(rows).fill("");
+    else b.nums = new Float64Array(rows);
+    byName.set(e.name, b);
+  }
+
+  await h.parquetRead({
+    file: buffer,
+    metadata: meta,
+    columns: elems.map((e) => e.name),
+    onChunk({ columnName, columnData, rowStart }) {
+      const b = byName.get(columnName);
+      if (!b) return;
+      if (b.strs) {
+        for (let i = 0; i < columnData.length; i++) {
+          const v = columnData[i];
+          if (v == null) {
+            b.validity[rowStart + i] = 0;
+            b.anyNull = true;
+          } else {
+            b.strs[rowStart + i] = String(v);
+          }
+        }
+      } else {
+        for (let i = 0; i < columnData.length; i++) {
+          const v = columnData[i];
+          if (v == null) {
+            b.validity[rowStart + i] = 0;
+            b.anyNull = true;
+          } else {
+            // BigInt (INT64), Date (DATE/TIMESTAMP -> ms), boolean all coerce
+            b.nums[rowStart + i] =
+              typeof v === "boolean" ? (v ? 1 : 0) : v instanceof Date ? v.getTime() : Number(v);
+          }
+        }
+      }
+    },
+    onComplete() {},
+  });
+
+  return {
+    rows,
+    columns: elems.map((e) => {
+      const b = byName.get(e.name);
+      const validity = b.anyNull ? b.validity : undefined;
+      if (b.strs) {
+        const offsets = new Uint32Array(rows + 1);
+        const parts = new Array(rows);
+        let total = 0;
+        for (let i = 0; i < rows; i++) {
+          parts[i] = enc.encode(b.strs[i]);
+          total += parts[i].byteLength;
+          offsets[i + 1] = total;
+        }
+        const bytes = new Uint8Array(total);
+        for (let i = 0; i < rows; i++) bytes.set(parts[i], offsets[i]);
+        return { name: e.name, kind: "text", offsets, bytes, validity };
+      }
+      // int-vs-float comes from the parquet physical type, never from values
+      // (a DOUBLE column of integral values must stay Float64, like the CLI);
+      // DATE/TIMESTAMP become real temporal columns (days / ms since epoch)
+      const temporal = temporalOf(e);
+      if (temporal === "date") {
+        // hyparquet yields Date objects at UTC midnight; store days
+        for (let i = 0; i < rows; i++) b.nums[i] = Math.round(b.nums[i] / 86400000);
+      }
+      return { name: e.name, kind: "num", data: b.nums, isInt: isInt(e), temporal, validity };
+    }),
+  };
+}
+

package/worker.js

@@ -0,0 +1,214 @@
+// The engine's dedicated worker: owns the wasm instance, all tables, and the
+// OPFS sync access handles (createSyncAccessHandle only exists in workers).
+// Protocol: {id, cmd, ...} in, {id, ok|error, ...} out; query column buffers
+// are transferred, not cloned.
+
+import { instantiate, transferables, QueryError } from "./core.js";
+import { parquetToColumns } from "./parquet.js";
+
+let engine = null;
+const tables = new Map(); // name -> handle
+let lastTable = null;
+
+// OPFS file registry: the wasm's opfs_read import addresses files by these ids.
+const opfsFiles = new Map(); // fileId -> { handle: FileSystemSyncAccessHandle, path }
+let nextFileId = 1;
+
+function opfsRead(fileId, offset, dest) {
+  const e = opfsFiles.get(fileId);
+  if (!e) return -1;
+  return e.handle.read(dest, { at: offset });
+}
+
+// Sync access handles are exclusive locks; deleting or rewriting a path
+// requires closing ours first. Any table still reading through a closed
+// handle gets clean read errors rather than corrupt data.
+function closeHandlesFor(path) {
+  for (const [id, e] of opfsFiles) {
+    if (e.path === path) {
+      try { e.handle.close(); } catch { /* already closed */ }
+      opfsFiles.delete(id);
+    }
+  }
+}
+
+async function opfsDir(path, create) {
+  let dir = await navigator.storage.getDirectory();
+  const parts = path.split("/").filter(Boolean);
+  const file = parts.pop();
+  for (const p of parts) dir = await dir.getDirectoryHandle(p, { create });
+  return { dir, file };
+}
+
+async function opfsWrite(path, bytes) {
+  closeHandlesFor(path);
+  const { dir, file } = await opfsDir(path, true);
+  const fh = await dir.getFileHandle(file, { create: true });
+  const h = await fh.createSyncAccessHandle();
+  try {
+    h.truncate(0);
+    h.write(bytes, { at: 0 });
+    h.flush();
+  } finally {
+    h.close();
+  }
+}
+
+/** Open an OPFS-backed table lazily (metadata now, segments on demand). */
+async function opfsOpenTable(name, path, cacheBytes) {
+  // sync access handles are exclusive — reuse ours if this path is already
+  // open (positional reads are stateless, so tables can share a handle)
+  let fileId = null;
+  let h = null;
+  for (const [id, e] of opfsFiles) {
+    if (e.path === path) {
+      fileId = id;
+      h = e.handle;
+      break;
+    }
+  }
+  const opened = fileId === null;
+  if (opened) {
+    const { dir, file } = await opfsDir(path, false);
+    const fh = await dir.getFileHandle(file);
+    h = await fh.createSyncAccessHandle();
+    fileId = nextFileId++;
+    opfsFiles.set(fileId, { handle: h, path });
+  }
+  try {
+    const { handle, rows } = engine.openOpfsTable(fileId, h.getSize(), cacheBytes);
+    tables.set(name, handle);
+    lastTable = handle;
+    return { rows, fileLen: h.getSize() };
+  } catch (err) {
+    if (opened) {
+      opfsFiles.delete(fileId);
+      h.close();
+    }
+    throw err;
+  }
+}
+
+// ---------------- Parquet transcode (baseline browser compiler) ------------
+
+let hyparquetUrl = "hyparquet"; // bare specifier for bundlers; override in init
+let hp = null;
+
+async function loadHyparquet() {
+  if (hp) return hp;
+  // literal specifier when unconfigured, so bundlers resolve + code-split the
+  // optional peer dependency; explicit URLs bypass the bundler entirely
+  hp =
+    hyparquetUrl === "hyparquet"
+      ? await import("hyparquet")
+      : await import(/* @vite-ignore */ hyparquetUrl);
+  return hp;
+}
+
+/** Transcode Parquet -> image handle (in wasm memory). */
+async function transcodeParquet(buffer) {
+  const { rows, columns } = await parquetToColumns(await loadHyparquet(), buffer);
+  return { rows, img: engine.compileTable(rows, columns) };
+}
+
+async function sha256hex(buffer) {
+  const d = await crypto.subtle.digest("SHA-256", buffer);
+  return [...new Uint8Array(d)].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+self.onmessage = async (e) => {
+  const { id, cmd } = e.data;
+  const reply = (msg, transfer = []) => postMessage({ id, ...msg }, transfer);
+  try {
+    if (cmd === "init") {
+      if (e.data.hyparquetUrl) hyparquetUrl = e.data.hyparquetUrl;
+      const wasmBytes = await (await fetch(e.data.wasmUrl)).arrayBuffer();
+      engine = await instantiate(wasmBytes, opfsRead);
+      reply({ ok: true });
+    } else if (cmd === "load") {
+      const { handle, rows } = engine.openTable(e.data.buffer);
+      tables.set(e.data.name, handle);
+      lastTable = handle;
+      reply({ ok: true, rows });
+    } else if (cmd === "loadParquet") {
+      // transcode-only path: parquet buffer -> in-memory table
+      const t0 = performance.now();
+      const { rows, img } = await transcodeParquet(e.data.buffer);
+      const { handle } = engine.openImage(img);
+      tables.set(e.data.name, handle);
+      lastTable = handle;
+      reply({ ok: true, rows, transcodeMs: performance.now() - t0 });
+    } else if (cmd === "openParquet") {
+      // cached path: reopen the compiled image from OPFS when the same source
+      // (by content hash) was seen before; otherwise transcode + persist
+      const t0 = performance.now();
+      const hash = await sha256hex(e.data.buffer);
+      const path = `facetful-cache/${hash}-v${engine.w.format_version()}.facetful`;
+      try {
+        const { rows } = await opfsOpenTable(e.data.name, path, e.data.cacheBytes);
+        reply({ ok: true, rows, source: "cache", openMs: performance.now() - t0 });
+        return;
+      } catch {
+        // cache miss (or OPFS unavailable) — fall through to transcode
+      }
+      const { rows, img } = await transcodeParquet(e.data.buffer);
+      const transcodeMs = performance.now() - t0;
+      let cached = false;
+      let imageBytes = null;
+      try {
+        imageBytes = engine.imageBytes(img);
+      } catch { /* copy-out failed: open uncached */ }
+      const { handle } = engine.openImage(img);
+      tables.set(e.data.name, handle);
+      lastTable = handle;
+      if (imageBytes) {
+        try {
+          await opfsWrite(path, imageBytes);
+          cached = true;
+        } catch { /* non-secure context or quota: stay memory-only */ }
+      }
+      reply({ ok: true, rows, source: "transcode", cached, transcodeMs });
+    } else if (cmd === "storeOpfs") {
+      await opfsWrite(e.data.path, new Uint8Array(e.data.buffer));
+      reply({ ok: true, bytes: e.data.buffer.byteLength });
+    } else if (cmd === "loadOpfs") {
+      const { rows, fileLen } = await opfsOpenTable(e.data.name, e.data.path, e.data.cacheBytes);
+      reply({ ok: true, rows, fileLen });
+    } else if (cmd === "removeOpfs") {
+      closeHandlesFor(e.data.path);
+      const { dir, file } = await opfsDir(e.data.path, false);
+      await dir.removeEntry(file);
+      reply({ ok: true });
+    } else if (cmd === "warm") {
+      const handle = e.data.table ? tables.get(e.data.table) : lastTable;
+      if (!handle) throw new Error("no table loaded");
+      let bytes = 0;
+      for (const name of e.data.cols) {
+        const idx = engine.colByName(handle, name);
+        if (idx < 0) throw new Error(`no column '${name}'`);
+        bytes += engine.warmColumn(handle, idx);
+        // yield between columns so queued queries interleave with warming
+        await new Promise((r) => setTimeout(r, 0));
+      }
+      reply({ ok: true, bytes });
+    } else if (cmd === "cacheStats") {
+      const handle = e.data.table ? tables.get(e.data.table) : lastTable;
+      if (!handle) throw new Error("no table loaded");
+      reply({ ok: true, ...engine.cacheStats(handle) });
+    } else if (cmd === "query") {
+      const handle = e.data.table ? tables.get(e.data.table) : lastTable;
+      if (!handle) throw new Error(`no table loaded${e.data.table ? `: '${e.data.table}'` : ""}`);
+      const t0 = performance.now();
+      const result = engine.query(handle, e.data.sql);
+      result.elapsedMs = performance.now() - t0;
+      reply({ ok: true, result }, transferables(result));
+    } else {
+      throw new Error(`unknown command '${cmd}'`);
+    }
+  } catch (err) {
+    reply({
+      error: String(err && err.message ? err.message : err),
+      isQueryError: err instanceof QueryError,
+    });
+  }
+};