@fjader/svelte 0.2.4 → 0.2.5

package/dist/query-engine/duckdb.js

@@ -1,43 +1,47 @@
 /**
  * DuckDB-WASM initialization — replaces mosaic-core's wasmConnector.
  *
- * Returns a Promise<DuckDBConnection> suitable for passing to setQueryContext().
- * Uses jsdelivr CDN bundles (same as mosaic-core's wasmConnector).
+ * initDuckDB() returns a Promise<DuckDBConnection> suitable for passing to
+ * setQueryContext(). It loads the jsDelivr bundles unless the app passes its own.
  *
- * Uses the fast IPC query path: bypasses duckdb-wasm's Arrow JS deserialization
- * and returns raw Arrow IPC bytes. QueryEngine decodes them with flechette for
- * zero-copy typed arrays that go straight to GPU buffers.
+ * connectionFrom() adapts a duckdb-wasm connection the app created itself, for
+ * apps that need the AsyncDuckDB instance (registerFileBuffer, extensions, …).
  *
- * Safe to call during SSR — returns a permanently pending promise on the
- * server. The component re-runs on the client where Worker is available.
+ * Both use the fast IPC query path: bypasses duckdb-wasm's Arrow JS
+ * deserialization and returns raw Arrow IPC bytes. QueryEngine decodes them with
+ * flechette into typed arrays for the GPU buffers. These are views on the IPC
+ * bytes only up to one record batch: DuckDB-WASM writes a batch per DuckDB vector
+ * (~2k rows), and flechette concatenates a column that spans several.
  */
+import { PendingQueryLane } from './sliced-lane';
 /**
- * Initialize DuckDB-WASM and return a connection.
+ * How long the worker runs a sliced statement before serving other messages:
+ * a quick query on another connection waits at most about this long behind a
+ * slow one. Measured in Phase 0: 1 ms at 5 ms, ~50 ms at the default (~100 ms).
+ */
+export const QUERY_POLLING_INTERVAL_MS = 5;
+/**
+ * Adapt a duckdb-wasm connection to the DuckDBConnection QueryEngine expects,
+ * with the fast IPC query path, and sliced lanes: further connections on the
+ * same database (`openSlicedLane`).
  *
- * Usage:
- *   const { queryEngine } = setQueryContext(initDuckDB());
+ *   const db = new duckdb.AsyncDuckDB(logger, worker);
+ *   await db.instantiate(…);
+ *   await db.open({ query: { queryPollingInterval: 5 } }); // short slices, see below
+ *   await db.registerFileBuffer('cached.parquet', bytes);
+ *   initFjader(connectionFrom(await db.connect(), { pollingIntervalMs: 5 }));
+ *
+ * Sliced statements run in slices of the database's `queryPollingInterval`.
+ * At DuckDB's default (~100 ms) a quick query can wait ~50 ms behind a slow
+ * one; open the database with ~5 ms, as `initDuckDB` does. It is a setting of
+ * the database, which this function cannot change or read: say what you set
+ * as `pollingIntervalMs`. Without it, lanes measure their slices and warn if
+ * they run long.
  */
-export function initDuckDB() {
-    if (typeof window === 'undefined') {
-        // SSR: return pending promise. QueryEngine defers until resolved.
-        // Component re-runs on client where the real init happens.
-        return new Promise(() => { });
-    }
-    return doInit();
-}
-async function doInit() {
-    const duckdb = await import('@duckdb/duckdb-wasm');
-    const bundles = duckdb.getJsDelivrBundles();
-    const bundle = await duckdb.selectBundle(bundles);
-    const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: 'text/javascript' }));
-    const worker = new Worker(workerUrl);
-    const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
-    await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
-    URL.revokeObjectURL(workerUrl);
-    const conn = await db.connect();
-    // Wrap the duckdb-wasm connection to expose the fast IPC query path.
+export function connectionFrom(conn, options = {}) {
     // useUnsafe + bindings.runQuery bypasses Arrow JS deserialization entirely,
-    // returning raw IPC bytes that flechette decodes into zero-copy typed arrays.
+    // returning raw IPC bytes that flechette decodes into typed arrays (one copy per
+    // column once a result spans more than one ~2k-row record batch).
     return {
         queryIPC(sql) {
             return conn.useUnsafe((bindings, connId) => bindings.runQuery(connId, sql));
@@ -56,6 +60,54 @@ async function doInit() {
         },
         insertArrowFromIPCStream(data, options) {
             return conn.insertArrowFromIPCStream(data, options);
+        },
+        async openSlicedLane(laneOptions = {}) {
+            // Another connection on the same AsyncDuckDB: one catalog, its own
+            // pending query.
+            const lane = await conn.bindings.connect();
+            return lane.useUnsafe((bindings, connId) => new PendingQueryLane(bindings, connId, () => lane.close(), {
+                pollingIntervalMs: options.pollingIntervalMs,
+                hold: laneOptions.hold
+            }));
         }
     };
 }
+/**
+ * Initialize DuckDB-WASM and return a connection.
+ *
+ * Safe to call during SSR — returns a permanently pending promise on the
+ * server. The component re-runs on the client where Worker is available.
+ *
+ * Usage:
+ *   const { queryEngine } = setQueryContext(initDuckDB());
+ */
+export function initDuckDB(options = {}) {
+    if (typeof window === 'undefined') {
+        // SSR: return pending promise. QueryEngine defers until resolved.
+        // Component re-runs on client where the real init happens.
+        return new Promise(() => { });
+    }
+    return doInit(options);
+}
+async function doInit({ bundles }) {
+    const duckdb = await import('@duckdb/duckdb-wasm');
+    const bundle = await duckdb.selectBundle(bundles ?? duckdb.getJsDelivrBundles());
+    // The worker starts from a blob: URL so a cross-origin (CDN) worker script can
+    // load. importScripts there resolves against the blob: URL, so an app's
+    // relative bundle URL has to be made absolute first.
+    const mainWorker = new URL(bundle.mainWorker, location.href).href;
+    const workerUrl = URL.createObjectURL(new Blob([`importScripts(${JSON.stringify(mainWorker)});`], { type: 'text/javascript' }));
+    const worker = new Worker(workerUrl);
+    const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
+    try {
+        await db.instantiate(new URL(bundle.mainModule, location.href).href, bundle.pthreadWorker && new URL(bundle.pthreadWorker, location.href).href);
+    }
+    finally {
+        URL.revokeObjectURL(workerUrl);
+    }
+    // Short slices for sliced lanes. Only the polling interval: the query
+    // config's other fields decide how big integers, decimals, timestamps and
+    // durations come back, and stay at their defaults.
+    await db.open({ query: { queryPollingInterval: QUERY_POLLING_INTERVAL_MS } });
+    return connectionFrom(await db.connect(), { pollingIntervalMs: QUERY_POLLING_INTERVAL_MS });
+}

package/dist/query-engine/index.d.ts

@@ -11,7 +11,7 @@
  *   context.set('filter-groups', filterGroups);
  */
 import { QueryEngine } from './QueryEngine';
-import type { DuckDBConnection } from './QueryEngine';
+import type { DuckDBConnection, QueryEngineOptions } from './QueryEngine';
 import { FilterGroupRegistry } from './FilterGroupRegistry.svelte';
 /**
  * Create the FilterGroupRegistry for an engine and wire both to the schema
@@ -27,16 +27,19 @@ export declare function createQueryServices(queryEngine: QueryEngine): {
  * Call this synchronously in a component's <script> block.
  * Accepts a Promise<DuckDBConnection> — queries defer until it resolves.
  */
-export declare function setQueryContext(connection: DuckDBConnection | Promise<DuckDBConnection>): {
+export declare function setQueryContext(connection: DuckDBConnection | Promise<DuckDBConnection>, options?: QueryEngineOptions): {
     queryEngine: QueryEngine;
     filterGroups: FilterGroupRegistry;
 };
 /** Get the FilterGroupRegistry from context. */
 export declare function getFilterGroups(): FilterGroupRegistry;
-export { initDuckDB } from './duckdb';
+export { initDuckDB, connectionFrom } from './duckdb';
+export type { SlicedLane } from './sliced-lane';
+export type { InitDuckDBOptions } from './duckdb';
 export { QueryEngine } from './QueryEngine';
-export type { DuckDBConnection, QueryOpts, ColumnInfo, ColumnSchema, JsColumnType, QueryDescriptor, ActiveDim, ActivationSource, QueryEngineOptions } from './QueryEngine';
+export type { DuckDBConnection, QueryOpts, ColumnInfo, ColumnSchema, JsColumnType, QueryDescriptor, ActiveDim, ActivationSource, QueryEngineOptions, StatementClass, StatementRecord } from './QueryEngine';
 export { mapSqlTypeToJs, buildSchemaMap } from './QueryEngine';
+export type { BuildEvent } from './preagg/Preagg';
 export { MarkHandle } from './MarkHandle.svelte';
 export type { QueryParams } from './MarkHandle.svelte';
 export { FilterGroupRegistry } from './FilterGroupRegistry.svelte';

package/dist/query-engine/index.js

@@ -30,8 +30,8 @@ export function createQueryServices(queryEngine) {
  * Call this synchronously in a component's <script> block.
  * Accepts a Promise<DuckDBConnection> — queries defer until it resolves.
  */
-export function setQueryContext(connection) {
-    const { queryEngine, filterGroups } = createQueryServices(new QueryEngine(connection));
+export function setQueryContext(connection, options) {
+    const { queryEngine, filterGroups } = createQueryServices(new QueryEngine(connection, options));
     setContext('query-engine', queryEngine);
     setContext('filter-groups', filterGroups);
     return { queryEngine, filterGroups };
@@ -41,7 +41,7 @@ export function getFilterGroups() {
     return getContext('filter-groups');
 }
 // Re-exports
-export { initDuckDB } from './duckdb';
+export { initDuckDB, connectionFrom } from './duckdb';
 export { QueryEngine } from './QueryEngine';
 export { mapSqlTypeToJs, buildSchemaMap } from './QueryEngine';
 export { MarkHandle } from './MarkHandle.svelte';

package/dist/query-engine/preagg/BuildQueue.d.ts

@@ -29,16 +29,27 @@ export interface QueuedBuild {
 }
 /** What the queue needs from its owner. */
 export interface BuildQueueHost {
-    exec(sql: string): Promise<void>;
-    /** Whether a mark query is running or waiting. */
+    exec(sql: string, signal: AbortSignal): Promise<void>;
+    /**
+     * Whether an interaction statement is on the connection, or a mark query is
+     * waiting to be sent. Background statements (other builds, tooltip writes)
+     * do not count.
+     */
     busy(): boolean;
     /**
-     * Whether a build is still wanted, checked when it would be sent: a mark
-     * re-registered, the view moved or a baked filter changed while it waited.
+     * Why a build is no longer wanted, checked when it would be sent (a mark
+     * re-registered, the view moved or a baked filter changed while it
+     * waited), or null when it still is.
+     */
+    unwanted(item: QueuedBuild): string | null;
+    /** A build is being sent. */
+    started?(item: QueuedBuild): void;
+    /**
+     * A build was dropped before it ran, or cancelled while running, so the
+     * next activation plans it again. `reason` says which: a check that
+     * failed (`unwanted`), 'newer batch', 'cleared' or 'cancelled'.
      */
-    stillWanted(item: QueuedBuild): boolean;
-    /** A queued build was dropped before it ran, so the next activation plans it again. */
-    dropped(item: QueuedBuild): void;
+    dropped(item: QueuedBuild, reason: string): void;
     built(item: QueuedBuild): void;
     failed(item: QueuedBuild, error: unknown): void;
 }
@@ -52,6 +63,8 @@ export declare class BuildQueue {
      *   behind the build instead.
      */
     constructor(host: BuildQueueHost, maxWaitMs: number);
+    /** The current hover batch's number, for build records. */
+    get batch(): number;
     /** Run a hover's activations as one batch. Nested calls join the outer batch. */
     runBatch(activations: () => void): void;
     /** Start a batch unless one is running: an activation outside runBatch() is a batch of one. */
@@ -59,11 +72,12 @@ export declare class BuildQueue {
     /** Queue a build in the current batch. Settles when it has run or been dropped. */
     push(build: Omit<QueuedBuild, 'batch' | 'queuedAt' | 'done'>): Promise<void>;
     /**
-     * Send the next queued build if nothing else is on the connection: no build
-     * running, and no mark query waiting or running. Called when a build is
-     * queued, when one finishes, and when a mark query settles. Queries without
-     * a caller id (identity lookups, describe, ad-hoc queries) do not hold builds
-     * back and can wait behind a running one; none is on an interaction path.
+     * Send the next queued build if nothing someone waits for is on the
+     * connection: no build running, no interaction statement in flight, and no
+     * mark query waiting. Called when a build is queued, when one finishes, when
+     * a mark query settles, and when the last interaction statement settles.
+     * Background statements (tooltip-table writes, cleanup) do not hold builds
+     * back.
      *
      * Mark queries hold a build back for at most the wait limit. A drag that
      * starts before its source's tables are built reads the base table, each
@@ -71,7 +85,13 @@ export declare class BuildQueue {
      * never run; past the limit it is sent anyway, and one frame waits for it.
      */
     pump(): void;
-    /** Drop every queued build (a data reload); a running build still finishes. */
+    /**
+     * Cancel the running build if `unwanted` says so. On a lane it is
+     * interrupted and rolls back (no table); on a shared connection it cannot
+     * be, and finishes, but is not registered.
+     */
+    abortRunning(unwanted: (item: QueuedBuild) => boolean): void;
+    /** Drop every queued build and cancel the running one (a data reload). */
     clear(): void;
     /** Stop for good: settle every waiting activation and send nothing more. */
     destroy(): void;

package/dist/query-engine/preagg/BuildQueue.js

@@ -19,6 +19,8 @@ export class BuildQueue {
     /** Waiting builds, oldest first. */
     #queue = [];
     #running = false;
+    /** The build being sent, and how to cancel it. */
+    #current = null;
     #destroyed = false;
     /** Pumps the queue when the oldest build's wait runs out, if no query does first. */
     #waitTimer = null;
@@ -36,6 +38,10 @@ export class BuildQueue {
         this.#host = host;
         this.#maxWaitMs = maxWaitMs;
     }
+    /** The current hover batch's number, for build records. */
+    get batch() {
+        return this.#batch;
+    }
     /** Run a hover's activations as one batch. Nested calls join the outer batch. */
     runBatch(activations) {
         if (this.#inBatch) {
@@ -63,11 +69,12 @@ export class BuildQueue {
         });
     }
     /**
-     * Send the next queued build if nothing else is on the connection: no build
-     * running, and no mark query waiting or running. Called when a build is
-     * queued, when one finishes, and when a mark query settles. Queries without
-     * a caller id (identity lookups, describe, ad-hoc queries) do not hold builds
-     * back and can wait behind a running one; none is on an interaction path.
+     * Send the next queued build if nothing someone waits for is on the
+     * connection: no build running, no interaction statement in flight, and no
+     * mark query waiting. Called when a build is queued, when one finishes, when
+     * a mark query settles, and when the last interaction statement settles.
+     * Background statements (tooltip-table writes, cleanup) do not hold builds
+     * back.
      *
      * Mark queries hold a build back for at most the wait limit. A drag that
      * starts before its source's tables are built reads the base table, each
@@ -90,32 +97,61 @@ export class BuildQueue {
                 }
             }
             const item = this.#queue.shift();
-            if (!this.#host.stillWanted(item)) {
-                this.#drop(item);
+            const reason = this.#host.unwanted(item);
+            if (reason !== null) {
+                this.#drop(item, reason);
                 continue;
             }
             this.#running = true;
+            this.#host.started?.(item);
+            const abort = new AbortController();
+            this.#current = { item, abort };
             this.#host
-                .exec(item.createSQL)
-                .then(() => this.#host.built(item))
-                .catch((e) => this.#host.failed(item, e))
+                .exec(item.createSQL, abort.signal)
+                .then(() => {
+                // Cancelled but finished anyway (the cancel lost the race): the
+                // caller gave it up, so it does not land.
+                if (abort.signal.aborted)
+                    this.#host.dropped(item, 'cancelled');
+                else
+                    this.#host.built(item);
+            })
+                .catch((e) => {
+                if (abort.signal.aborted)
+                    this.#host.dropped(item, 'cancelled');
+                else
+                    this.#host.failed(item, e);
+            })
                 .finally(() => {
                 this.#running = false;
+                this.#current = null;
                 item.done();
                 this.pump();
             });
             return;
         }
     }
-    /** Drop every queued build (a data reload); a running build still finishes. */
+    /**
+     * Cancel the running build if `unwanted` says so. On a lane it is
+     * interrupted and rolls back (no table); on a shared connection it cannot
+     * be, and finishes, but is not registered.
+     */
+    abortRunning(unwanted) {
+        const current = this.#current;
+        if (current && unwanted(current.item))
+            current.abort.abort();
+    }
+    /** Drop every queued build and cancel the running one (a data reload). */
     clear() {
         for (const item of this.#queue)
-            this.#drop(item);
+            this.#drop(item, 'cleared');
         this.#queue = [];
+        this.abortRunning(() => true);
     }
     /** Stop for good: settle every waiting activation and send nothing more. */
     destroy() {
         this.#destroyed = true;
+        this.abortRunning(() => true);
         if (this.#waitTimer)
             clearTimeout(this.#waitTimer);
         this.#waitTimer = null;
@@ -130,12 +166,12 @@ export class BuildQueue {
             if (item.batch === this.#batch)
                 keep.push(item);
             else
-                this.#drop(item);
+                this.#drop(item, 'newer batch');
         }
         this.#queue = keep;
     }
-    #drop(item) {
-        this.#host.dropped(item);
+    #drop(item, reason) {
+        this.#host.dropped(item, reason);
         item.done();
     }
 }

package/dist/query-engine/preagg/Preagg.d.ts

@@ -25,18 +25,56 @@ export interface PreaggMark {
     handle: MarkHandle;
     filterGroup: FilterGroup;
 }
+/**
+ * One step of a preagg build, for `QueryEngine.observeBuilds` (debugging and
+ * measurement). In order for one build: planned, queued, started, then built,
+ * failed or dropped. A planning that builds nothing is `skipped`.
+ */
+export interface BuildEvent {
+    kind: 'skipped' | 'planned' | 'queued' | 'started' | 'built' | 'failed' | 'dropped';
+    markId: string;
+    sourceId: string;
+    /** The preagg table, once planned. */
+    table: string | null;
+    /** performance.now(). */
+    time: number;
+    /** The hover batch the build was queued in. */
+    batch?: number;
+    /**
+     * skipped: 'unplannable' or 'too large'. dropped: the check that failed
+     * ('unregistered', 'inactive', 'superseded', 'parts changed', 'bins do not
+     * fit the view', 'baked clauses changed'), or 'newer batch', 'cleared',
+     * 'cancelled'.
+     */
+    reason?: string;
+    /** The values the failed check compared (see #dropDetail), or the size guard's. */
+    detail?: Record<string, unknown>;
+    /** failed: the error. */
+    error?: unknown;
+}
 /** What Preagg needs from the engine. */
 export interface PreaggHost {
-    exec(sql: string): Promise<void>;
+    exec(sql: string, signal?: AbortSignal): Promise<void>;
     query(sql: string, opts: {
         cache: boolean;
     }): Promise<Table>;
     /** Whether a mark query is running or waiting. */
     busy(): boolean;
+    /**
+     * Whether background statements run sliced on their own connection (a
+     * lane), where a build no longer holds up interaction.
+     */
+    sliced(): boolean;
     identity: PreaggIdentity;
     /** The mark's current registration, which a re-setup replaces. */
     mark(markId: string): PreaggMark | undefined;
     marks(): Iterable<[string, PreaggMark]>;
+    /**
+     * Whether anything listens to onBuild. Checked first, so with no
+     * listener no record is built.
+     */
+    observingBuilds?(): boolean;
+    onBuild?(event: BuildEvent): void;
 }
 export declare class Preagg {
     #private;
@@ -97,13 +135,18 @@ export declare class Preagg {
      * A data reload: drop every queued build, every mark's tables and the tables
      * themselves (CREATE TABLE IF NOT EXISTS would otherwise keep serving
      * pre-reload aggregates), and the row counts.
+     *
+     * The running build is cancelled too (`clear`). On a lane it runs beside the
+     * reload, reading the old rows through its snapshot; should it finish anyway
+     * (the cancel lost the race), `built` finds its mark gone from `#marks` and
+     * does not register it, and the DROP below follows it on the same lane.
      */
     reset(): void;
     /**
      * The engine is going: settle waiting activations and drop this session's
      * tables. Resolves once the drops have run; call it while the engine can
-     * still exec. A build already running is ahead of the drops on the one
-     * connection, so its table is dropped too.
+     * still exec. A build already running is cancelled, and is ahead of the
+     * drops on its connection either way, so its table is dropped too.
      */
     destroy(): Promise<void>;
 }