@fjader/svelte 0.2.3 → 0.2.5

package/dist/query-engine/QueryEngine.js

@@ -21,6 +21,8 @@ import { tableFromIPC } from '@uwdata/flechette';
 import { assembleQuery, castNumeric, splitWindowSelect } from '../components/marks/lifecycle/query-builder';
 import { Preagg } from './preagg/Preagg';
 import { MarkHandle } from './MarkHandle.svelte';
+import { LaneSlot } from './sliced-lane';
+import { SourceRouting, sourceDDL } from './source-routing';
 import { CategoricalIdentity } from './categorical-identity';
 import { NO_ROUTE } from './CrossfilterSelection.svelte';
 /**
@@ -31,6 +33,33 @@ import { NO_ROUTE } from './CrossfilterSelection.svelte';
  * build instead.
  */
 const BUILD_MAX_WAIT_MS = 100;
+/**
+ * Sliced lanes for interaction reads of unknown or slow sources. A source
+ * sticks to one; more sources than lanes share them (reads of one source are
+ * serial anyway).
+ */
+const SOURCE_LANES = 4;
+/**
+ * How long the main connection must stay idle before a lane runs its next
+ * slice. Longer than the JS gap between two statements of a dependent chain
+ * (one settles, the next is built and sent), so the chain runs unbroken
+ * instead of letting one slice in at every gap.
+ */
+const MAIN_QUIET_MS = 2;
+/**
+ * While the main connection works, all lanes together get one slice per this
+ * many ms: half the worker at 5 ms slices. A slow source keeps a minimum
+ * rate, and a fast query waits at most one slice for it. One budget for
+ * every lane, the background lane too: a budget per lane would let five busy
+ * lanes take the whole worker between them.
+ *
+ * Measured beside a slow chart during a brush (/test/query-load?slow=1,
+ * scripted drag, two runs each): the fast charts' p90 stayed at ~20 ms and
+ * their updates the same at 25, 15 and 10 ms (their cost follows the slow
+ * chart's large results decoding, not the share), while the slow chart's
+ * data age fell from ~500 to ~375 to ~345 ms median.
+ */
+const SLICE_SHARE_MS = 10;
 // DESCRIBE reports canonical type names, so these match exactly. Anything else (VARCHAR,
 // ENUM, lists, STRUCT, INTERVAL, extension types) is a string, which is never cast to REAL.
 const NUMBER_TYPES = new Set([
@@ -85,11 +114,41 @@ export function buildSchemaMap(columns) {
 // ---------------------------------------------------------------------------
 // QueryEngine
 // ---------------------------------------------------------------------------
+/**
+ * How a mark's data read is placed. Routed by the table it reads
+ * (SourceRouting): the preagg table when one answers, else the mark's BASE
+ * table (its identity view reads the same rows, and its range and stats reads
+ * name the base table: one record per source). `slow` (holdsEntrance={false})
+ * applies to the base query only: a preagg read is ~1 ms, and forced onto a
+ * lane during a brush it would cost ~20 ms of round trips in exactly the case
+ * preagg exists to make fast.
+ */
+export function markReadOptions(markId, descriptor, preaggTable) {
+    return {
+        callerId: markId,
+        source: preaggTable ?? descriptor.baseTable,
+        data: true,
+        slow: !preaggTable && descriptor.slow
+    };
+}
+/**
+ * DuckDB's catalog error for a table or view this connection cannot see.
+ * Matched by its wording, which an upgrade could change: the integration
+ * test missing-relation.integration.test.ts checks it against real DuckDB.
+ */
+export function isMissingRelation(e) {
+    const message = e instanceof Error ? e.message : String(e);
+    return /Catalog Error: (Table|View) with name .+ does not exist/i.test(message);
+}
 export class QueryEngine {
     #conn = null;
     #connPromise = null;
     #inflight = new Map();
     #pending = new Map();
+    /** Interaction statements handed to the connection and not yet settled (see #send). */
+    #interactionInFlight = 0;
+    /** Debug listeners for every statement sent (observeStatements). */
+    #statementObservers = new Set();
     #describeCache = new Map();
     #marks = new Map();
     #destroyed = false;
@@ -97,7 +156,15 @@ export class QueryEngine {
     #preagg;
     /** When a filter last changed, in any group (performance.now()). */
     #lastFilterChangeAt = 0;
-    /** LRU query cache: SQL string → Arrow Table (or Promise<Table> while inflight). */
+    /**
+     * LRU query cache: Arrow Table (or Promise<Table> while inflight), keyed by
+     * `#cacheKey`, the SQL text and the dims' version. A mark's query reads dim
+     * tables (dense padding pads every category the dim holds), and dims grow
+     * after the first queries run: a table folding in, a topN mark reserving its
+     * Other. The text does not change when they do, so keyed by text alone the
+     * cache served a result from before, with fewer rows: the row set changed
+     * between updates and the bars snapped instead of morphing.
+     */
     #queryCache = lruCache({ max: 1000, ttl: 3 * 60 * 60 * 1000 });
     /**
      * Per-instance suffix for every table and view this engine creates.
@@ -118,19 +185,44 @@ export class QueryEngine {
                 rows.push(table.get(i));
             }
             return rows;
+        },
+        // Seen created as a table through the engine, or listed in the catalog
+        // (which lists tables, not views).
+        isBaseTable: (table) => this.#known.get(table) === 'table' || !!this.#catalog?.has(table),
+        observing: () => this.#identityObservers.size > 0,
+        onRequest: (table, fields, time) => {
+            this.#notifyIdentity({ kind: 'request', table, fields, time });
+        },
+        onDrain: (table, fields, time) => {
+            this.#notifyIdentity({ kind: 'drain', table, fields, time });
+        },
+        execBatch: async (statements) => {
+            const table = await this.execBatch(statements);
+            const rows = [];
+            for (let i = 0; i < table.numRows; i++) {
+                rows.push(table.get(i));
+            }
+            return rows;
         }
     }, this.#session);
     /** Notified after clearQueryCache() drops the identity dims and views. */
     #identityResetListeners = new Set();
     constructor(connection, options = {}) {
         this.#preagg = new Preagg({
-            exec: (sql) => this.exec(sql),
-            query: (sql, opts) => this.query(sql, opts),
-            busy: () => this.#inflight.size > 0 || this.#pending.size > 0,
+            exec: (sql, signal) => this.exec(sql, { class: 'background', signal }),
+            query: (sql, opts) => this.query(sql, { class: 'background', ...opts }),
+            // A statement someone waits for is on the connection, or a mark
+            // query is waiting to be sent.
+            busy: () => this.#interactionInFlight > 0 || this.#pending.size > 0,
+            sliced: () => this.#slicedReady(),
             identity: this.#identity,
             mark: (markId) => this.#marks.get(markId),
-            marks: () => this.#marks
+            marks: () => this.#marks,
+            observingBuilds: () => this.#buildObservers.size > 0,
+            onBuild: (event) => this.#notifyBuild(event)
         }, options.buildMaxWaitMs ?? BUILD_MAX_WAIT_MS);
+        this.#sliceShareMs = options.sliceShareMs ?? SLICE_SHARE_MS;
+        this.#unbatch = options.unbatch ?? false;
         if (connection instanceof Promise) {
             this.#connPromise = connection;
             connection.then((c) => {
@@ -285,18 +377,316 @@ export class QueryEngine {
             }
         }
     }
+    /**
+     * The one place a statement reaches the connection. Everything that talks to
+     * DuckDB goes through here: queries, statements, DESCRIBE, Arrow inserts.
+     *
+     * What attaches here: the interaction count builds yield to, and statement
+     * observers (debugging, measurement). Later: routing a class to a connection,
+     * and per-connection queues.
+     */
+    async #send(kind, sql, place, how) {
+        const { run, runSliced, signal } = how;
+        if (!this.#conn)
+            await this.#ensureConnection();
+        const cls = place.class;
+        // Awaited only while a lane is opening (LaneSlot.get): once its state is
+        // known the choice is made in this tick.
+        const slot = runSliced ? this.#slotFor(place) : null;
+        const got = slot ? slot.get() : null;
+        const lane = got instanceof Promise ? await got : got;
+        // The main connection cannot interrupt a statement once sent: a signal
+        // only stops one that has not gone yet. A lane cancels (SlicedLane).
+        if (!lane && signal?.aborted)
+            throw new DOMException('Statement aborted', 'AbortError');
+        const conn = this.#conn;
+        const interaction = cls === 'interaction';
+        if (interaction)
+            this.#interactionInFlight++;
+        const start = performance.now();
+        let error;
+        // Paired with #mainIdle in the finally below.
+        if (!lane)
+            this.#mainBusy();
+        try {
+            return lane ? await runSliced(lane) : await run(conn);
+        }
+        catch (e) {
+            error = e;
+            throw e;
+        }
+        finally {
+            if (this.#statementObservers.size > 0) {
+                const record = {
+                    sql,
+                    kind,
+                    class: cls,
+                    connection: lane ? 'lane' : 'main',
+                    count: how.count,
+                    source: place.source,
+                    requested: place.requestedAt,
+                    caller: place.callerId,
+                    start,
+                    end: performance.now(),
+                    error
+                };
+                // A listener is for debugging: one that throws must not fail the
+                // statement, nor skip the in-flight count below.
+                for (const observe of this.#statementObservers) {
+                    try {
+                        observe(record);
+                    }
+                    catch (e) {
+                        console.error('[QueryEngine] statement observer threw:', e);
+                    }
+                }
+            }
+            if (!lane)
+                this.#mainIdle();
+            // The last interaction statement settling frees the connection for a build.
+            if (interaction && --this.#interactionInFlight === 0)
+                this.#preagg.pump();
+        }
+    }
     /**
      * Execute a SQL query and return a flechette Table.
      * Uses the fast IPC path (queryIPC → tableFromIPC) when available,
      * falling back to the standard conn.query() for mocks/testing.
      */
-    async #queryRaw(sql) {
+    #queryRaw(sql, place, kind = 'query', signal) {
+        let sliced = false;
+        const read = this.#send(kind, sql, place, {
+            run: async (conn) => {
+                if (!conn.queryIPC)
+                    return conn.query(sql);
+                const t0 = performance.now();
+                const bytes = await conn.queryIPC(sql);
+                const ms = performance.now() - t0;
+                const table = tableFromIPC(bytes);
+                if (this.#notesCost(place))
+                    this.#routing.noteMainRead(place.source, ms, table.numRows);
+                return table;
+            },
+            runSliced: async (lane) => {
+                sliced = true;
+                const result = await lane.query(sql, signal);
+                const table = tableFromIPC(result.ipc);
+                if (this.#notesCost(place)) {
+                    this.#routing.noteLaneRead(place.source, result.polls, result.execMs, table.numRows);
+                }
+                return table;
+            },
+            signal
+        });
+        // Safety net for a TEMP source made where the engine could not see it
+        // (on the app's connection, which is the main one): a lane does not
+        // have it. Once found missing there, it stays on the main connection
+        // for good, and this read is sent there. Local is the safe direction:
+        // a race (a lane read during a drop and re-create made outside the
+        // engine) can only make a source less sliced, which is how every
+        // source read before lanes.
+        return read.catch((e) => {
+            const source = place.source;
+            if (!sliced || !source || !isMissingRelation(e))
+                throw e;
+            this.#localSources.add(source);
+            return this.#queryRaw(sql, place, kind, signal);
+        });
+    }
+    /**
+     * The lane a statement runs on, or null for the main connection:
+     * - connection-local state, or a statement with no sliced form: main;
+     * - background: the background lane (preagg builds and their reads);
+     * - interaction naming a source: by the source's cost (SourceRouting),
+     *   unknown or slow on that source's lane, known-fast on main.
+     * Without the capability every slot answers null: one connection.
+     */
+    #slotFor(place) {
+        if (place.connectionLocal)
+            return null;
+        if (place.class === 'background')
+            return this.#backgroundLane;
+        if (!place.source || !this.#conn?.openSlicedLane)
+            return null;
+        if (this.#localSources.has(place.source))
+            return null;
+        const i = place.slow ? this.#routing.laneOf(place.source) : this.#routing.route(place.source);
+        return i == null ? null : this.#sourceLanes[i];
+    }
+    /** Whether a read's cost goes into its source's record: interaction data reads. */
+    #notesCost(place) {
+        // A read known to be slow says nothing about its source: a slow mark's
+        // heavy query would otherwise turn a fast table slow for every mark.
+        return place.class === 'interaction' && !!place.data && !!place.source && !place.slow;
+    }
+    /**
+     * Whether background statements run sliced, on their own connection, so
+     * they no longer hold up interaction. Asks for the lane, so the first ask
+     * opens it; false until it is open.
+     */
+    #slicedReady() {
+        this.#backgroundLane.get();
+        return this.#backgroundLane.isOpen;
+    }
+    #openLaneFn = () => {
         const conn = this.#conn;
-        if (conn.queryIPC) {
-            const ipc = await conn.queryIPC(sql);
-            return tableFromIPC(ipc);
+        if (!conn?.openSlicedLane || this.#destroyed)
+            return null;
+        return conn.openSlicedLane({ hold: this.#sliceGrant });
+    };
+    // --- Holding lanes while the main connection works ---
+    //
+    // A lane's slice is a message at the back of the worker's queue, so it
+    // runs between two main-connection statements; in a dependent chain (a
+    // mark's setup: dims, views, then data) every step then waits a slice.
+    // So while the main connection works, lanes share a budget of slices
+    // (SLICE_SHARE_MS, across all of them); once it has been idle for
+    // MAIN_QUIET_MS they run freely. Counted here, per connection: a slow
+    // source's reads are interaction class too, and a lane waiting on
+    // #interactionInFlight would wait for itself.
+    /** Statements on the main connection, sent and not settled. */
+    #mainInFlight = 0;
+    /** When the main connection last went idle. */
+    #mainIdleAt = 0;
+    #sliceShareMs;
+    #unbatch;
+    /** When any lane last took a slice from the share. */
+    #lastSliceAt = -Infinity;
+    /** Lanes waiting for a slice, first come first served. */
+    #sliceWaiters = [];
+    #shareTimer = null;
+    #quietTimer = null;
+    #mainBusy() {
+        this.#mainInFlight++;
+        if (this.#quietTimer) {
+            clearTimeout(this.#quietTimer);
+            this.#quietTimer = null;
         }
-        return conn.query(sql);
+    }
+    #mainIdle() {
+        if (--this.#mainInFlight > 0)
+            return;
+        this.#mainIdleAt = performance.now();
+        if (this.#sliceWaiters.length > 0)
+            this.#armQuiet();
+    }
+    #mainQuietNow() {
+        return this.#mainInFlight === 0 && performance.now() - this.#mainIdleAt >= MAIN_QUIET_MS;
+    }
+    /** Once the main connection has stayed idle long enough, every waiting lane goes. */
+    #armQuiet() {
+        if (this.#quietTimer)
+            clearTimeout(this.#quietTimer);
+        const left = MAIN_QUIET_MS - (performance.now() - this.#mainIdleAt);
+        this.#quietTimer = setTimeout(() => {
+            this.#quietTimer = null;
+            if (this.#mainInFlight > 0)
+                return;
+            const waiting = this.#sliceWaiters;
+            this.#sliceWaiters = [];
+            for (const go of waiting)
+                go();
+        }, Math.max(0, left));
+    }
+    /** The next waiting lane gets the share's next slice. */
+    #armShare() {
+        if (this.#shareTimer)
+            return;
+        const left = this.#lastSliceAt + this.#sliceShareMs - performance.now();
+        this.#shareTimer = setTimeout(() => {
+            this.#shareTimer = null;
+            const go = this.#sliceWaiters.shift();
+            if (!go)
+                return;
+            this.#lastSliceAt = performance.now();
+            go();
+            if (this.#sliceWaiters.length > 0)
+                this.#armShare();
+        }, Math.max(0, left));
+    }
+    /** LaneHold: may a lane run its next slice now? */
+    #sliceGrant = () => {
+        if (this.#mainQuietNow())
+            return null;
+        const now = performance.now();
+        if (this.#sliceWaiters.length === 0 && now - this.#lastSliceAt >= this.#sliceShareMs) {
+            this.#lastSliceAt = now;
+            return null;
+        }
+        return new Promise((resolve) => {
+            this.#sliceWaiters.push(resolve);
+            this.#armShare();
+            if (this.#mainInFlight === 0)
+                this.#armQuiet();
+        });
+    };
+    #laneFailed = (e) => {
+        if (this.#warnedLaneFailed)
+            return;
+        this.#warnedLaneFailed = true;
+        console.warn('[QueryEngine] could not open a sliced lane; its statements stay on the main connection:', e);
+    };
+    #warnedLaneFailed = false;
+    /** Background statements: preagg builds, their counts and drops. */
+    #backgroundLane = new LaneSlot(this.#openLaneFn, this.#laneFailed);
+    /** Interaction reads of unknown and slow sources, a source sticking to one. */
+    #sourceLanes = Array.from({ length: SOURCE_LANES }, () => new LaneSlot(this.#openLaneFn, this.#laneFailed));
+    #routing = new SourceRouting(SOURCE_LANES);
+    /**
+     * Debug: every categorical identity request (a mark's ensureColumns call),
+     * with its table, fields and time, before deduplication, and every drain
+     * start (a table's dims beginning to build, with the fields it found
+     * pending). Returns the unsubscribe; costs nothing while no one listens, and
+     * a listener that throws is logged without failing identity work.
+     */
+    observeIdentityRequests(listener) {
+        this.#identityObservers.add(listener);
+        return () => this.#identityObservers.delete(listener);
+    }
+    #identityObservers = new Set();
+    /**
+     * Debug: every step of every preagg build (BuildEvent): planned or
+     * skipped (with the size guard's numbers), queued (with its hover batch),
+     * started, then built, failed or dropped (with the check that failed and
+     * the values it compared). Returns the unsubscribe; costs nothing while no
+     * one listens, and a listener that throws is logged without failing the
+     * build.
+     */
+    observeBuilds(listener) {
+        this.#buildObservers.add(listener);
+        return () => this.#buildObservers.delete(listener);
+    }
+    #buildObservers = new Set();
+    /** Each listener in its own try/catch: a debugging hook must not fail a build. */
+    #notifyBuild(event) {
+        for (const observe of this.#buildObservers) {
+            try {
+                observe(event);
+            }
+            catch (e) {
+                console.error('[QueryEngine] build observer threw:', e);
+            }
+        }
+    }
+    /** Each listener in its own try/catch: a debugging hook must not fail identity work. */
+    #notifyIdentity(request) {
+        for (const observe of this.#identityObservers) {
+            try {
+                observe(request);
+            }
+            catch (e) {
+                console.error('[QueryEngine] identity observer threw:', e);
+            }
+        }
+    }
+    /**
+     * Listen to every statement sent to the connection, as it settles. For
+     * debugging and measurement; returns the unsubscribe. Costs nothing while
+     * no one listens.
+     */
+    observeStatements(listener) {
+        this.#statementObservers.add(listener);
+        return () => this.#statementObservers.delete(listener);
     }
     /** Resolve #conn from #connPromise if needed. No-op when already resolved. */
     #ensureConnection() {
@@ -324,12 +714,21 @@ export class QueryEngine {
         if (!this.#conn)
             await this.#ensureConnection();
         const sqlStr = typeof sql === 'string' ? sql : sql.toString();
+        const place = {
+            class: opts?.class ?? 'interaction',
+            connectionLocal: opts?.connectionLocal,
+            source: opts?.source,
+            data: opts?.data,
+            requestedAt: performance.now(),
+            callerId: opts?.callerId,
+            slow: opts?.slow
+        };
         // Cache check: return cached result immediately, bypassing
         // inflight/pending bookkeeping entirely.
         // Opt out with cache: false for queries against volatile temp tables.
         const useCache = opts?.cache !== false;
         if (useCache) {
-            const cached = this.#queryCache.get(sqlStr);
+            const cached = this.#queryCache.get(this.#cacheKey(sqlStr));
             if (cached !== undefined) {
                 return cached instanceof Promise ? cached : Promise.resolve(cached);
             }
@@ -348,27 +747,66 @@ export class QueryEngine {
                 }
                 // Store as pending — will be executed when inflight completes
                 return new Promise((resolve, reject) => {
-                    this.#pending.set(callerId, { sql: sqlStr, resolve, reject, cache: useCache });
+                    this.#pending.set(callerId, {
+                        sql: sqlStr,
+                        resolve,
+                        reject,
+                        cache: useCache,
+                        place
+                    });
                 });
             }
-            return this.#executeForCaller(callerId, sqlStr, opts.signal, useCache);
+            return this.#executeForCaller(callerId, sqlStr, opts.signal, useCache, place);
+        }
+        // No callerId, cache asked for: the first caller's promise goes in the
+        // cache, so a concurrent identical query (two marks asking the same
+        // range) shares it, and a later one reads the result. A cached hit
+        // already returned above.
+        if (opts?.cache === true) {
+            if (opts.signal?.aborted)
+                throw new DOMException('Query aborted', 'AbortError');
+            const key = this.#cacheKey(sqlStr);
+            const promise = this.#queryRaw(sqlStr, place, 'query', opts.signal);
+            this.#queryCache.set(key, promise);
+            let result;
+            try {
+                result = await promise;
+                this.#queryCache.set(key, result);
+            }
+            catch (e) {
+                this.#queryCache.delete(key);
+                throw e;
+            }
+            // Aborted while running: the result stays cached for others (it is
+            // correct), but this caller asked not to receive it.
+            if (opts.signal?.aborted)
+                throw new DOMException('Query aborted', 'AbortError');
+            return result;
         }
         // No callerId: direct execution, still respect external signal
         if (opts?.signal) {
             if (opts.signal.aborted) {
                 throw new DOMException('Query aborted', 'AbortError');
             }
-            const result = await this.#queryRaw(sqlStr);
+            const result = await this.#queryRaw(sqlStr, place, 'query', opts.signal);
             if (opts.signal.aborted) {
                 throw new DOMException('Query aborted', 'AbortError');
             }
             return result;
         }
-        return this.#queryRaw(sqlStr);
+        return this.#queryRaw(sqlStr, place);
     }
-    async #executeForCaller(callerId, sqlStr, signal, cache = true) {
+    /**
+     * The cache key for a query: its text and the dims' version, so a result read
+     * against a dim that has since grown is not served again.
+     */
+    #cacheKey(sqlStr) {
+        return `${this.#identity.version}\u0000${sqlStr}`;
+    }
+    async #executeForCaller(callerId, sqlStr, signal, cache = true, place = { class: 'interaction' }) {
         const controller = new AbortController();
         this.#inflight.set(callerId, controller);
+        let key = null;
         if (signal) {
             signal.addEventListener('abort', () => controller.abort(), { once: true });
         }
@@ -376,24 +814,28 @@ export class QueryEngine {
             if (controller.signal.aborted) {
                 throw new DOMException('Query superseded', 'AbortError');
             }
+            // Keyed when it runs, not when it was asked for: a pending query runs
+            // later, against whatever the dims hold by then.
+            key = this.#cacheKey(sqlStr);
             // Cache the Promise so concurrent identical queries share one inflight
-            const queryPromise = this.#queryRaw(sqlStr);
+            const queryPromise = this.#queryRaw(sqlStr, place, 'query', controller.signal);
             if (cache)
-                this.#queryCache.set(sqlStr, queryPromise);
+                this.#queryCache.set(key, queryPromise);
             const result = await queryPromise;
             // Replace Promise with resolved data for future lookups
             if (cache)
-                this.#queryCache.set(sqlStr, result);
+                this.#queryCache.set(key, result);
             if (controller.signal.aborted) {
                 // Don't cache results from aborted queries
-                this.#queryCache.delete(sqlStr);
+                this.#queryCache.delete(key);
                 throw new DOMException('Query superseded', 'AbortError');
             }
             return result;
         }
         catch (e) {
             // Remove failed query from cache so we don't serve a rejected Promise
-            this.#queryCache.delete(sqlStr);
+            if (key !== null)
+                this.#queryCache.delete(key);
             throw e;
         }
         finally {
@@ -406,7 +848,7 @@ export class QueryEngine {
             const pending = this.#pending.get(callerId);
             if (pending) {
                 this.#pending.delete(callerId);
-                this.#executeForCaller(callerId, pending.sql, undefined, pending.cache)
+                this.#executeForCaller(callerId, pending.sql, undefined, pending.cache, pending.place)
                     .then(pending.resolve)
                     .catch(pending.reject);
             }
@@ -414,28 +856,117 @@ export class QueryEngine {
             this.#preagg.pump();
         }
     }
+    /**
+     * Run several statements as ONE message on the main connection, returning
+     * the last statement's rows. Each statement commits on its own and
+     * execution stops at the first failure, as when sent one by one (measured
+     * on DuckDB-WASM, /test/query-slicing §13); the error does not say which
+     * statement failed.
+     *
+     * For statements on tables the main connection owns (fjader's identity
+     * dims, present tables, views): one connection per table. Deliberately no
+     * class, source or routing options: it always runs on the main connection
+     * as interaction (so lanes hold their slices while it runs). What it saves
+     * is queue entries: a dependent chain sent one by one goes to the back of
+     * the queue at every step.
+     *
+     * Logged as one record, kind 'batch', with the joined text and `count`.
+     * Without `queryIPC` (mocks, custom connections), or with
+     * `QueryEngineOptions.unbatch`, the statements go one by one.
+     */
+    async execBatch(statements) {
+        if (this.#destroyed)
+            throw new Error('QueryEngine has been destroyed');
+        if (statements.length === 0)
+            throw new Error('execBatch: no statements');
+        if (!this.#conn)
+            await this.#ensureConnection();
+        const place = { class: 'interaction' };
+        let result;
+        if (this.#unbatch || !this.#conn?.queryIPC) {
+            for (const stmt of statements.slice(0, -1))
+                await this.exec(stmt);
+            result = await this.#queryRaw(statements[statements.length - 1], place);
+        }
+        else {
+            const sql = statements.join(';\n');
+            result = await this.#send('batch', sql, place, {
+                run: async (conn) => tableFromIPC(await conn.queryIPC(sql)),
+                count: statements.length
+            });
+        }
+        for (const stmt of statements)
+            this.#noteDDL(stmt);
+        return result;
+    }
     /**
      * Execute DDL/DML statements (CREATE TABLE, DROP TABLE, INSERT, etc.).
      * Accepts a single SQL string or an array of statements to execute
      * sequentially.
      */
-    async exec(sql) {
+    async exec(sql, opts) {
         if (this.#destroyed) {
             throw new Error('QueryEngine has been destroyed');
         }
-        if (!this.#conn)
-            await this.#ensureConnection();
+        const place = {
+            class: opts?.class ?? 'interaction',
+            connectionLocal: opts?.connectionLocal
+        };
+        const signal = opts?.signal;
         const statements = Array.isArray(sql) ? sql : [sql];
-        const execFn = this.#conn.exec?.bind(this.#conn);
+        // One statement at a time, each committing on its own.
         for (const stmt of statements) {
-            if (execFn) {
-                await execFn(stmt);
-            }
-            else {
-                await this.#conn.query(stmt);
-            }
+            await this.#send('exec', stmt, place, {
+                run: async (conn) => {
+                    if (conn.exec)
+                        await conn.exec(stmt);
+                    else
+                        await conn.query(stmt);
+                },
+                runSliced: async (lane) => {
+                    await lane.exec(stmt, signal);
+                },
+                signal
+            });
+            this.#noteDDL(stmt);
         }
     }
+    /**
+     * Learn what a source is from the statement that made it (SourceRouting's
+     * prior), at no cost: a table created under a plain name starts fast, a
+     * view unknown, a dropped name forgets. Covers everything created through
+     * the engine (loadParquet, Arrow inserts, fjader's own tables); a source
+     * made elsewhere falls back to the catalog (#classifySource).
+     */
+    #noteDDL(sql) {
+        const ddl = sourceDDL(sql);
+        if (!ddl)
+            return;
+        if (ddl.kind === 'drop') {
+            this.#known.delete(ddl.name);
+            this.#localSources.delete(ddl.name);
+            this.#routing.forget(ddl.name);
+            return;
+        }
+        this.#known.set(ddl.name, ddl.kind);
+        if (ddl.temp)
+            this.#localSources.add(ddl.name);
+        else
+            this.#localSources.delete(ddl.name);
+        // Replacing a name re-opens its record: a view over a table, a new table.
+        this.#routing.forget(ddl.name);
+        if (ddl.kind === 'table')
+            this.#routing.prior(ddl.name, 'fast');
+    }
+    /** Sources seen created through the engine, by name. */
+    #known = new Map();
+    /**
+     * Sources that exist only on the main connection: TEMP tables and views
+     * (they belong to the connection that made them), whether seen created
+     * through the engine or found missing on a lane. Never routed to a lane,
+     * whatever their cost.
+     */
+    #localSources = new Set();
     /**
      * Get column info for a table via DESCRIBE.
      * Results are cached — same table returns the same promise.
@@ -449,13 +980,20 @@ export class QueryEngine {
             return cached;
         const promise = this.#fetchDescribe(table);
         this.#describeCache.set(table, promise);
+        // A failure is not remembered: a DESCRIBE sent before its table existed
+        // would otherwise fail every later one for good.
+        promise.catch(() => {
+            if (this.#describeCache.get(table) === promise)
+                this.#describeCache.delete(table);
+        });
         return promise;
     }
     async #fetchDescribe(table) {
-        if (!this.#conn)
-            await this.#ensureConnection();
         const escaped = table.replaceAll('"', '""');
-        const result = await this.#queryRaw(`DESCRIBE "${escaped}"`);
+        // A source not seen created through the engine is looked up in the
+        // catalog, off the critical path: until the answer it reads sliced.
+        void this.#classifySource(table);
+        const result = await this.#queryRaw(`DESCRIBE "${escaped}"`, { class: 'interaction' }, 'describe');
         const columns = [];
         for (let i = 0; i < result.numRows; i++) {
             const row = result.get(i);
@@ -467,6 +1005,67 @@ export class QueryEngine {
         }
         return columns;
     }
+    /**
+     * Set a source's prior from what it is (SourceRouting): a base or
+     * temporary table in an in-memory catalog starts fast; a view, or a
+     * table in an attached database with a path (a file, a URL), stays
+     * unknown and is read sliced until measured. Only where lanes exist: a
+     * single connection has nothing to route.
+     */
+    async #classifySource(table) {
+        if (!this.#conn?.openSlicedLane)
+            return;
+        if (this.#known.has(table))
+            return;
+        let known = this.#catalog?.get(table);
+        if (known === undefined)
+            known = (await this.#refreshCatalog()).get(table);
+        if (known === 'fast')
+            this.#routing.prior(table, 'fast');
+    }
+    /**
+     * Every user table, by name: 'fast' when every catalog holding that name
+     * is in memory, 'unknown' otherwise. A name not in it (a view, a table
+     * function) stays unknown, so the views need not be listed: duckdb_views()
+     * enumerates DuckDB's internal views before any filter and was most of
+     * the cost (information_schema per table: 30-38 ms at load on WASM;
+     * tables and views together still 33-41 ms; tables and databases alone
+     * about a quarter of that, natively). Refreshed when a name is missing;
+     * concurrent misses (a dozen marks mounting) share the refresh in flight.
+     */
+    #refreshCatalog() {
+        this.#catalogRefresh ??= this.#queryRaw(`SELECT t.table_name AS name, d.path AS path FROM duckdb_tables() t ` +
+            `JOIN duckdb_databases() d ON d.database_name = t.database_name WHERE NOT t.internal`, { class: 'interaction' })
+            .then((rows) => {
+            const catalog = new Map();
+            for (let i = 0; i < rows.numRows; i++) {
+                const row = rows.get(i);
+                const inMemory = !row.path || row.path === ':memory:';
+                // A name in two catalogs is fast only if every one is.
+                if (catalog.get(row.name) !== 'unknown') {
+                    catalog.set(row.name, inMemory ? 'fast' : 'unknown');
+                }
+            }
+            this.#catalog = catalog;
+            return catalog;
+        })
+            .catch(() => this.#catalog ?? new Map())
+            .finally(() => {
+            this.#catalogRefresh = null;
+        });
+        return this.#catalogRefresh;
+    }
+    #catalog = null;
+    #catalogRefresh = null;
+    /**
+     * Tell the engine how a source reads, overriding its prior: `'slow'` for a
+     * table that is slow to query (it reads sliced, so it never holds up
+     * other charts), `'fast'` for a view that is cheap (it reads on the main
+     * connection from its first query). A measured read still corrects it.
+     */
+    sourceCost(source, cost) {
+        this.#routing.hint(source, cost);
+    }
     /** Clear cached DESCRIBE results. Pass a table name to clear one, or omit for all. */
     clearDescribeCache(table) {
         if (table) {
@@ -509,6 +1108,16 @@ export class QueryEngine {
         // against dropped state (see CategoricalIdentity's generation counter).
         const identityReset = this.#identity.reset().finally(() => this.#notifyIdentityReset());
         this.#preagg.reset();
+        // New data: a source's cost is unknown again (a reload can be slower),
+        // and the catalog is read again.
+        this.#routing.reset();
+        this.#catalog = null;
+        // Tables learned from DDL are still tables: their prior comes back
+        // (#classifySource skips names it already knows).
+        for (const [name, kind] of this.#known) {
+            if (kind === 'table')
+                this.#routing.prior(name, 'fast');
+        }
         this.#queryCache.clear();
         return identityReset;
     }
@@ -538,13 +1147,17 @@ export class QueryEngine {
      * Throws if the underlying connection doesn't support it (e.g. REST).
      */
     async insertArrowFromIPCStream(data, options) {
-        if (!this.#conn)
-            await this.#ensureConnection();
-        const conn = this.#conn;
-        if (!conn.insertArrowFromIPCStream) {
-            throw new Error('Arrow IPC loading requires a DuckDB-WASM connection');
-        }
-        await conn.insertArrowFromIPCStream(data, options);
+        await this.#send('insert', `INSERT ARROW INTO "${options.name}" (${data.byteLength} bytes)`, { class: 'interaction' }, {
+            run: async (conn) => {
+                if (!conn.insertArrowFromIPCStream) {
+                    throw new Error('Arrow IPC loading requires a DuckDB-WASM connection');
+                }
+                await conn.insertArrowFromIPCStream(data, options);
+            }
+        });
+        // The notebook widget's load path: a table made from Arrow is a table.
+        if (options.create)
+            this.#noteDDL(`CREATE TABLE "${options.name.replaceAll('"', '""')}" (x)`);
     }
     // -----------------------------------------------------------------
     // v3: Mark registration + batch filter queries
@@ -571,15 +1184,29 @@ export class QueryEngine {
         }
         // Reuse existing handle to preserve arrowTable during re-query
         const handle = prev?.handle ?? new MarkHandle();
-        const reg = { descriptor, handle, filterGroup };
+        const reg = {
+            descriptor,
+            handle,
+            filterGroup,
+            stale: false,
+            needsFirst: !prev || prev.descriptor.parts !== descriptor.parts
+        };
         this.#marks.set(markId, reg);
         this.#preagg.trackTable(descriptor.baseTable);
         handle._onParamsChange = () => this.#queryMark(markId, reg);
+        handle._onActivate = () => {
+            if (!reg.stale)
+                return false;
+            reg.stale = false;
+            this.#queryMark(markId, reg, 'catchUp');
+            return true;
+        };
         handle.destroy = () => {
             this.cancel(markId);
             this.#preagg.dropMark(markId);
             this.#marks.delete(markId);
             handle._onParamsChange = null;
+            handle._onActivate = null;
         };
         // Re-registration: trigger re-query immediately with updated descriptor.
         // The params effect won't fire since the handle reference didn't change.
@@ -644,16 +1271,37 @@ export class QueryEngine {
      * Query a single mark: on a filter change or a params
      * change (via _onParamsChange). Reads a preagg table when one answers; if
      * that read fails, the table is set aside and the base table is queried once.
+     *
+     * `cause` travels with this query's own promise chain to `_setResult`, not
+     * on the handle: a catch-up in flight and an update waiting behind it are
+     * two chains, so each result lands with its own cause. A filter change
+     * during a catch-up lets the catch-up land (without animating) and the
+     * update then animates from there; a superseded pending query rejects, and
+     * its chain sets nothing.
      */
-    #queryMark(markId, reg) {
+    #queryMark(markId, reg, cause = 'update') {
         const { handle } = reg;
+        // Off screen and already holding data for this registration: nothing to
+        // show for it now. A registration's first query still runs (it can feed
+        // a domain another chart shares), including after a re-registration
+        // with new parts, where the handle still holds the old data.
+        if (!handle.active && handle.dataVersion > 0 && !reg.needsFirst) {
+            reg.stale = true;
+            return;
+        }
+        reg.stale = false;
         const preagg = this.#preagg.sqlFor(markId, reg);
         const sqlStr = preagg?.sql || this.#baseSQL(markId, reg);
         if (!sqlStr)
             return;
+        // Only once a first query actually goes out.
+        reg.needsFirst = false;
         // Use the public query() path — its execute-latest logic ensures
         // at most 1 inflight + 1 pending per callerId during rapid brushing.
-        this.query(sqlStr, { callerId: markId })
+        // A preagg table is built in memory by fjader: fast until measured.
+        if (preagg)
+            this.#routing.prior(preagg.table, 'fast');
+        this.query(sqlStr, markReadOptions(markId, reg.descriptor, preagg?.table ?? null))
             .catch((e) => {
             if (!preagg || e?.name === 'AbortError')
                 throw e;
@@ -661,9 +1309,9 @@ export class QueryEngine {
             const baseSQL = this.#baseSQL(markId, reg);
             if (!baseSQL)
                 throw e;
-            return this.query(baseSQL, { callerId: markId });
+            return this.query(baseSQL, markReadOptions(markId, reg.descriptor, null));
         })
-            .then((table) => handle._setResult(table))
+            .then((table) => handle._setResult(table, cause))
             .catch((e) => {
             if (e?.name === 'AbortError')
                 return;
@@ -756,6 +1404,8 @@ export class QueryEngine {
             pending.reject(new DOMException('Engine destroyed', 'AbortError'));
         }
         this.#pending.clear();
+        // The preagg drops above ran on the lane; close it, then the connection.
+        await Promise.all([this.#backgroundLane, ...this.#sourceLanes].map((slot) => slot.close()));
         const conn = this.#conn ?? (this.#connPromise ? await this.#connPromise : null);
         if (conn)
             await conn.close();