@fjader/svelte 0.2.4 → 0.2.5

package/dist/query-engine/FieldActivity.svelte.d.ts

@@ -1,7 +1,6 @@
 import type { ClockLoop } from '../animation';
 import { ActivityTable } from '../scales/ActivityTable';
 import type { FieldIdentity } from './categorical-identity';
-/** What a FieldActivity needs from the page around it. */
 /** One mark's Other rows for one bucket: what a click on it selects. */
 export interface OtherEntry {
     /** The mark's filter group, or null when it is filtered by the selection (`'filter'`). */
@@ -17,6 +16,7 @@ export interface OtherClick {
     action: 'replace' | 'add' | 'remove';
     values: unknown[];
 }
+/** What a FieldActivity needs from the page around it. */
 export interface FieldActivityHost {
     /** Drives the fades; null in unit tests and before a coordinator exists, and fades land at once. */
     readonly loop: ClockLoop | null;
@@ -49,8 +49,7 @@ export declare const DETACHED_HOST: FieldActivityHost;
  * existing ids keep their fades while new ones start in their state.
  * `reconcile` runs from the packer and from `set`, never from a reader, so
  * nothing is resized or retargeted while a template renders.
- */
-/**
+ *
  * THE FREE SLOT. Each group's table holds one slot past the field's ids: 1 while
  * the group selects nothing on the field, 0 while it selects anything, faded on
  * the same clock as every id. The shader reads it as the last slot of the

package/dist/query-engine/FieldActivity.svelte.js

@@ -44,8 +44,7 @@ class GroupFade {
  * existing ids keep their fades while new ones start in their state.
  * `reconcile` runs from the packer and from `set`, never from a reader, so
  * nothing is resized or retargeted while a template renders.
- */
-/**
+ *
  * THE FREE SLOT. Each group's table holds one slot past the field's ids: 1 while
  * the group selects nothing on the field, 0 while it selects anything, faded on
  * the same clock as every id. The shader reads it as the last slot of the

package/dist/query-engine/MarkHandle.svelte.d.ts

@@ -15,15 +15,42 @@ export interface QueryParams {
     sort?: NormalizedSortConfig;
     limit?: number;
 }
+/**
+ * Why a result was queried. `catchUp`: the mark came back into view stale (see
+ * `active`), so the result brings it up to date with changes it missed, which
+ * nobody just made; it lands without animating. `update`: anything else.
+ */
+export type ResultCause = 'update' | 'catchUp';
 export declare class MarkHandle {
     #private;
     arrowTable: Table<import("@uwdata/flechette").TypeMap>;
     dataVersion: number;
+    /**
+     * The cause of the result `dataVersion` counts, set with it. Plain, not
+     * $state: read by whatever reacts to `dataVersion`, never on its own.
+     */
+    resultCause: ResultCause;
     error: Error;
     get params(): QueryParams;
     /** Called by mark lifecycle when sort/limit change. */
     updateParams(params: QueryParams): void;
     _onParamsChange: (() => void) | null;
+    /**
+     * Whether the mark is on screen (or about to be). The engine does not
+     * re-query an inactive mark that has its first result: a filter or params
+     * change only marks it stale, and it queries once, with whatever is current,
+     * when it becomes active again. Its first query always runs, since the
+     * first result can feed domains other charts share. Not reactive: the
+     * engine reads it when it would send.
+     */
+    get active(): boolean;
+    /**
+     * Returns whether this call sent a catch-up query: the mark came back
+     * stale. A return with no query means the data it holds is current, so
+     * the caller can act on its return now rather than after a result.
+     */
+    setActive(active: boolean): boolean;
+    _onActivate: (() => boolean) | null;
     /**
      * Engine-internal: called to deliver query results.
      *
@@ -33,7 +60,7 @@ export declare class MarkHandle {
      * The error is cleared either way, so a success after a failure still
      * clears it.
      */
-    _setResult(table: Table): void;
+    _setResult(table: Table, cause?: ResultCause): void;
     /** Engine-internal: called on query error. */
     _setError(error: Error): void;
     /** Clean up registration. Set by engine during registerMark. */

package/dist/query-engine/MarkHandle.svelte.js

@@ -8,6 +8,11 @@
 export class MarkHandle {
     arrowTable = $state(null);
     dataVersion = $state(0);
+    /**
+     * The cause of the result `dataVersion` counts, set with it. Plain, not
+     * $state: read by whatever reacts to `dataVersion`, never on its own.
+     */
+    resultCause = 'update';
     error = $state(null);
     #params = $state({ sort: undefined, limit: undefined });
     get params() {
@@ -21,6 +26,32 @@ export class MarkHandle {
     // Engine sets this callback during registration.
     // Conventional underscore — engine needs external access.
     _onParamsChange = null;
+    /**
+     * Whether the mark is on screen (or about to be). The engine does not
+     * re-query an inactive mark that has its first result: a filter or params
+     * change only marks it stale, and it queries once, with whatever is current,
+     * when it becomes active again. Its first query always runs, since the
+     * first result can feed domains other charts share. Not reactive: the
+     * engine reads it when it would send.
+     */
+    get active() {
+        return this.#active;
+    }
+    /**
+     * Returns whether this call sent a catch-up query: the mark came back
+     * stale. A return with no query means the data it holds is current, so
+     * the caller can act on its return now rather than after a result.
+     */
+    setActive(active) {
+        if (active === this.#active)
+            return false;
+        this.#active = active;
+        return active ? (this._onActivate?.() ?? false) : false;
+    }
+    #active = true;
+    // Engine sets this during registration: query a mark that went stale, and
+    // say whether it did.
+    _onActivate = null;
     /**
      * Engine-internal: called to deliver query results.
      *
@@ -30,10 +61,11 @@ export class MarkHandle {
      * The error is cleared either way, so a success after a failure still
      * clears it.
      */
-    _setResult(table) {
+    _setResult(table, cause = 'update') {
         this.error = null;
         if (table === this.arrowTable)
             return;
+        this.resultCause = cause;
         this.arrowTable = table;
         this.dataVersion++;
     }

package/dist/query-engine/QueryEngine.d.ts

@@ -20,9 +20,11 @@
 import { type Table } from '@uwdata/flechette';
 export type { BrushActivation } from './sufficient-statistics';
 import { type QueryParts } from '../components/marks/lifecycle/query-builder';
+import { type BuildEvent } from './preagg/Preagg';
 import type { ActivationSource } from './preagg/plan';
 export type { ActiveDim, ActivationSource } from './preagg/plan';
 import { MarkHandle } from './MarkHandle.svelte';
+import { type LaneHold, type SlicedLane } from './sliced-lane';
 import { type DimValue, type FieldIdentity, type IdentityField, type SyntheticId } from './categorical-identity';
 import type { FilterGroup } from './FilterGroup.svelte';
 import { type Clause, type MarkRoute } from './CrossfilterSelection.svelte';
@@ -33,13 +35,141 @@ export interface QueryOpts {
     callerId?: string;
     /** AbortSignal for external cancellation. */
     signal?: AbortSignal;
-    /** Enable query result caching. Default true. Set false for queries against
-     *  volatile temp tables whose contents change without the SQL changing. */
+    /**
+     * Result caching, and sharing an identical query already running.
+     *
+     * - Mark queries (with a `callerId`): on by default; `false` for queries
+     *   against tables whose contents change without the SQL changing.
+     * - Ad-hoc queries (no `callerId`): off unless `true`, so an app re-reading
+     *   a table it writes to sees its writes. fjader's own setup queries pass
+     *   `true`: they read base tables, which change only on a data reload, after
+     *   which `clearQueryCache()` is the contract.
+     *
+     * The key includes the dims' version (see #cacheKey).
+     */
     cache?: boolean;
+    /**
+     * Who is waiting for it; see StatementClass. Default 'interaction'.
+     *
+     * `'background'` runs on another connection when the engine has one (a
+     * sliced lane on DuckDB-WASM), so a TEMP table created with `exec` on the
+     * main connection is invisible there and the query fails: pass
+     * `connectionLocal: true` for statements on connection-local state.
+     */
+    class?: StatementClass;
+    /** Reads or writes connection-local state; see StatementPlacement. */
+    connectionLocal?: boolean;
+    /**
+     * The table this read reads (base table, view or preagg table). An
+     * interaction read with one is routed by its source's cost (see
+     * SourceRouting): known-fast on the main connection, unknown or slow sliced
+     * on that source's lane. Without one it stays on the main connection.
+     */
+    source?: string;
+    /** A mark's data read: its cost updates the source's record. */
+    data?: boolean;
+    /**
+     * This read is known to be slow (a heavy query on whatever table, as a
+     * mark with `holdsEntrance={false}` says). The engine reads it where a
+     * slow read cannot hold up others (on DuckDB-WASM, sliced on its source's
+     * lane) and does not learn the source's cost from it: the source's other
+     * reads may be quick. A backend with one connection can ignore it.
+     */
+    slow?: boolean;
+}
+/**
+ * Where a statement may run, beside who waits for it.
+ *
+ * `connectionLocal`: it reads or writes state that belongs to one connection
+ * (a TEMP table, a session setting), so it stays on the main connection
+ * whatever its class. fjader's tooltip tables are TEMP: their writes, drops,
+ * hover lookups and the fits that read them pass it. Each table is written
+ * through one connection only, so statements on it keep their order. A
+ * backend with one connection, or truly shared state, can ignore it.
+ */
+export interface StatementPlacement {
+    class: StatementClass;
+    connectionLocal?: boolean;
+    /** See QueryOpts.source. */
+    source?: string;
+    /** See QueryOpts.data. */
+    data?: boolean;
+    /**
+     * When the read was asked for (query() called): for a mark's
+     * execute-latest read, the state it reflects. Recorded, for data age.
+     */
+    requestedAt?: number;
+    /** The caller (a mark's id), for the log. */
+    callerId?: string;
+    /** See QueryOpts.slow. */
+    slow?: boolean;
+}
+/**
+ * Who is waiting for a statement. Callers state a class, never a connection:
+ * which connection (or later, which backend) serves it is the engine's
+ * business, decided in one place, `#send`.
+ *
+ * - `'interaction'`: someone is looking at the result or about to: mark data,
+ *   hover lookups, a mark's setup (domains, dims). The default.
+ * - `'background'`: nobody waits for it directly: preagg builds, tooltip-table
+ *   writes, cleanup. On DuckDB-WASM it runs on a sliced lane (another
+ *   connection, see SlicedLane) unless `connectionLocal`, so a slow build no
+ *   longer holds up interaction. Without a lane, preagg builds yield to
+ *   interaction statements, not to each other's background work.
+ */
+export type StatementClass = 'interaction' | 'background';
+/**
+ * For `observeIdentityRequests`: a mark asking for categorical identity
+ * columns, or a table's drain starting (when its dims actually begin to build,
+ * which is what a chart's first read waits on).
+ */
+export interface IdentityRequest {
+    /** 'request': a mark's ensureColumns call; 'drain': one pass of a table's drain started. */
+    kind: 'request' | 'drain';
+    table: string;
+    /** For a request, the fields asked for; for a drain pass, the fields it took. */
+    fields: string[];
+    /** performance.now() when asked. */
+    time: number;
+}
+/** One statement as it reached the connection, for `observeStatements`. */
+export interface StatementRecord {
+    sql: string;
+    /** How it was sent: a query returning rows, a statement, a DESCRIBE, an Arrow insert. */
+    kind: 'query' | 'exec' | 'describe' | 'insert' | 'batch';
+    /** For a batch (execBatch): how many statements it ran as one message. */
+    count?: number;
+    class: StatementClass;
+    /** Which connection ran it: the main one, or a sliced lane. */
+    connection: 'main' | 'lane';
+    /** The source it was routed by, when it named one. */
+    source?: string;
+    /**
+     * When the read was asked for (StatementPlacement.requestedAt). With
+     * `end`, how old the state a result reflects is when it lands.
+     */
+    requested?: number;
+    /** Who asked (a mark's id), when it said. */
+    caller?: string;
+    /** performance.now() when handed to the connection, and when it settled. */
+    start: number;
+    end: number;
+    error?: unknown;
 }
 export interface QueryEngineOptions {
     /** Overrides BUILD_MAX_WAIT_MS; tests use it to make the queue deterministic. */
     buildMaxWaitMs?: number;
+    /**
+     * Debug. Send an execBatch's statements one by one, for per-statement
+     * timing. It changes what it measures: each statement then re-enters the
+     * queue behind whatever was sent meanwhile, which a batch avoids.
+     */
+    unbatch?: boolean;
+    /**
+     * Advanced. While the main connection works, all sliced lanes together get
+     * one slice per this many ms (see SLICE_SHARE_MS, the default).
+     */
+    sliceShareMs?: number;
 }
 export interface ColumnInfo {
     name: string;
@@ -58,11 +188,15 @@ export declare function buildSchemaMap(columns: ColumnInfo[]): Map<string, Colum
 /**
  * Minimal connection interface.
  * Supports two query paths:
- * - queryIPC (fast): returns raw Arrow IPC bytes, decoded by flechette (zero-copy)
+ * - queryIPC (fast): returns raw Arrow IPC bytes, decoded by flechette into typed arrays
  * - query (fallback): returns a Table directly (for mocks/testing)
  */
 export interface DuckDBConnection {
-    /** Fast path: raw Arrow IPC bytes → flechette tableFromIPC → zero-copy typed arrays. */
+    /**
+     * Fast path: raw Arrow IPC bytes → flechette tableFromIPC → typed arrays. Views on
+     * the bytes within one record batch (~2k rows from DuckDB-WASM); past that,
+     * flechette concatenates each column once.
+     */
     queryIPC?(sql: string): Promise<Uint8Array>;
     /** Fallback query path. Used when queryIPC is not available (mocks, testing). */
     query(sql: string): Promise<Table>;
@@ -75,6 +209,17 @@ export interface DuckDBConnection {
         name: string;
         create: boolean;
     }): Promise<void>;
+    /**
+     * Open another connection on the same database whose statements run
+     * sliced (see SlicedLane): a slow read there no longer blocks this
+     * connection's queries. DuckDB-WASM via `connectionFrom`; a connection
+     * without it keeps everything on itself. The engine opens lanes itself and
+     * sends through them from `#send`, so every statement keeps its record,
+     * class and accounting; nothing hands a raw lane to an app.
+     */
+    openSlicedLane?(options?: {
+        hold?: LaneHold;
+    }): Promise<SlicedLane>;
 }
 export interface QueryDescriptor {
     /** The table the mark queries: its identity view when it has categorical channels. */
@@ -86,7 +231,29 @@ export interface QueryDescriptor {
     transformCompiler: TransformCompiler;
     /** What routing needs to know about the mark. Absent in tests that route nothing. */
     route?: MarkRoute;
+    /**
+     * The mark is known to be slow (`holdsEntrance={false}`): its data reads
+     * go sliced from the first, whatever its source's cost. The mark, not its
+     * source: other marks on a fast table keep reading on the main connection.
+     */
+    slow?: boolean;
 }
+/**
+ * How a mark's data read is placed. Routed by the table it reads
+ * (SourceRouting): the preagg table when one answers, else the mark's BASE
+ * table (its identity view reads the same rows, and its range and stats reads
+ * name the base table: one record per source). `slow` (holdsEntrance={false})
+ * applies to the base query only: a preagg read is ~1 ms, and forced onto a
+ * lane during a brush it would cost ~20 ms of round trips in exactly the case
+ * preagg exists to make fast.
+ */
+export declare function markReadOptions(markId: string, descriptor: Pick<QueryDescriptor, 'baseTable' | 'slow'>, preaggTable: string | null): QueryOpts;
+/**
+ * DuckDB's catalog error for a table or view this connection cannot see.
+ * Matched by its wording, which an upgrade could change: the integration
+ * test missing-relation.integration.test.ts checks it against real DuckDB.
+ */
+export declare function isMissingRelation(e: unknown): boolean;
 export declare class QueryEngine {
     #private;
     constructor(connection: DuckDBConnection | Promise<DuckDBConnection>, options?: QueryEngineOptions);
@@ -178,6 +345,29 @@ export declare class QueryEngine {
      * dropped state still invalidates every id handed out before it.
      */
     onIdentityReset(listener: () => void): () => void;
+    /**
+     * Debug: every categorical identity request (a mark's ensureColumns call),
+     * with its table, fields and time, before deduplication, and every drain
+     * start (a table's dims beginning to build, with the fields it found
+     * pending). Returns the unsubscribe; costs nothing while no one listens, and
+     * a listener that throws is logged without failing identity work.
+     */
+    observeIdentityRequests(listener: (request: IdentityRequest) => void): () => void;
+    /**
+     * Debug: every step of every preagg build (BuildEvent): planned or
+     * skipped (with the size guard's numbers), queued (with its hover batch),
+     * started, then built, failed or dropped (with the check that failed and
+     * the values it compared). Returns the unsubscribe; costs nothing while no
+     * one listens, and a listener that throws is logged without failing the
+     * build.
+     */
+    observeBuilds(listener: (event: BuildEvent) => void): () => void;
+    /**
+     * Listen to every statement sent to the connection, as it settles. For
+     * debugging and measurement; returns the unsubscribe. Costs nothing while
+     * no one listens.
+     */
+    observeStatements(listener: (record: StatementRecord) => void): () => void;
     /**
      * Execute a query and return the Arrow result table.
      * Accepts a SQL string or any object with toString() (e.g. mosaic-sql Query).
@@ -190,17 +380,47 @@ export declare class QueryEngine {
     query(sql: string | {
         toString(): string;
     }, opts?: QueryOpts): Promise<Table>;
+    /**
+     * Run several statements as ONE message on the main connection, returning
+     * the last statement's rows. Each statement commits on its own and
+     * execution stops at the first failure, as when sent one by one (measured
+     * on DuckDB-WASM, /test/query-slicing §13); the error does not say which
+     * statement failed.
+     *
+     * For statements on tables the main connection owns (fjader's identity
+     * dims, present tables, views): one connection per table. Deliberately no
+     * class, source or routing options: it always runs on the main connection
+     * as interaction (so lanes hold their slices while it runs). What it saves
+     * is queue entries: a dependent chain sent one by one goes to the back of
+     * the queue at every step.
+     *
+     * Logged as one record, kind 'batch', with the joined text and `count`.
+     * Without `queryIPC` (mocks, custom connections), or with
+     * `QueryEngineOptions.unbatch`, the statements go one by one.
+     */
+    execBatch(statements: string[]): Promise<Table>;
     /**
      * Execute DDL/DML statements (CREATE TABLE, DROP TABLE, INSERT, etc.).
      * Accepts a single SQL string or an array of statements to execute
      * sequentially.
      */
-    exec(sql: string | string[]): Promise<void>;
+    exec(sql: string | string[], opts?: {
+        class?: StatementClass;
+        connectionLocal?: boolean;
+        signal?: AbortSignal;
+    }): Promise<void>;
     /**
      * Get column info for a table via DESCRIBE.
      * Results are cached — same table returns the same promise.
      */
     describe(table: string): Promise<ColumnInfo[]>;
+    /**
+     * Tell the engine how a source reads, overriding its prior: `'slow'` for a
+     * table that is slow to query (it reads sliced, so it never holds up
+     * other charts), `'fast'` for a view that is cheap (it reads on the main
+     * connection from its first query). A measured read still corrects it.
+     */
+    sourceCost(source: string, cost: 'fast' | 'slow'): void;
     /** Clear cached DESCRIBE results. Pass a table name to clear one, or omit for all. */
     clearDescribeCache(table?: string): void;
     /**