@fjader/svelte 0.2.3 → 0.2.5

package/dist/query-engine/preagg/Preagg.js

@@ -51,23 +51,46 @@ export class Preagg {
     constructor(host, buildMaxWaitMs) {
         this.#host = host;
         this.#builds = new BuildQueue({
-            exec: (sql) => host.exec(sql),
-            busy: () => host.busy(),
-            stillWanted: (item) => this.#stillWanted(item),
-            dropped: ({ markId, sourceId, build }) => {
+            exec: (sql, signal) => host.exec(sql, signal),
+            // Builds wait for interaction only on a shared connection. On a lane
+            // they run at once, and interaction still comes first on its own:
+            // each slice of a build is a poll, a message at the back of the
+            // worker's queue, so mark queries already waiting there are served
+            // before the next slice. One query waits at most a slice (~5 ms)
+            // behind a build, not the whole build: measured on 2M rows, an
+            // update's first queries 3-9 ms during a build, 52-57 ms on one
+            // connection. A burst of queries still stretches: the build takes a
+            // slice between two of them, and its polls need main-thread turns
+            // while results decode (+~65 ms on that burst, still under the one
+            // connection's 195 ms median); the build is the one that waits.
+            busy: () => !host.sliced() && host.busy(),
+            unwanted: (item) => this.#unwanted(item),
+            started: (item) => this.#emitFor('started', item),
+            dropped: (item, reason) => {
+                // Before the bookkeeping below: the detail reads it.
+                if (this.#observing()) {
+                    this.#emitFor('dropped', item, { reason, detail: this.#dropDetail(item, reason) });
+                }
+                const { markId, sourceId, build } = item;
                 const own = this.#marks.get(markId);
                 if (own?.building.get(sourceId) === build)
                     own.building.delete(sourceId);
             },
-            built: ({ markId, sourceId, build }) => {
+            built: (item) => {
+                this.#emitFor('built', item);
+                const { markId, sourceId, build } = item;
                 const own = this.#marks.get(markId);
                 if (own?.building.get(sourceId) !== build)
                     return;
                 own.building.delete(sourceId);
                 own.tables.set(sourceId, build);
             },
-            failed: ({ markId, sourceId, build }, e) => {
-                console.warn(`[QueryEngine] preagg failed for ${markId}:`, e);
+            failed: (item, e) => {
+                this.#emitFor('failed', item, { error: e });
+                const { markId, sourceId, build } = item;
+                if (!(e instanceof DOMException && e.name === 'AbortError')) {
+                    console.warn(`[QueryEngine] preagg failed for ${markId}:`, e);
+                }
                 const own = this.#marks.get(markId);
                 if (own?.building.get(sourceId) === build)
                     own.building.delete(sourceId);
@@ -82,6 +105,8 @@ export class Preagg {
      *  tables — hash-named tables are reused via CREATE TABLE IF NOT EXISTS. */
     dropMark(markId) {
         this.#marks.delete(markId);
+        // Its running build is no longer wanted: cancelled, it rolls back.
+        this.#builds.abortRunning((item) => item.markId === markId);
     }
     /**
      * Pre-aggregate marks for a filter group before a source is used.
@@ -110,6 +135,10 @@ export class Preagg {
         for (const [markId, reg] of this.#host.marks()) {
             if (reg.filterGroup !== filterGroup)
                 continue;
+            // A mark off screen is not re-queried, so a table for it would only
+            // take the connection from the ones that are.
+            if (!reg.handle.active)
+                continue;
             // A mark the source never filters gets no table for it, and keeps
             // the tables it holds for other sources.
             const route = reg.descriptor.route ?? NO_ROUTE;
@@ -132,10 +161,25 @@ export class Preagg {
             own.tables.delete(activeSourceId);
             own.building.delete(activeSourceId);
             const plan = planPreagg(reg.descriptor, dims, this.#host.identity);
-            if (!plan)
+            if (!plan) {
+                if (this.#observing()) {
+                    this.#emit({ kind: 'skipped', markId, sourceId: activeSourceId, reason: 'unplannable' });
+                }
                 continue;
-            if (this.#tooLarge(reg, plan.activeKeys))
+            }
+            const size = this.#size(reg, plan.activeKeys);
+            if (size.groups != null && size.groups > size.cap) {
+                if (this.#observing()) {
+                    this.#emit({
+                        kind: 'skipped',
+                        markId,
+                        sourceId: activeSourceId,
+                        reason: 'too large',
+                        detail: { ...size, markKeys: size.markKeys, activeKeys: plan.activeKeys }
+                    });
+                }
                 continue;
+            }
             // Bake every other clause into the preagg table. Sorted by source ID,
             // so the same filters always give the same SQL, and so the same table.
             const bakedClauses = selection.bakedClauses(route, activeSourceId);
@@ -167,9 +211,19 @@ export class Preagg {
             // Most recently planned last, for the cap on unused tables.
             this.#tables.delete(preaggTable);
             this.#tables.add(preaggTable);
+            if (this.#observing()) {
+                const event = { markId, sourceId: activeSourceId, table: preaggTable };
+                this.#emit({ kind: 'planned', ...event, detail: { groups: size.groups, cap: size.cap } });
+                this.#emit({ kind: 'queued', ...event, batch: this.#builds.batch });
+            }
             promises.push(this.#builds.push({ markId, sourceId: activeSourceId, build, createSQL }));
         }
-        this.#builds.pump();
+        // From a microtask, not now: activation runs in the pointer-enter handler,
+        // which first marks the chart hovered, and the effect that writes the
+        // hovered marks' tooltip tables (what the first hover waits for) flushes
+        // in an earlier microtask. That write is then in flight, `busy()` holds
+        // the build back, and the tooltip does not wait behind it.
+        queueMicrotask(() => this.#builds.pump());
         this.#evictUnused();
         await Promise.all(promises);
     }
@@ -233,6 +287,11 @@ export class Preagg {
      * A data reload: drop every queued build, every mark's tables and the tables
      * themselves (CREATE TABLE IF NOT EXISTS would otherwise keep serving
      * pre-reload aggregates), and the row counts.
+     *
+     * The running build is cancelled too (`clear`). On a lane it runs beside the
+     * reload, reading the old rows through its snapshot; should it finish anyway
+     * (the cancel lost the race), `built` finds its mark gone from `#marks` and
+     * does not register it, and the DROP below follows it on the same lane.
      */
     reset() {
         this.#builds.clear();
@@ -246,8 +305,8 @@ export class Preagg {
     /**
      * The engine is going: settle waiting activations and drop this session's
      * tables. Resolves once the drops have run; call it while the engine can
-     * still exec. A build already running is ahead of the drops on the one
-     * connection, so its table is dropped too.
+     * still exec. A build already running is cancelled, and is ahead of the
+     * drops on its connection either way, so its table is dropped too.
      */
     async destroy() {
         this.#destroyed = true;
@@ -298,23 +357,93 @@ export class Preagg {
         return own;
     }
     /**
-     * Whether a queued build is still wanted when it would be sent: checked
-     * against the mark's current registration, which a re-setup replaces,
-     * against the view now (a zoom in or a long pan while it waited means a
-     * table settling would only rebuild), and against the filters now: a clause
-     * baked into the table may have changed.
+     * Why a queued build is no longer wanted when it would be sent, or null if
+     * it still is: checked against the mark's current registration, which a
+     * re-setup replaces, against the view now (a zoom in or a long pan while it
+     * waited means a table settling would only rebuild), and against the
+     * filters now: a clause baked into the table may have changed. The reason
+     * goes into the drop's build record.
      */
-    #stillWanted({ markId, sourceId, build }) {
+    #unwanted({ markId, sourceId, build }) {
         const reg = this.#host.mark(markId);
-        return (!!reg &&
-            this.#marks.get(markId)?.building.get(sourceId) === build &&
-            reg.descriptor.parts === build.basis.parts &&
-            this.#binsFitCurrentView(build) &&
-            bakedClausesHold(reg.descriptor.route ?? NO_ROUTE, reg.filterGroup.selection, build));
+        if (!reg)
+            return 'unregistered';
+        // Gone off screen while the build waited.
+        if (!reg.handle.active)
+            return 'inactive';
+        if (this.#marks.get(markId)?.building.get(sourceId) !== build)
+            return 'superseded';
+        if (reg.descriptor.parts !== build.basis.parts)
+            return 'parts changed';
+        if (!this.#binsFitCurrentView(build))
+            return 'bins do not fit the view';
+        if (!bakedClausesHold(reg.descriptor.route ?? NO_ROUTE, reg.filterGroup.selection, build)) {
+            return 'baked clauses changed';
+        }
+        return null;
+    }
+    // --- Build records (observeBuilds) ---
+    #observing() {
+        return !!this.#host.observingBuilds?.();
+    }
+    #emit(event) {
+        this.#host.onBuild?.({ table: null, ...event, time: performance.now() });
+    }
+    #emitFor(kind, item, extra = {}) {
+        if (!this.#observing())
+            return;
+        this.#emit({
+            kind,
+            markId: item.markId,
+            sourceId: item.sourceId,
+            table: item.build.table,
+            batch: item.batch,
+            ...extra
+        });
+    }
+    /**
+     * The values a failed check compared, for a drop's record. For the bin
+     * check, each bin dim's built window against the view it was checked
+     * against: the domain read now, through the source's view callback.
+     */
+    #dropDetail(item, reason) {
+        const { build } = item;
+        if (reason === 'bins do not fit the view') {
+            const out = this.#viewScratch;
+            out.fill(NaN);
+            build.basis.view?.(out);
+            const dims = [];
+            let bin = 0;
+            for (const meta of build.activeMeta) {
+                if (meta.kind !== 'bin')
+                    continue;
+                const i = 3 * bin++;
+                const width = meta.bins / meta.scaleFactor;
+                dims.push({
+                    built: {
+                        d0: meta.d0,
+                        d1: meta.d0 + width,
+                        bins: meta.bins,
+                        scaleFactor: meta.scaleFactor
+                    },
+                    view: { v0: out[i], v1: out[i + 1], pixelExtent: out[i + 2] },
+                    fits: binFitsView(meta, out[i], out[i + 1], out[i + 2]),
+                    fineEnough: binsFineEnough(meta.scaleFactor, out[i + 1] - out[i], out[i + 2])
+                });
+            }
+            return { dims };
+        }
+        if (reason === 'parts changed')
+            return { sameParts: false };
+        if (reason === 'baked clauses changed')
+            return { baked: build.basis.bakedClauses.length };
+        return undefined;
     }
     /**
-     * The size guard: whether a planned table would hold more than a quarter of
-     * its base table's rows (a fixed 100,000 until the row count has been read).
+     * The size guard's numbers: the groups a planned table would hold, and the
+     * cap it must not exceed, a quarter of its base table's rows (a fixed
+     * 100,000 until the row count has been read). activate() skips a table
+     * over the cap.
      *
      * Measured on 1M rows (DuckDB-WASM, one thread): a toggle click on a table a
      * quarter the size of the base table is still ~3.7× faster than on the base
@@ -323,15 +452,14 @@ export class Preagg {
      * when the connection is idle, so their cost shows only as the wait of a
      * click that lands mid-build, and as memory; past a quarter that outweighs
      * the gain. The groups are estimated by estimateGroups; an unknown estimate
-     * builds.
+     * (null) builds.
      */
-    #tooLarge(reg, activeKeys) {
-        const groups = estimateGroups(markKeys(reg.descriptor, reg.handle, this.#host.identity), activeKeys);
-        if (groups == null)
-            return false;
+    #size(reg, activeKeys) {
+        const keys = markKeys(reg.descriptor, reg.handle, this.#host.identity);
+        const groups = estimateGroups(keys, activeKeys);
         const rows = this.#rowCounts.get(reg.descriptor.baseTable);
         const cap = typeof rows === 'number' ? rows / 4 : 100_000;
-        return groups > cap;
+        return { groups, cap, markKeys: keys };
     }
     /** Read a base table's row count once, for the size guard. */
     #countRows(table) {

package/dist/query-engine/preagg/plan.d.ts

@@ -70,6 +70,11 @@ export interface PreaggCreation {
     select: Record<string, ExprNode>;
     groupby: ExprNode[];
     orderby: ExprNode[];
+    /**
+     * The mark's own row filters (`filter()`), applied to the source rows inside the
+     * build: the table groups them away, so the read could not apply them.
+     */
+    where: ExprNode[];
     /**
      * Fields grouped by value inside and mapped to their ids on the grouped
      * result: `raw` is the inner column, `alias` the id column, `dim` the dim
@@ -80,7 +85,22 @@ export interface PreaggCreation {
         alias: string;
         dim: string;
     }>;
+    /**
+     * The mark's padding rows, held in the table so its reads need no padding
+     * (see TransformCompiler.padRows), or null. `rows` is their SELECT over
+     * every channel; the table takes its group channels (`dims`), 0 for every
+     * partial (`partials`: count and sum only, for which 0 adds nothing), and
+     * `__pad` true. Source rows whose bin is NULL are left out (in `where`):
+     * those bins are never drawn, and padding a base read drops them too.
+     */
+    pad: {
+        rows: string;
+        dims: string[];
+        partials: string[];
+    } | null;
 }
+/** The column marking a preagg table's padding rows; real rows hold false. */
+export declare const PAD_COLUMN = "__pad";
 /**
  * How each active dim appears in a preagg table. A `bin` dim is a pixel-bin
  * index over a window (`buildBinExpr`): bins 0 .. bins - 1 from d0, with -1
@@ -107,6 +127,12 @@ export type ActiveMeta = {
 export interface GroupKey {
     column: string | null;
     groups: number | null;
+    /**
+     * For a mark's key over several columns: the mark's own bins per binned
+     * source column (from its query parts, see binCounts), so a bin dim on one
+     * of them merges into the key instead of multiplying it.
+     */
+    columns?: ReadonlyMap<string, number>;
 }
 export interface PreaggState {
     table: string;
@@ -165,7 +191,13 @@ export declare function markKeys(descriptor: QueryDescriptor, handle: Pick<MarkH
  * mark with x and fill on the same field has that field's groups, not their
  * square, and a bin dim on a column the mark already groups by adds its bins
  * to that key's (each pixel bin falls in about one of the mark's bins)
- * instead of multiplying.
+ * instead of multiplying. For a mark key over several columns (a histogram
+ * coloured by a category) the same holds per value of the other columns: a
+ * bin dim on a column the mark bins into `d` bins makes each group of the
+ * rest hold about `d + bins` groups instead of `d`, so the key's groups grow
+ * by `1 + bins / d`. Only one bin dim merges into a mark key, on purpose: a
+ * 2D brush over two binned columns of the same mark multiplies the second,
+ * which overestimates (the guard then skips rather than builds too much).
  */
 export declare function estimateGroups(markKeys: GroupKey[], activeKeys: GroupKey[]): number | null;
 /**

package/dist/query-engine/preagg/plan.js

@@ -11,10 +11,12 @@
  * the query. Copyright (c) 2023-2025, UW Interactive Data Lab. BSD 3-Clause
  * License; see NOTICE.
  */
-import { isAggregateExpression, isNode, collectColumns, column, and, isBetween, literal, Query, sql as sqlTag } from '@uwdata/mosaic-sql';
-import { togglePredicate } from '../toggle-predicate';
+import { isAggregateExpression, isNode, collectColumns, column, and, isBetween, literal, Query, collectAggregates, sql as sqlTag } from '@uwdata/mosaic-sql';
+import { BITMAP_MIN_IDS, idBitmapPredicate, togglePredicate } from '../toggle-predicate';
 import { rewriteForPreagg, buildBinExpr, binsFineEnough, BIN_WINDOW_MARGIN } from '../sufficient-statistics';
 import { assembleQuery, castNumeric, castReal } from '../../components/marks/lifecycle/query-builder';
+/** The column marking a preagg table's padding rows; real rows hold false. */
+export const PAD_COLUMN = '__pad';
 /**
  * The preagg table a mark would build for a source's active dims, or null
  * when the mark cannot be pre-aggregated (no aggregates, or an aggregate that
@@ -37,6 +39,11 @@ export function planPreagg(descriptor, dims, identity) {
     const { parts, table, baseTable, schema } = descriptor;
     if (!parts.hasAggregates && !parts.hasWindowAggregates)
         return null;
+    // A having() on a preagg table would count its groups, not the source rows, and
+    // return the wrong rows without an error. The built-in transforms add none; see
+    // TransformCompiler.hasHaving for why, and for what supporting it would take.
+    if (descriptor.transformCompiler.hasHaving())
+        return null;
     // Analyze selectDef for preagg compatibility
     const analysis = rewriteForPreagg(parts.selectDef, parts.passthroughChannels);
     if (!analysis)
@@ -99,6 +106,7 @@ export function planPreagg(descriptor, dims, identity) {
         activeMeta.push({ kind: 'value', field, column: alias, byId: false });
         activeKeys.push({ column: null, groups: null });
     });
+    const pad = analysis.windowChannels ? null : padPlan(descriptor, preaggCols);
     // CAST-wrap dimension preaggCols (bake CAST into the preagg table).
     // Partial-aggregate and active columns stay raw for recombination.
     for (const [ch, expr] of Object.entries(preaggCols)) {
@@ -115,7 +123,9 @@ export function planPreagg(descriptor, dims, identity) {
             groupby: [...preaggGroupBy, ...activeGroupBy],
             // Sorted by bin columns for fast range scans
             orderby: binOrder,
-            joins
+            where: [...descriptor.transformCompiler.rowFilters(), ...(pad?.where ?? [])],
+            joins,
+            pad: pad?.pad ?? null
         },
         outputSelect,
         activeMeta,
@@ -123,6 +133,46 @@ export function planPreagg(descriptor, dims, identity) {
         activeKeys
     };
 }
+/**
+ * The padding rows a mark's table can hold, or null when its reads must pad
+ * themselves: the mark pads nothing, a transform would see the rows differently
+ * (TransformCompiler.padRows), or a measure is not a count or a sum. A mean of
+ * an empty bin would read 0 / 0 from a padding row, where padding gives 0.
+ */
+function padPlan(descriptor, preaggCols) {
+    const { parts, transformCompiler } = descriptor;
+    for (const expr of Object.values(parts.selectDef)) {
+        if (!isNode(expr) || !isAggregateExpression(expr))
+            continue;
+        for (const agg of collectAggregates(expr)) {
+            const name = agg.name.toLowerCase();
+            if (agg.isDistinct || (name !== 'count' && name !== 'sum'))
+                return null;
+        }
+    }
+    const rows = transformCompiler.padRows(Object.keys(parts.selectDef), {
+        groupKeys: parts.groupKeys,
+        bins: parts.bins ?? null
+    });
+    if (!rows)
+        return null;
+    const dims = [];
+    const partials = [];
+    for (const [col, expr] of Object.entries(preaggCols)) {
+        if (col.startsWith('__ps'))
+            partials.push(col);
+        else if (col.startsWith('__active') || col.startsWith('__raw'))
+            continue;
+        else if (!isAggregateExpression(expr))
+            dims.push(col);
+    }
+    // A filter on the source column, not on the bin: on the bin it would be
+    // pushed below the GROUP BY and compute every row's bin twice (measured:
+    // half again the build time), where a NULL test on the column is free.
+    const sources = new Set(dims.flatMap((col) => parts.bins?.[col]?.source ?? []));
+    const where = [...sources].map((source) => sqlTag([`(${source}) IS NOT NULL`]));
+    return { pad: { rows, dims, partials }, where };
+}
 /** The mark's channel that is `field`'s id channel (see idFieldOf), or null. */
 function idChannelFor(selectDef, field, identity) {
     for (const [channel, expr] of Object.entries(selectDef)) {
@@ -174,6 +224,14 @@ export function markKeys(descriptor, handle, identity) {
         else
             fields.add(field);
     }
+    // A bin() channel resolves to SQL text that collectColumns cannot see into,
+    // so its column comes from the plan: the mark groups by it too.
+    const bins = binCounts(parts);
+    if (bins) {
+        allIds = false;
+        for (const name of bins.keys())
+            columns.add(name);
+    }
     if (allIds) {
         return [...fields].map((field) => ({
             column: null,
@@ -181,7 +239,35 @@ export function markKeys(descriptor, handle, identity) {
         }));
     }
     const rows = handle.params.limit == null ? handle.arrowTable?.numRows : undefined;
-    return [{ column: columns.size === 1 ? [...columns][0] : null, groups: rows ?? 1 }];
+    if (columns.size === 1)
+        return [{ column: [...columns][0], groups: rows ?? 1 }];
+    return [{ column: null, groups: rows ?? 1, columns: bins }];
+}
+/**
+ * The mark's own bins per source column, from its query parts: each `bin()`
+ * channel records its binned source and how many bins it has (`parts.bins`,
+ * what `dense()` pads by). Read from the plan, never from the data. Only
+ * plain columns count; an expression's bins, or a mark without bin metadata,
+ * give no entry, and the estimate then multiplies as before.
+ */
+function binCounts(parts) {
+    if (!parts.bins)
+        return undefined;
+    const counts = new Map();
+    for (const padding of Object.values(parts.bins)) {
+        const name = plainColumn(padding.source);
+        if (name == null || !(padding.count > 0))
+            continue;
+        counts.set(name, Math.max(counts.get(name) ?? 0, padding.count));
+    }
+    return counts.size > 0 ? counts : undefined;
+}
+/** The column a bin's source SQL names, when it is a plain (optionally quoted) column. */
+function plainColumn(source) {
+    const quoted = /^"((?:[^"]|"")+)"$/.exec(source);
+    if (quoted)
+        return quoted[1].replaceAll('""', '"');
+    return /^[A-Za-z_][A-Za-z0-9_]*$/.test(source) ? source : null;
 }
 /**
  * The groups a planned table would hold: the mark's own keys (markKeys) and
@@ -190,7 +276,13 @@ export function markKeys(descriptor, handle, identity) {
  * mark with x and fill on the same field has that field's groups, not their
  * square, and a bin dim on a column the mark already groups by adds its bins
  * to that key's (each pixel bin falls in about one of the mark's bins)
- * instead of multiplying.
+ * instead of multiplying. For a mark key over several columns (a histogram
+ * coloured by a category) the same holds per value of the other columns: a
+ * bin dim on a column the mark bins into `d` bins makes each group of the
+ * rest hold about `d + bins` groups instead of `d`, so the key's groups grow
+ * by `1 + bins / d`. Only one bin dim merges into a mark key, on purpose: a
+ * 2D brush over two binned columns of the same mark multiplies the second,
+ * which overestimates (the guard then skips rather than builds too much).
  */
 export function estimateGroups(markKeys, activeKeys) {
     if (activeKeys.some((key) => key.groups == null))
@@ -198,6 +290,15 @@ export function estimateGroups(markKeys, activeKeys) {
     const merged = new Set();
     let groups = 1;
     for (const key of markKeys) {
+        if (key.column == null && key.columns) {
+            const within = activeKeys.find((active) => active.column != null && key.columns.has(active.column) && !merged.has(active));
+            if (within) {
+                merged.add(within);
+                const d = Math.max(1, key.columns.get(within.column));
+                groups *= Math.ceil(key.groups * (1 + within.groups / d));
+                continue;
+            }
+        }
         const same = key.column == null
             ? undefined
             : activeKeys.find((active) => active.column === key.column && !merged.has(active));
@@ -280,8 +381,9 @@ export function preaggQuerySQL(route, descriptor, selection, preagg, params, ide
         }
         if (valueExprs && valueMeta) {
             // A toggle's clause holds tuples in its field order. Id columns
-            // filter by id; a value with no id (NULL, or not in the dim) cannot
-            // be expressed, so this query goes to the base table instead.
+            // filter by id (the NULL category has one); a value with no id (not in
+            // the dim yet) cannot be expressed, so this query goes to the base
+            // table instead.
             const tuples = [];
             for (const tuple of clause.value) {
                 const mapped = [];
@@ -294,7 +396,17 @@ export function preaggQuerySQL(route, descriptor, selection, preagg, params, ide
                 }
                 tuples.push(mapped);
             }
-            filters.push(togglePredicate(valueExprs, tuples));
+            // One field by id, and many of them (a click on a large Other): a
+            // constant-size bitmap rather than a long IN list, which DuckDB spends
+            // most of its time parsing. Measured on a preagg table: 1 ms against
+            // 20 ms for IN of 9 000 ids.
+            const only = valueMeta.length === 1 ? valueMeta[0] : null;
+            if (only?.byId && tuples.length > BITMAP_MIN_IDS) {
+                filters.push(idBitmapPredicate(valueExprs[0], tuples.map((t) => t[0]), identity.identity(only.field).values.length));
+            }
+            else {
+                filters.push(togglePredicate(valueExprs, tuples));
+            }
         }
     }
     // --- Build selectDef: CAST recombined aggregates; dimensions are already
@@ -315,33 +427,65 @@ export function preaggQuerySQL(route, descriptor, selection, preagg, params, ide
             castSelect[ch] = castReal(expr);
         }
     }
+    // A table holding its padding: those rows pass whatever the brush says, and
+    // the read's GROUP BY folds them into the real rows of the same group.
+    if (preagg.creation.pad && filters.length > 0) {
+        filters.splice(0, filters.length, sqlTag `(${column(PAD_COLUMN)} OR ${and(filters)})`);
+    }
     const preaggParts = {
         selectDef: castSelect,
         nonAggregateGroupBy: preaggGroupBy,
         hasAggregates: true,
         groupKeys: preaggGroupKeys,
+        // The table keeps the bin channels as columns of the same bins, so the
+        // read pads exactly as the base read does, or is padded already.
+        bins: parts.bins,
+        padded: !!preagg.creation.pad,
         hasWindowAggregates: false,
         hasSQLTransforms: parts.hasSQLTransforms,
         orderChannels: parts.orderChannels,
         aliasMap: parts.aliasMap,
-        passthroughChannels: parts.passthroughChannels
+        passthroughChannels: parts.passthroughChannels,
+        // nulls="drop", read through this select: each dropped channel is a group
+        // key, so the table keeps NULL as its own group, and dropping that group
+        // here removes exactly what the base scan would have left out.
+        nullDrops: parts.nullDrops
     };
     return assembleQuery({
         table: preagg.table,
         parts: preaggParts,
         filter: filters,
         transformCompiler,
+        // The table was built with them.
+        skipRowFilters: true,
         limit: params.limit,
         window: windowChannels ? { channels: windowChannels, cteName: '__preagg_cte' } : null
     }).toString();
 }
 /** A preagg table's SELECT. Build and validity check share it, so equal SQL means an equal table. */
 export function preaggSelectSQL(creation, filters) {
+    const pad = creation.pad;
+    if (!pad)
+        return realRowsSQL(creation, filters, true);
+    // By name: the padding rows name only what they set, and the brush and
+    // value columns they leave out come back NULL. Ordered once, outside.
+    const quote = (name) => `"${name.replaceAll('"', '""')}"`;
+    const padRows = `SELECT ${pad.dims.map((col) => `p.${quote(col)}`).join(', ')}` +
+        pad.partials.map((col) => `, 0 AS ${quote(col)}`).join('') +
+        `, true AS ${quote(PAD_COLUMN)} FROM (${pad.rows}) AS p`;
+    const order = creation.orderby.length ? ` ORDER BY ${creation.orderby.join(', ')}` : '';
+    return (`SELECT * FROM (SELECT *, false AS ${quote(PAD_COLUMN)} FROM (${realRowsSQL(creation, filters, false)}) AS r` +
+        ` UNION ALL BY NAME ${padRows})${order}`);
+}
+/** The table's real rows: the mark's groups over the source rows. */
+function realRowsSQL(creation, filters, ordered) {
     const query = Query.from(creation.table).select(creation.select).groupby(creation.groupby);
-    if (filters.length > 0)
-        query.where(and(filters));
-    if (creation.joins.length === 0)
-        return query.orderby(creation.orderby).toString();
+    const where = [...creation.where, ...filters];
+    if (where.length > 0)
+        query.where(and(where));
+    if (creation.joins.length === 0) {
+        return (ordered ? query.orderby(creation.orderby) : query).toString();
+    }
     // Map fields grouped by value to their ids on the grouped result: a join
     // over the table's groups, not over every source row. The filters stay in
     // the inner query, where the source's columns are in scope.
@@ -353,7 +497,7 @@ export function preaggSelectSQL(creation, filters) {
     const joins = creation.joins
         .map((j, k) => `LEFT JOIN ${quote(j.dim)} AS d${k} ON p.${quote(j.raw)} = d${k}.value`)
         .join(' ');
-    const order = creation.orderby.length ? ` ORDER BY ${creation.orderby.join(', ')}` : '';
+    const order = ordered && creation.orderby.length ? ` ORDER BY ${creation.orderby.join(', ')}` : '';
     return `SELECT p.* EXCLUDE (${raws}), ${ids} FROM (${query}) AS p ${joins}${order}`;
 }
 /**