@fjader/svelte 0.2.3 → 0.2.5

package/dist/transforms/TransformCompiler.d.ts

@@ -1,4 +1,4 @@
-import type { SelectQuery } from '@uwdata/mosaic-sql';
+import type { ExprNode, SelectQuery } from '@uwdata/mosaic-sql';
 import type { Transform, SqlTransform, JsTransform, MarkConfig, ScaleMap } from '../types';
 import type { DataAccessor } from '../data/DataAccessor';
 export declare class TransformCompiler {
@@ -19,7 +19,50 @@ export declare class TransformCompiler {
      * than of the mark — its GROUP BY keys, say — merged over the mark config for
      * this call only, so the shared config object is never written per query.
      */
-    applyToQuery(query: SelectQuery, facts?: Partial<MarkConfig>): SelectQuery;
+    applyToQuery(query: SelectQuery, facts?: Partial<MarkConfig>, 
+    /** The row filters are already in the table the query reads (a preagg table). */
+    options?: {
+        skipRowFilters?: boolean;
+    }): SelectQuery;
+    /**
+     * The padding rows of this mark's queries (see SqlTransform.padRows), for a
+     * pre-aggregated table to hold, or null when they cannot live in a table.
+     *
+     * Only when padding is the last thing that changes which rows exist: every
+     * other SQL transform is one that runs the same over a padded table as over
+     * a padded result. A topN ranks before padding, so zero rows in its table
+     * would enter the ranking; anything else not listed is not known to be safe.
+     */
+    padRows(aliases: string[], facts?: Partial<MarkConfig>): string | null;
+    /**
+     * Whether a SQL transform filters on the groups' aggregates (`having`). Its
+     * condition names aggregates over the source rows, which a pre-aggregated table
+     * would recompute over its own groups: `count(*)` counts the table's groups, not
+     * the rows, so it gives a different answer with no error.
+     *
+     * Only the user-facing `having()` does this. The built-ins (topN, rank, stack,
+     * normalize, dense) add no HAVING: each wraps the grouped query in a subquery
+     * (`Query.from({ __topn_base: query })`) and adds window functions or joins, and
+     * those work the same on a preagg table's aggregated result, which is why a topN
+     * mark is served from preagg. The only CTE is assembleQuery's window wrapper for
+     * window aggregates, applied after the transforms. A test in transforms.test.ts
+     * fails if a built-in starts to emit a HAVING; this then has to know it.
+     *
+     * Supporting `having()` on a preagg table would mean rewriting its condition the
+     * way a select channel is rewritten (sufficient-statistics: collectAggregates,
+     * sufficientStats, rewrite; an aggregate not in the select adds a partial
+     * column) and applying it to the inner grouped query BEFORE the transforms wrap
+     * it, so it ends up inside their subquery as it does on the base table.
+     */
+    hasHaving(): boolean;
+    /**
+     * The row predicates of the SQL transforms (`filter`), for a pre-aggregated
+     * table to be built with: its read cannot apply them (see SqlTransform.where).
+     * A row filter applies before grouping, so building the table with it gives the
+     * same rows the base query sees; only `filter()` is one (see hasHaving for the
+     * transforms that are not).
+     */
+    rowFilters(): ExprNode[];
     /**
      * Which channels a query with these aliases comes back aligned on, leading
      * term first, or null. Asked with the same facts as `applyToQuery`, so the
@@ -60,5 +103,7 @@ export declare class TransformCompiler {
      * before the transform can write it.
      */
     hasSyntheticOtherBucket(): boolean;
+    /** The `by` a topN Other bucket ranks by when stated explicitly, else null (its default). */
+    otherBy(): string | null;
     getCategoryTransformKey(): string | null;
 }

package/dist/transforms/TransformCompiler.js

@@ -1,3 +1,9 @@
+/**
+ * SQL transforms that give the same rows over a table holding its padding as
+ * padding gives their result: row filters (applied to the table's build), and
+ * what runs after padding (rank, stack, normalize).
+ */
+const PAD_SAFE = new Set(['dense', 'filter', 'rank', 'stack', 'normalize']);
 export class TransformCompiler {
     transforms;
     sqlTransforms;
@@ -35,10 +41,14 @@ export class TransformCompiler {
      * than of the mark — its GROUP BY keys, say — merged over the mark config for
      * this call only, so the shared config object is never written per query.
      */
-    applyToQuery(query, facts) {
+    applyToQuery(query, facts, 
+    /** The row filters are already in the table the query reads (a preagg table). */
+    options) {
         let result = query;
         const config = this.#configFor(facts);
         for (const transform of this.sqlTransforms) {
+            if (options?.skipRowFilters && transform.where !== undefined)
+                continue;
             try {
                 result = transform.apply(result, config);
             }
@@ -48,6 +58,59 @@ export class TransformCompiler {
         }
         return result;
     }
+    /**
+     * The padding rows of this mark's queries (see SqlTransform.padRows), for a
+     * pre-aggregated table to hold, or null when they cannot live in a table.
+     *
+     * Only when padding is the last thing that changes which rows exist: every
+     * other SQL transform is one that runs the same over a padded table as over
+     * a padded result. A topN ranks before padding, so zero rows in its table
+     * would enter the ranking; anything else not listed is not known to be safe.
+     */
+    padRows(aliases, facts) {
+        if (!this.sqlTransforms.every((t) => PAD_SAFE.has(t.type)))
+            return null;
+        const config = this.#configFor(facts);
+        for (const transform of this.sqlTransforms) {
+            const rows = transform.padRows?.(aliases, config);
+            if (rows)
+                return rows;
+        }
+        return null;
+    }
+    /**
+     * Whether a SQL transform filters on the groups' aggregates (`having`). Its
+     * condition names aggregates over the source rows, which a pre-aggregated table
+     * would recompute over its own groups: `count(*)` counts the table's groups, not
+     * the rows, so it gives a different answer with no error.
+     *
+     * Only the user-facing `having()` does this. The built-ins (topN, rank, stack,
+     * normalize, dense) add no HAVING: each wraps the grouped query in a subquery
+     * (`Query.from({ __topn_base: query })`) and adds window functions or joins, and
+     * those work the same on a preagg table's aggregated result, which is why a topN
+     * mark is served from preagg. The only CTE is assembleQuery's window wrapper for
+     * window aggregates, applied after the transforms. A test in transforms.test.ts
+     * fails if a built-in starts to emit a HAVING; this then has to know it.
+     *
+     * Supporting `having()` on a preagg table would mean rewriting its condition the
+     * way a select channel is rewritten (sufficient-statistics: collectAggregates,
+     * sufficientStats, rewrite; an aggregate not in the select adds a partial
+     * column) and applying it to the inner grouped query BEFORE the transforms wrap
+     * it, so it ends up inside their subquery as it does on the base table.
+     */
+    hasHaving() {
+        return this.sqlTransforms.some((t) => t.type === 'having');
+    }
+    /**
+     * The row predicates of the SQL transforms (`filter`), for a pre-aggregated
+     * table to be built with: its read cannot apply them (see SqlTransform.where).
+     * A row filter applies before grouping, so building the table with it gives the
+     * same rows the base query sees; only `filter()` is one (see hasHaving for the
+     * transforms that are not).
+     */
+    rowFilters() {
+        return this.sqlTransforms.flatMap((t) => (t.where === undefined ? [] : [t.where]));
+    }
     /**
      * Which channels a query with these aliases comes back aligned on, leading
      * term first, or null. Asked with the same facts as `applyToQuery`, so the
@@ -147,6 +210,12 @@ export class TransformCompiler {
     hasSyntheticOtherBucket() {
         return this.transforms.some((t) => t.type === 'topN' && t.spec?.includeOthers === true);
     }
+    /** The `by` a topN Other bucket ranks by when stated explicitly, else null (its default). */
+    otherBy() {
+        const t = this.transforms.find((t) => t.type === 'topN' && t.spec?.includeOthers === true);
+        const by = t?.spec?.by;
+        return typeof by === 'string' ? by : null;
+    }
     getCategoryTransformKey() {
         const parts = [];
         for (const t of this.transforms) {

package/dist/transforms/index.js

@@ -6,6 +6,7 @@
  * - JS: Process data via the DataAccessor interface (zero-copy on Arrow data)
  */
 import { Query, column, sql as sqlTag } from '@uwdata/mosaic-sql';
+import { numericBinEdgeSQL } from '../sql/bin';
 // =============================================================================
 // SQL TRANSFORMS (Query Modifiers)
 // =============================================================================
@@ -14,6 +15,7 @@ export function filter(condition) {
         stage: 'sql',
         type: 'filter',
         spec: { transform: 'filter', condition: condition.toString() },
+        where: condition,
         apply(query) {
             return query.where(condition);
         },
@@ -207,18 +209,29 @@ const BAND_SOURCE_COLUMN = '__dplot_band_src';
  * instead, and the query assembler orders the outermost query by it.
  */
 function densify(inner, aliases, plan) {
-    const { band, dim, cross } = plan;
+    const { band, dim, cross, bins } = plan;
     const crossAlias = new Map(cross.map(([key], i) => [key, `k${i}`]));
+    const binOf = new Map(bins);
     const select = {};
     for (const alias of aliases) {
         const crossed = crossAlias.get(alias);
-        if (alias === band) {
-            select[alias] = sqlTag `d."id"`;
+        const bin = binOf.get(alias);
+        if (bin) {
+            // Every row's edge from its bin index, real and padded alike, so the
+            // edges are the same bits on every update: they are what tells the
+            // renderer the row set has not changed (alignmentKey).
+            select[alias] = rawSql(binEdge(bin, 'd.k'));
+        }
+        else if (alias === band) {
+            // REAL, as the inner query's id columns are: an INTEGER column would
+            // reach the GPU through a copy, and hand the renderer other bits than
+            // a preagg read, which keeps the REAL ids.
+            select[alias] = rawSql(`CAST(d."id" AS REAL)`);
         }
         else if (crossed) {
             // Another group key, enumerated from its own dim: the padded row for a
             // missing (category, series) pair carries that series' real id.
-            select[alias] = rawSql(`${crossed}."id"`);
+            select[alias] = rawSql(`CAST(${crossed}."id" AS REAL)`);
         }
         else if (alias.startsWith(OTHER_MEMBER_PREFIX)) {
             // Other's member lists: NULL on a padded row, as on every row but
@@ -226,7 +239,7 @@ function densify(inner, aliases, plan) {
             // a list.
             select[alias] = rawSql(`a.${column(alias)}`);
         }
-        else if (plan.identity[alias]?.dim === dim) {
+        else if (!bins && plan.identity[alias]?.dim === dim) {
             // Same dim as the band, so this channel IS the category — the ordinary
             // "colour the bars by their own category" chart. Its value for a row
             // with no data is not unknown, it is that category's id, so take it
@@ -235,7 +248,7 @@ function densify(inner, aliases, plan) {
             // This is what keeps a DEPARTING bar its colour. It is still on screen
             // while it shrinks, and a sentinel would turn it grey on the frame its
             // data disappeared — the one frame where the colour matters most.
-            select[alias] = sqlTag `d."id"`;
+            select[alias] = rawSql(`CAST(d."id" AS REAL)`);
         }
         else if (plan.identity[alias]) {
             // A category id from some OTHER field that is not a group key — an
@@ -243,7 +256,7 @@ function densify(inner, aliases, plan) {
             // so the absent sentinel it is. Never 0, which is a real category: it
             // would enter the colour scale's domain, shift every palette rank after
             // it, and repaint bars that had not changed.
-            select[alias] = sqlTag `COALESCE(a.${column(alias)}, -1)`;
+            select[alias] = sqlTag `COALESCE(a.${column(alias)}, CAST(-1 AS REAL))`;
         }
         else {
             // A measure. 0 is the point: it is what makes a departing bar shrink to
@@ -274,11 +287,52 @@ function densify(inner, aliases, plan) {
     const joins = cross
         .map(([key, keyDim], i) => ` CROSS JOIN (SELECT "id" FROM ${padIds(plan.identity[key], keyDim)} UNION ALL SELECT -1) AS k${i}`)
         .join('');
-    const on = [
-        `a.${column(band)} = d."id"`,
-        ...cross.map(([key], i) => `a.${column(key)} = k${i}."id"`)
-    ].join(' AND ');
-    return Query.from(rawSql(`${padIds(plan.identity[band], dim)} AS d${joins} LEFT JOIN (${inner}) AS a ON ${on}`)).select(select);
+    const bandBin = binOf.get(band);
+    const bandOn = bandBin ? binMatch(bandBin, `a.${column(band)}`) : `a.${column(band)} = d."id"`;
+    const on = [bandOn, ...cross.map(([key], i) => `a.${column(key)} = k${i}."id"`)].join(' AND ');
+    const source = bandBin ? binSource(bandBin) : `${padIds(plan.identity[band], dim)} AS d`;
+    return Query.from(rawSql(`${source}${joins} LEFT JOIN (${inner}) AS a ON ${on}`)).select(select);
+}
+/**
+ * Every bin, as `d.k`: an index for numeric bins, a bucket start for time
+ * buckets. Time buckets come from `time_bucket` itself over the field's
+ * extent, so they are exactly the buckets the query's own expression makes,
+ * months and years included, which are not a fixed width.
+ */
+function binSource(bin) {
+    if (bin.kind === 'number')
+        return `range(0, ${bin.count}) AS d(k)`;
+    const first = `time_bucket(${bin.interval}, epoch_ms(${Math.round(bin.minMs)}))`;
+    const last = `time_bucket(${bin.interval}, epoch_ms(${Math.round(bin.maxMs)}))`;
+    return `range(${first}, ${last} + ${bin.interval}, ${bin.interval}) AS d(k)`;
+}
+/**
+ * Bin `k`'s edge as the REAL the query draws. Numeric: `min + alpha * (k +
+ * offset)`, in DOUBLE arithmetic. Time: the bucket start plus `offset`
+ * intervals, as seconds from `t0`, the form the mark's time channels take.
+ */
+function binEdge(bin, k) {
+    if (bin.kind === 'number') {
+        return `CAST(${numericBinEdgeSQL(bin.min, bin.alpha, bin.offset, k)} AS REAL)`;
+    }
+    return `CAST((epoch(${k} + (${bin.interval}) * ${bin.offset}) - ${bin.t0}) AS REAL)`;
+}
+/**
+ * Whether the result row's edge `edge` is bin `d.k`. Numeric bins match by
+ * index, recovered from the edge: the edge is a REAL in the result (and in a
+ * preagg table), so comparing it to an edge computed here could miss by a
+ * rounding. A time edge is computed by the same expression on both sides, so
+ * the REALs are equal.
+ */
+function binMatch(bin, edge) {
+    if (bin.kind === 'number') {
+        return `round((CAST(${edge} AS DOUBLE) - ${sqlNumber(bin.min)}) / ${sqlNumber(bin.alpha)}) - ${bin.offset} = d.k`;
+    }
+    return `${edge} = ${binEdge(bin, 'd.k')}`;
+}
+/** A JS number as a DOUBLE literal: a bare 0.1 would be read as a DECIMAL. */
+function sqlNumber(n) {
+    return `CAST(${JSON.stringify(n)} AS DOUBLE)`;
 }
 /**
  * Most padded rows `dense()` will produce before refusing.
@@ -303,9 +357,11 @@ export const DENSE_ROW_LIMIT = 10_000;
  */
 function padIds(id, dim) {
     const present = id?.present;
+    // A mark that drops NULL pads no row for it: its rows were filtered out.
+    const notNull = id?.dropNullId != null ? ` WHERE "id" <> ${id.dropNullId}` : '';
     if (!present)
-        return `(SELECT "id" FROM ${quoteTable(dim)})`;
-    return (`(SELECT "id" FROM ${quoteTable(present)} UNION ALL ` +
+        return `(SELECT "id" FROM ${quoteTable(dim)}${notNull})`;
+    return (`(SELECT "id" FROM ${quoteTable(present)}${notNull} UNION ALL ` +
         `SELECT "id" FROM ${quoteTable(dim)} WHERE "label" IS NOT NULL)`);
 }
 /**
@@ -344,20 +400,24 @@ function densePlan(aliases, config, enabled, reported) {
         if (reported.has(key))
             return;
         reported.add(key);
-        console.info(`[dense] ${reason} Categories on this mark will snap between updates ` +
-            `rather than morphing, and a category leaving will vanish rather than shrink out.`);
+        console.info(`[dense] ${reason} Categories or bins on this mark will snap between updates ` +
+            `rather than morphing, and one leaving will vanish rather than shrink out.`);
     };
     if (!enabled)
         return null;
-    const identity = config?.identity;
-    if (!identity)
-        return null;
     // Stated by whoever built the select, because it cannot be recovered from
     // the query by the time this runs: topN has already wrapped it, and the
     // GROUP BY is several levels down.
     const groupKeys = config?.groupKeys;
     if (!groupKeys)
         return null;
+    // A histogram: its bins are enumerable like a dim's ids.
+    const binned = binPlan(aliases, groupKeys, config, say);
+    if (binned !== undefined)
+        return binned;
+    const identity = config?.identity;
+    if (!identity)
+        return null;
     const candidates = aliases.filter((ch) => identity[ch]?.positional);
     if (candidates.length !== 1) {
         if (candidates.length > 1) {
@@ -397,7 +457,59 @@ function densePlan(aliases, config, enabled, reported) {
             `(${size} categories${cross.length > 0 ? ` across ${cross.length + 1} group keys` : ''}).`);
         return null;
     }
-    return { band, dim, cross, identity };
+    return { band, dim, cross, bins: null, identity };
+}
+/**
+ * The plan for a binned axis: one row per bin, from the bins `bin()`
+ * resolved, crossed with each other group key's ids as for a categorical band.
+ * Null with nothing binned. Refuses, saying why, what it cannot enumerate: bins
+ * on both axes (a 2D histogram) or beside a categorical axis, a group key with
+ * no dim, and too many rows. Silent while identity has not landed for a group
+ * key that will get it.
+ */
+function binPlan(aliases, groupKeys, config, say) {
+    const specs = config?.bins;
+    if (!specs)
+        return undefined;
+    const bins = aliases
+        .filter((ch) => specs[ch] && groupKeys.includes(ch))
+        .map((ch) => [ch, specs[ch]])
+        .sort((a, b) => a[1].offset - b[1].offset);
+    if (bins.length === 0)
+        return undefined;
+    const identity = config?.identity ?? null;
+    if (new Set(bins.map(([ch]) => ch[0])).size > 1) {
+        say('Bins on both axes (a 2D histogram), and padding is defined for one.');
+        return null;
+    }
+    const categorical = identity ? aliases.filter((ch) => identity[ch]?.positional) : [];
+    if (categorical.length > 0) {
+        say(`A binned axis beside a categorical one (${categorical.join(', ')}).`);
+        return null;
+    }
+    const binned = new Set(bins.map(([ch]) => ch));
+    const cross = [];
+    let rows = bins[0][1].count;
+    for (const key of groupKeys) {
+        if (binned.has(key) || !aliases.includes(key))
+            continue;
+        const keyIdentity = identity?.[key];
+        // Before identity setup a series has no dim yet; it will.
+        if (!identity)
+            return null;
+        if (!keyIdentity) {
+            say(`the group key "${key}" has no dim, so its missing combinations cannot be enumerated.`);
+            return null;
+        }
+        cross.push([key, keyIdentity.dim]);
+        rows *= keyIdentity.size + 1;
+    }
+    if (rows > DENSE_ROW_LIMIT) {
+        say(`padding would produce about ${rows} rows, over the ${DENSE_ROW_LIMIT} limit ` +
+            `(${bins[0][1].count} bins${cross.length > 0 ? ` across ${cross.length + 1} group keys` : ''}).`);
+        return null;
+    }
+    return { band: bins[0][0], dim: '', cross, bins, identity: identity ?? {} };
 }
 /**
  * Return a row for every category, not only the ones that survived.
@@ -437,10 +549,27 @@ export function dense(options = {}) {
             return plan ? [plan.band, ...plan.cross.map(([key]) => key)] : null;
         },
         apply(query, config) {
+            // A preagg table built with padRows() below: its rows are already one
+            // per category or bin, and only the order is left, which the query's
+            // ORDER BY takes from alignment() as for any other read.
+            if (config?.padded)
+                return query;
             const aliases = getSelectAliases(query);
             const plan = densePlan(aliases, config, enabled, reported);
             return plan ? densify(query, aliases, plan) : query;
         },
+        /**
+         * The rows apply() would add, as the padding of an empty result: the same
+         * join and the same rules for every channel, so a table built with them
+         * holds exactly what padding its reads would have added.
+         */
+        padRows(aliases, config) {
+            const plan = densePlan(aliases, config, enabled, reported);
+            if (!plan)
+                return null;
+            const empty = Query.select(Object.fromEntries(aliases.map((alias) => [alias, rawSql('NULL')]))).where(rawSql('false'));
+            return String(densify(empty, aliases, plan));
+        },
         describe() {
             return 'Dense categories';
         }
@@ -806,7 +935,9 @@ export function stackX(options = {}) {
     return stackSQL('x', options);
 }
 /** Resolve band (group) / value column aliases for a stack direction, validated against the select list. */
-function resolveStackColumns(aliases, config, direction) {
+function resolveStackColumns(aliases, config, direction, 
+/** Called when there is no series column; the caller decides whether that is worth saying. */
+onNoSeries) {
     const bandAxis = direction === 'y' ? 'x' : 'y';
     const band = config?.[bandAxis];
     const value = config?.[direction];
@@ -820,20 +951,31 @@ function resolveStackColumns(aliases, config, direction) {
     }
     const fill = config?.fill;
     const series = typeof fill === 'string' && aliases.includes(fill) ? fill : null;
-    // With no series column there is one row per band, so the cumulative sum equals
-    // that row's own value and the stack is arithmetically a no-op — the output is
-    // identical to an unstacked mark. That is the correct result, but it is
-    // indistinguishable from a broken stack to anyone looking at the chart, so say
-    // what is missing rather than silently drawing an ordinary bar.
-    if (!series) {
-        console.warn(`stack${direction === 'y' ? 'Y' : 'X'}: no series column, so stacking has no effect ` +
-            `(one row per band). Stacking groups rows within a band by the ordinal ` +
-            `fill channel — add fill="<field>" to the mark. Select list: [${aliases.join(', ')}].`);
-    }
+    if (!series)
+        onNoSeries(aliases);
     return { band, value, series };
 }
-function stackSQL(direction, options) {
+/**
+ * `implicit`: added by the mark (withImplicitStack), not written by the user.
+ *
+ * With no series column there is one row per band, so the cumulative sum equals
+ * that row's own value and the stack is arithmetically a no-op. For a stack the
+ * user wrote, that is indistinguishable from a broken one on screen, so it says
+ * what is missing, once per transform rather than on every query assembled. An
+ * implicit stack says nothing: every bar mark without a fill gets one, and it is
+ * the ordinary bar it draws, not a mistake.
+ */
+function stackSQL(direction, options, implicit = false) {
     const [loName, hiName] = direction === 'y' ? ['y0', 'y1'] : ['x0', 'x1'];
+    let warned = implicit;
+    const onNoSeries = (aliases) => {
+        if (warned)
+            return;
+        warned = true;
+        console.warn(`stack${direction === 'y' ? 'Y' : 'X'}: no series column, so stacking has no effect ` +
+            `(one row per band). Stacking groups rows within a band by the ordinal ` +
+            `fill channel — add fill="<field>" to the mark. Select list: [${aliases.join(', ')}].`);
+    };
     return {
         stage: 'sql',
         type: 'stack',
@@ -844,7 +986,7 @@ function stackSQL(direction, options) {
         channels: [loName, hiName],
         apply(query, config) {
             const aliases = getSelectAliases(query);
-            const { band, value, series } = resolveStackColumns(aliases, config, direction);
+            const { band, value, series } = resolveStackColumns(aliases, config, direction, onNoSeries);
             const valueCol = column(value);
             const bandCol = column(band);
             const notNaN = sqlTag `NOT isnan(${valueCol})`;
@@ -954,12 +1096,6 @@ function stackXJS(options) {
         }
     };
 }
-/**
- * Core stacking computation on DataAccessor columns.
- *
- * Single O(n) pass: detects group boundaries in the sorted group column,
- * accumulates value offsets, writes lo/hi Float32Arrays, applies offset mode.
- */
 /**
  * Core stacking computation on DataAccessor columns.
  *
@@ -1185,9 +1321,7 @@ export function withImplicitStack(transforms, hasExplicitBounds, direction) {
         ? direction === 'y'
             ? stackY({ engine: 'js' })
             : stackX({ engine: 'js' })
-        : direction === 'y'
-            ? stackY()
-            : stackX();
+        : stackSQL(direction, {}, true);
     return [...transforms, implicitStack];
 }
 export { TransformCompiler } from './TransformCompiler';