@toclocoinc/lattice-grid 1.16.0 → 1.17.0

package/docs/api-detail.html

@@ -437,7 +437,7 @@
 <div class="shell">
   <aside class="rail">
     <p class="rail__brand">Lattice Grid</p>
-    <p class="rail__sub">Developer guide · v1.16.0</p>
+    <p class="rail__sub">Developer guide · v1.17.0</p>
     <nav>
       <div class="rail__group">
         <span class="rail__label">Start here</span>
@@ -2554,9 +2554,9 @@ grid.destroy();
     </p>
 
     <div class="example">
-      <p class="example__label">On by default; turn it off or change the cap</p>
-      <pre><code>createGrid(element, { stickyGroupHeaders: <span class="kw">false</span> });   <span class="cmt">// off</span>
-createGrid(element, { stickyGroupHeaders: 3 });       <span class="cmt">// stack up to three</span></code></pre>
+      <p class="example__label">Off by default; opt in or set the cap</p>
+      <pre><code>createGrid(element, { stickyGroupHeaders: <span class="kw">true</span> });    <span class="cmt">// on, up to two</span>
+createGrid(element, { stickyGroupHeaders: 3 });       <span class="cmt">// on, up to three</span></code></pre>
     </div>
 
     <div class="why">
@@ -2608,7 +2608,8 @@ createGrid(element, { stickyGroupHeaders: 3 });       <span class="cmt">// stack
       <table>
         <thead><tr><th>Option</th><th>Type</th><th>Description</th></tr></thead>
         <tbody>
-          <tr><td class="name">retainSource</td><td class="type">boolean</td><td class="desc">Default <code>true</code>. Set <code>false</code> to keep only the packed columns and reconstruct a plain row object from them on demand. <code>rows.data()</code> then returns freshly reconstructed objects — a new object each call — so <code>row === sourceObject</code> and a custom renderer reading <code>row.sourceObject</code> no longer hold, and equality becomes value-based. Cell values are identical either way, so <code>get()</code>, <code>byKey()</code>, <code>value()</code> and <code>values()</code> are unaffected.</td></tr>
+          <tr><td class="name">retainSource</td><td class="type">boolean</td><td class="desc">Default <code>true</code>. Set <code>false</code> to keep only the packed columns and reconstruct a plain row object from them on demand. <code>rows.data()</code> then returns freshly reconstructed objects — a new object each call — so <code>row === sourceObject</code> and a custom renderer reading <code>row.sourceObject</code> no longer hold, and equality becomes value-based. Cell values are identical either way, so <code>get()</code>, <code>byKey()</code>, <code>value()</code> and <code>values()</code> are unaffected. On its own this drops only the store's reference array, not the objects: the source layer still holds them.</td></tr>
+          <tr><td class="name">dropSourceRows</td><td class="type">boolean</td><td class="desc">Default <code>false</code>. Set <code>true</code> to release the caller's row objects from the <em>source layer</em> and the grid config once the store is built, so the packed columns are the only resident copy. This is where the large reduction lives — roughly an order of magnitude at a million rows — because the caller's objects are the dominant term. Implies <code>retainSource: false</code> (the store must reconstruct), so it carries the same identity tradeoff. An impure computed column — a shadow or a rank/positional column — is never stored, so it cannot be served from the columns: it warns once and drops out rather than answering wrongly, and <code>rows.move()</code> is refused. Do not enable it on a grid that sorts, filters, groups or totals on such a column, or that reorders rows in place.</td></tr>
         </tbody>
       </table>
     </div>
@@ -2618,14 +2619,17 @@ createGrid(element, { stickyGroupHeaders: 3 });       <span class="cmt">// stack
   columns,
   rowKey: 'id',
   rows,
-  ingest: { retainSource: <span class="kw">false</span> },
+  <span class="cmt">// Release the caller's objects entirely; the packed columns are the sole copy.</span>
+  ingest: { dropSourceRows: <span class="kw">true</span> },
 });</code></pre>
     </div>
     <div class="why">
-      <p>The saving is the store's own copy of the object references, not the objects
-      themselves: whoever handed the grid its rows still owns them. The footprint drops
-      materially only when the grid becomes the sole holder of the data. Reach for this when
-      you can let go of the source array and do not depend on caller identity through
+      <p>With <code>retainSource: false</code> alone the saving is only the store's own copy of the
+      object references, not the objects themselves: the source layer and the grid config still
+      hold the array, so whoever handed the grid its rows keeps them alive. <code>dropSourceRows</code>
+      releases those references too, so once the caller lets go the objects can be collected and the
+      grid becomes the sole holder of the data. Reach for it on a large, read-mostly grid where you
+      can let go of the source array and do not depend on caller identity through
       <code>rows.data()</code>.</p>
     </div>
 
@@ -2746,7 +2750,8 @@ createGrid(el, { columns, rowKey: 'id', source });
 plan.pushed;      <span class="cmt">// the query the adapter was given</span>
 plan.residual;    <span class="cmt">// { filters, sort, quick } the grid applied after</span>
 plan.unpushed;    <span class="cmt">// ['filter'], the parts that stayed behind</span>
-plan.needsAll;    <span class="cmt">// true when the whole result had to be fetched</span></code></pre>
+plan.needsAll;    <span class="cmt">// true when the whole result had to be fetched</span>
+plan.full;        <span class="cmt">// true when fullDataset forced it, not just residual work</span></code></pre>
     </div>
     <p>
       The grid also warns once, naming the predicate that could not be pushed, because the fix is
@@ -2788,6 +2793,78 @@ createGrid(el, {
       with no server involved.</p>
     </div>
 
+    <h3 id="fulldataset">Whole-dataset statistics over a remote source</h3>
+    <p class="lead-in">
+      A windowed source reduces a total or statistic over the rows it has loaded, not the whole
+      matching set — a footer median of the 200 rows on screen, which is wrong and looks right.
+      <code>fullDataset</code> makes the source hold the entire matching set client-side so those
+      figures are computed over everything.
+    </p>
+    <div class="example">
+      <p class="example__label">Correct footer figures over a REST or DuckDB source</p>
+      <pre><code><span class="kw">const</span> source = createPushdownSource({
+  adapter: restAdapter({ url: '/api/trades' }),
+  fullDataset: {
+    enabled: <span class="kw">true</span>,            <span class="cmt">// hold the whole matching set, once per query</span>
+    maxRows: 1_000_000,        <span class="cmt">// refuse (visible error) past this</span>
+    maxBytesEstimate: 512 * 1024 * 1024,
+  },
+});</code></pre>
+    </div>
+    <div class="why">
+      <p>It is <strong>off by default</strong> and reuses the same whole-result path that residual
+      work already takes: the flag ORs into <code>needsAll</code>, so once the set is held it is
+      ordinary in-memory data and the grid's existing total and statistics kernels reduce over all of
+      it, with no per-stat change. When it is active and within the limits, the whole matching set is
+      covered, so the windowed-statistic warning (BACKLOG-0000731) stays silent — the figure is now
+      honestly whole-dataset.</p>
+      <p><strong>It is refused loudly, never truncated.</strong> A matching set past
+      <code>maxRows</code> or <code>maxBytesEstimate</code> is thrown and surfaced as a
+      <code>source:error</code> with no rows shown, rather than held as a fraction and presented as
+      the whole. A fraction shown as the whole is exactly the silent wrong answer this feature
+      exists to remove, so it is never how the feature fails. For a <code>restAdapter</code>, which
+      cannot compute, this is the only route to a correct whole-dataset statistic.</p>
+    </div>
+
+    <h3 id="aggregates">Pushing statistics down to the engine</h3>
+    <p class="lead-in">
+      A DuckDB-class engine computes a median or a standard deviation over the whole matching set far
+      faster than pulling every row here to do it. The <code>aggregates</code> config decides, at
+      grid setup, which statistics the engine computes and which the grid does — a design-time
+      developer choice, fixed for the life of the grid, never a runtime toggle and never shown to an
+      end user.
+    </p>
+    <div class="example">
+      <p class="example__label">Push the verified-identical stats, keep the rest exact</p>
+      <pre><code><span class="kw">const</span> source = createPushdownSource({
+  adapter: duckdbAdapter({ connection: conn, from: <span class="str">'trades'</span> }),
+  aggregates: {
+    <span class="cmt">// 'engine' pushes everything expressible; 'engine-if-identical' pushes only</span>
+    <span class="cmt">// the stats whose engine result is verified identical to the grid kernel;</span>
+    <span class="cmt">// 'client' (the default when absent) computes everything here.</span>
+    <span class="kw">default</span>: <span class="str">'engine-if-identical'</span>,
+    overrides: { mode: <span class="str">'client'</span> },   <span class="cmt">// I want the grid's null-when-distinct mode</span>
+  },
+});</code></pre>
+    </div>
+    <div class="why">
+      <p>Every statistic is classified <strong>IDENTICAL</strong> (the engine's result equals the
+      grid's own kernel, verified against it on the same data) or <strong>MAY-DIFFER</strong> (the
+      engine computes it by a method that can differ from the grid's definition — <code>mode</code>
+      returns a value where the grid returns null). The classification drives the docs and
+      build-time provenance, <em>not</em> whether a stat is pushed: that is your choice.
+      <code>weightedQuantile</code> is the one genuine fallback, always client-side, because the
+      engine cannot express the grid's midpoint convention. The published table of every stat, its
+      class and its SQL is generated from one map (<code>STAT_PUSHDOWN</code>) so it cannot drift.</p>
+      <p><strong>No mixed provenance.</strong> An engine figure and a client figure never appear in
+      one result set. Aggregates are pushed only when the filter is <em>fully</em> pushed; a residual
+      filter the engine could not apply forces every aggregate client-side, because an engine number
+      computed over a superset beside a client number over the real set would be wrong-but-plausible.
+      <code>source.lastPlan().aggregates</code> reports, per stat, whether the engine or the grid
+      computed it and the class it was assigned — inspection during the build, not a per-figure
+      runtime marker.</p>
+    </div>
+
     <h2 id="derived">Grids built from other grids</h2>
     <p class="lead-in">
       A derived grid takes its rows from another grid rather than from a load: grouped and
@@ -2982,6 +3059,27 @@ columns: [
       scrolling past a rounded corner is cut by it rather than squaring it off.
     </p>
 
+    <div class="example">
+      <p class="example__label">Zebra striping (opt-in)</p>
+      <pre><code>createGrid(element, {
+  columns, rows,
+  stripedRows: <span class="kw">true</span>,       <span class="cmt">// shade alternate data rows; off by default</span>
+});</code></pre>
+    </div>
+
+    <p class="lead-in">
+      <code>stripedRows</code> shades every other data row. It is strictly opt-in and off by default,
+      so a grid that never mentions it looks exactly as it did on upgrade. Parity is decided by each
+      row's <em>logical</em> index rather than its position in the DOM: rows are virtualised and
+      recycled, so a <code>:nth-child</code> rule would repaint the stripe onto whichever row landed
+      in an odd slot after a scroll, and a logical-index stripe keeps a row shaded consistently across
+      a scroll and across the left-pinned, centre and right-pinned segments of the same row. Group
+      headings, group footers and the grand total are structure rather than data, so they are never
+      striped. The stripe uses the theme's <code>--lattice-surface-alt</code> token, which every
+      palette defines, so dark, high-contrast and terminal are correct without any extra rule, and
+      both selection and hover still win over it.
+    </p>
+
     <h2 id="cards">Cards, lists and feeds</h2>
     <p class="lead-in">
       <code>rowTemplate</code> draws each row with a layout of your own instead of dividing it into

package/lattice-grid.d.ts

@@ -1,5 +1,5 @@
 /*!
- * Lattice Grid 1.16.0, type declarations
+ * Lattice Grid 1.17.0, type declarations
  * Copyright (c) 2026 TOCLOCO Inc. All rights reserved.
  * https://latticegrid.dev
  */
@@ -779,6 +779,41 @@ export interface IngestConfig {
    */
   retainSource?: boolean;
 
+  /**
+   * Release the caller's row objects from the *source layer* once the column
+   * store has been built, so the columns become the sole resident copy of the
+   * data. Default `false`, which keeps today's behaviour.
+   *
+   * `retainSource:false` stops the {@link https://en.wikipedia.org/wiki/Column-oriented_DBMS column store}
+   * from holding the caller's objects, but the memory source and the grid config
+   * still retain the supplied array by reference — so the objects stay alive and
+   * the resident footprint does not actually fall. This flag closes that gap: it
+   * clears `MemorySource`'s retained array and drops the array from the grid
+   * config, leaving nothing on the heap but the packed columns. That is where
+   * the large reduction comes from (roughly an order of magnitude at a million
+   * rows), not from `retainSource` on its own.
+   *
+   * Implies `retainSource:false`: dropping the caller's objects while the store
+   * still expects to read through them would leave the source with no data at
+   * all, so setting this on forces the store to reconstruct rows from columns.
+   * Every read is therefore served from the columns — `at()`, `byKey()`,
+   * `get()`, `value()`, `values()`, filtering, sorting, grouping, totals and
+   * export are all unaffected in their values. What changes is the same three
+   * identity behaviours `retainSource:false` documents: `rows.data()` returns
+   * freshly reconstructed objects (so `row === sourceObject` no longer holds), a
+   * custom renderer reaching for `row.sourceObject` gets a reconstruction, and
+   * equality against a row becomes value-based.
+   *
+   * One consumer cannot be served from the columns: an *impure computed column*
+   * (a shadow, or a rank/positional column) is deliberately never materialised
+   * into the store, so its handle is built by reading the source objects. Under
+   * `dropSourceRows` those objects are gone, so such a column reduces over
+   * nothing and warns once rather than returning a silently wrong figure. Do not
+   * enable `dropSourceRows` on a grid that sorts, filters, groups or totals on a
+   * shadow or a positional column.
+   */
+  dropSourceRows?: boolean;
+
   /**
    * Columnize `stream`-source ingest on a Worker so a large load does not block
    * the main thread. Default `false`. When on, an arriving chunk that clears
@@ -1158,6 +1193,21 @@ export interface GridConfig {
    */
   cornerRadius?: boolean | number | string;
 
+  /**
+   * Shade alternate data rows (zebra striping).
+   *
+   * Off by default, and strictly opt-in: an existing grid must look exactly the
+   * same on upgrade. When `true`, every other data row takes the theme's
+   * `--lattice-surface-alt` background, which every palette already defines, so
+   * dark, high-contrast and terminal stripe correctly without extra work.
+   *
+   * Parity follows the row's *logical* index, not its position in the DOM, so a
+   * row keeps its stripe across a scroll even though the rows are recycled.
+   * Structural rows — group headings, group footers and the grand total — are
+   * never striped, and both selection and hover still win over the stripe.
+   */
+  stripedRows?: boolean;
+
   /**
    * Show a bar above the column headings for filtering columns by tag.
    *
@@ -1312,7 +1362,12 @@ export interface GridConfig {
    * only the sort, filter and menu controls inside them.
    */
   showHeader?: boolean;
-  /** Header height in pixels. */
+  /**
+   * Header height in pixels. Omitted, the header takes its height from the
+   * density-scaled `--lattice-header-height` token, so `density` sizes the
+   * header as it sizes the rows. A number names one explicitly and outranks the
+   * token.
+   */
   headerHeight?: number;
   /** How many rows to render beyond the viewport. More costs memory and
    * smooths fast scrolling; fewer is lighter and can show a gap. */
@@ -1562,9 +1617,11 @@ export interface GridConfig {
    * Keep the enclosing group headings pinned above the viewport while
    * scrolling inside a group.
    *
-   * On by default, stacking at most two. `false` turns it off; a number, or
-   * `{ depth }`, sets how many may stack: each costs a row of viewport, so a
-   * deep grouping would otherwise spend the screen describing itself.
+   * Off by default — a deliberate product default; sticky group headers are
+   * opt-in. `true` turns it on, stacking at most two; a number, or `{ depth }`,
+   * sets how many may stack: each costs a row of viewport, so a deep grouping
+   * would otherwise spend the screen describing itself. `false` is off, the
+   * same as leaving it unset.
    */
   stickyGroupHeaders?: boolean | number | { depth?: number };
   /**
@@ -1892,6 +1949,103 @@ export interface PushdownPlan {
   needsAll: boolean;
   /** Which parts could not be pushed: `filter`, `sort`, `quick`. */
   unpushed: string[];
+  /**
+   * Whether the whole result was fetched because `fullDataset` is on, rather
+   * than only because residual work forced it. When true, totals and statistics
+   * reduce over the whole matching set and the windowed-stat warning is silent.
+   */
+  full: boolean;
+  /**
+   * Per-aggregate provenance, present only when the last request computed
+   * aggregates (BACKLOG-0000730 Part B): which statistics the engine computed
+   * and which the client did, with the class the pushdown map assigned each.
+   * Under grouping it also carries the `groupBy` the subtotals were computed
+   * over. Build-time inspection, not a runtime per-figure marker.
+   */
+  aggregates?: {
+    engine: AggregateProvenance[];
+    client: AggregateProvenance[];
+    groupBy?: string[];
+  };
+}
+
+/**
+ * Opt-in, sticky full-dataset pull for a pushdown/remote source
+ * (BACKLOG-0000730). Off by default. When enabled, the source materialises the
+ * entire matching set client-side once per query signature and serves every
+ * window, total and statistic from it, so those figures are computed over the
+ * whole set rather than the loaded window. A set past either limit is refused
+ * with a visible `source:error` — never silently truncated.
+ */
+export interface PushdownFullDatasetConfig {
+  /** Sticky: hold the whole matching set client-side. Default `false`. */
+  enabled?: boolean;
+  /** Refuse (visible error) past this many rows. Default `1_000_000`. */
+  maxRows?: number;
+  /** Refuse past this estimated heap cost, in bytes. Default `512 * 1024 * 1024`. */
+  maxBytesEstimate?: number;
+}
+
+/** How one requested aggregate should be computed. */
+export type AggregateMode = 'engine' | 'client' | 'engine-if-identical';
+
+/**
+ * Design-time aggregate-pushdown policy for a pushdown source
+ * (BACKLOG-0000730 Part B, ungrouped). The developer chooses, at grid setup
+ * before render, whether each statistic is computed by the engine (fast, over
+ * the matching set) or client-side (the grid's exact definition, needs a
+ * full-dataset pull). It is fixed for the life of the grid, never a runtime
+ * toggle, and never surfaced to an end user.
+ *
+ * Absent, every aggregate is computed client-side — today's behaviour, so no
+ * existing caller regresses. `engine-if-identical` is the recommended setting
+ * for a windowed DuckDB source: it pushes only the statistics whose engine
+ * result is verified identical to the grid kernel, keeping the documented
+ * MAY-DIFFER stats (e.g. `mode`) client-side. The engine is used only when the
+ * filter is fully pushed; a residual filter forces every aggregate client-side,
+ * so an engine figure and a client figure never mix in one result set.
+ */
+export interface PushdownAggregatesConfig {
+  /**
+   * The default policy for stats the engine can express. `'engine'` pushes
+   * everything expressible (using the engine's method for MAY-DIFFER stats);
+   * `'engine-if-identical'` pushes only the verified-identical ones; `'client'`
+   * computes everything client-side. Default `'client'`.
+   */
+  default?: AggregateMode;
+  /** Per-stat overrides, winning over `default`. A stat the engine cannot
+   *  express (`weightedQuantile`) is always client-side regardless. */
+  overrides?: Record<string, 'engine' | 'client'>;
+}
+
+/**
+ * One aggregate the grid asks the source to compute over the matching set.
+ * `params` carries e.g. `{ share: 0.1 }` so an adapter emits the matching SQL;
+ * `weight` names the second column for a two-column stat like `correlation`.
+ */
+export interface AggregateRequest {
+  /** Keys the result back to the request. */
+  id: string;
+  /** The column to reduce. */
+  col: string;
+  /** The statistic name, as used in `total: '<name>'`. */
+  fn: string;
+  /** The second column, for a two-column statistic. */
+  weight?: string;
+  /** Parameters the statistic takes, e.g. a trim share. */
+  params?: Record<string, unknown>;
+}
+
+/** How one aggregate was routed, for `lastPlan()` provenance. */
+export interface AggregateProvenance {
+  id: string;
+  col: string;
+  fn: string;
+  /** How the engine result relates to the grid kernel. */
+  class: 'identical' | 'may-differ' | 'fallback';
+  /** Why it is client-side, when it is (config, fallback, or the guard). */
+  reason?: string;
+  weight?: string;
 }
 
 export interface PushdownSourceConfig {
@@ -1899,6 +2053,16 @@ export interface PushdownSourceConfig {
   /** The compute barrel, for applying whatever the engine could not. */
   compute?: object;
   pageSize?: number;
+  /**
+   * Opt-in full-dataset pull. Off unless `fullDataset.enabled` is set. See
+   * {@link PushdownFullDatasetConfig}.
+   */
+  fullDataset?: PushdownFullDatasetConfig;
+  /**
+   * Design-time aggregate-pushdown policy. Absent = client-side (today's
+   * behaviour). See {@link PushdownAggregatesConfig}.
+   */
+  aggregates?: PushdownAggregatesConfig;
 }
 
 export interface StatisticsApi {
@@ -3414,7 +3578,51 @@ export function toneOf(direction: string, goodWhen: string): 'good' | 'bad' | 'f
  */
 export function createPushdownSource(
   config: PushdownSourceConfig,
-): SourceConfig & { lastPlan(): PushdownPlan | null };
+): SourceConfig & {
+  lastPlan(): PushdownPlan | null;
+  /**
+   * Compute a set of aggregates over the matching set, splitting them between
+   * the engine and the client by the design-time `aggregates` config
+   * (BACKLOG-0000730 Part B). Ungrouped, returns the engine-computed `values`
+   * keyed by id. When the request carries a `groupBy`, returns `groups` instead:
+   * one entry per subtotal level and the grand total (`level: 0`, produced by a
+   * single `GROUP BY ROLLUP`), each with its key values and its aggregate values
+   * keyed by id. The client list is what the caller computes itself over the
+   * full set. Aggregates are pushed only when the filter is fully pushed and —
+   * under grouping — every grouping key is a plain column the engine can group
+   * by; a residual filter or an unpushable group key forces every aggregate
+   * client-side (no mixed provenance).
+   */
+  aggregate(
+    request: RemoteRequest,
+    requested: AggregateRequest[],
+  ): Promise<{
+    values: Record<string, unknown>;
+    groups?: Array<{
+      keys: unknown[];
+      grouping?: number[];
+      level: number;
+      values: Record<string, unknown>;
+    }>;
+    engine: AggregateProvenance[];
+    client: AggregateProvenance[];
+  }>;
+};
+
+/**
+ * The pushdown map (BACKLOG-0000730 Part B): one published record per statistic
+ * giving whether the engine can express it, the DuckDB aggregate SQL it emits,
+ * and whether that result is IDENTICAL to the grid's own kernel or MAY-DIFFER.
+ * The single source of truth the push router, the docs and `lastPlan()` all read.
+ */
+export const STAT_PUSHDOWN: Readonly<Record<string, {
+  pushable: boolean;
+  class: 'identical' | 'may-differ' | 'fallback';
+  sql?: string;
+  note?: string;
+  twoColumn?: boolean;
+  blankAware?: boolean;
+}>>;
 
 /**
  * The capability set an adapter that declares nothing is treated as having: