@talelk/newui-data 1.0.0 → 1.1.0

package/README.md

@@ -21,6 +21,15 @@ The first four surfaces are the ones the examples had already copied:
 | `undo` | `createUndo` — a snapshot, a restore and a clear, without the page's notice copy |
 | `download` | `downloadBlob`, `downloadText`, `downloadCsv`, `csvText` — the Blob/anchor/revoke dance and RFC-shaped quoting |
 
+The analytics primitives, decided in issues #22, #23, #24 and #32, round out the
+headless layer:
+
+| surface | what it owns |
+| --- | --- |
+| `url-state` | `createUrlState` — link-reproducible signals behind a router adapter, replace-not-push, debounced search keys |
+| `config-store` | `createConfigStore` — versioned envelopes for saved views and dashboard config over a storage seam |
+| `aggregate` | `aggregate`, `timeBucket`, `rollup` — pure group-by and time-series transforms over rows the app already fetched |
+
 ## Query
 
 ```tsx
@@ -51,5 +60,7 @@ has shipped.
 The Frappe DocType client: `request`/`frappeRequest` transport with
 `setConfig`, document and list hooks (`useDoc`, `useDoctype`, `useList`,
 `useNewDoc`, `useCall`), an IndexedDB cache, and a shared WebSocket realtime
-multiplexer. It lives behind its own entry so importing `usd` never pulls a
-fetch stack, and it is app-specific — generic query needs belong on `/query`.
+multiplexer (`createRealtime`, plus `connectRealtime` to invalidate query keys
+from socket events). It lives behind its own entry so importing `usd` never
+pulls a fetch stack, and it is app-specific — generic query needs belong on
+`/query`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@talelk/newui-data",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "type": "module",
   "description": "The logic layer: formatting, selection, undo and downloads — no components.",
   "repository": {

package/skills/newui-data/SKILL.md

@@ -1,15 +1,17 @@
 ---
 name: newui-data
 description: Use the headless newui data layer for SolidJS collections,
-  repository queries, optimistic mutations, TanStack Query caching and form
-  submit lifecycles. Use when a page needs to load a list, create/update/delete
-  records, share a cache, or wire TanStack Form through the kit's Form and
-  field-error contract.
+  repository queries, optimistic mutations, TanStack Query caching, form submit
+  lifecycles, URL-synced state, saved-view/dashboard-config storage and
+  aggregation/rollup transforms. Use when a page needs to load a list,
+  create/update/delete records, share a cache, sync filters or views to the URL,
+  persist saved views, build chart series, or wire TanStack Form through the
+  kit's Form and field-error contract.
 metadata:
   type: core
   domain: data
   library: '@talelk/newui-data'
-  library_version: '1.0.0'
+  library_version: '1.1.0'
 sources:
   - talelk/newui:spec/data.md
   - talelk/newui:skills/README.md
@@ -18,6 +20,10 @@ sources:
   - talelk/newui:packages/data/src/query/collection.ts
   - talelk/newui:packages/data/src/query/repository.ts
   - talelk/newui:packages/data/src/form/form.tsx
+  - talelk/newui:packages/data/src/url-state.ts
+  - talelk/newui:packages/data/src/config-store.ts
+  - talelk/newui:packages/data/src/aggregate.ts
+  - talelk/newui:packages/data/src/resource/connect-realtime.ts
 ---
 
 # newui-data
@@ -114,3 +120,51 @@ const { form, submitting } = createAppForm({
   de-duplicates the same validator message.
 - Pass the same message to `onChange` and `onSubmit` when the field should
   validate on both; `fieldError` will not print it twice.
+
+### Config forms: the field map, pruning and autosave
+
+- `createAppForm({ schema, ui, pruneHidden })` takes a Standard Schema and a
+  **field map** (`ui`) keyed by *static* paths (`smtp.host`, never `items[1]`
+  — that throws; a repeater's row fields go under its `fields`). Pass the same
+  object to the `ConfigForm` organism's `fields`.
+- `when(scope, values)` hides a field: `scope` is the form values for a
+  top-level entry and **the row** for a repeater's row field. Hidden fields
+  are pruned from the submitted document (`pruneHidden` defaults to `true`),
+  and schema issues under them never block submit.
+- `app.document(values)` is that submitted document. Give
+  `createAutoSave(app, { save, id })` the `createAppForm` result, not
+  `app.form`, so autosave writes the same pruned document submit sends.
+- With `id`, autosave keeps a local, unpruned **draft**; a successful autosave
+  or submit drops it. `restoreDraft()` is untrusted input (it may predate the
+  schema) — reset the form with it and let validation report what no longer
+  fits.
+
+## URL state, saved views and transforms
+
+```ts
+import { createConfigStore, createUrlState, rollup } from '@talelk/newui-data'
+
+// Link-reproducible state; the app owns the router through `adapter`.
+const filters = createUrlState(
+  { 'board.view': 'kanban', q: '' },
+  { adapter, debounce: ['q'] },
+)
+filters['board.view']()
+filters.set('q', 'invoice') // undefined = absent, never ""
+
+// Saved views and dashboard configs: versioned envelopes, storage-agnostic.
+const views = createConfigStore({ key: 'acme.views' })
+views.save('week-only', { filters: [] }, { type: 'saved-view' })
+
+// Chart series from a time column — pure, no query, no components.
+const series = rollup(rows, { field: 'createdAt', unit: 'day', metric: { field: 'amount', metric: 'sum' } })
+```
+
+- `createUrlState` reads the URL once on mount and **replaces** (never pushes);
+  without an adapter it is a plain signal, so tests and SSR stay clean.
+- `createConfigStore` documents are capped at 64KB with a dev warning; a
+  server-backed adapter drops in by passing `storage`, no consumer changes.
+- `aggregate`/`rollup` run over rows the app already fetched — past that size
+  the answer is server-side, not a client pivot.
+- A collection can poll with `refetchInterval`; `connectRealtime` (from
+  `@talelk/newui-data/resource`) invalidates query keys from socket events.

package/src/aggregate.test.ts

@@ -0,0 +1,231 @@
+import { describe, expect, it } from 'vitest'
+import { type AggregateRow, aggregate, rollup, timeBucket } from './aggregate'
+
+/** Local-time dates so the buckets do not depend on the runner's zone. */
+const date = (year: number, month: number, day: number, hour = 0, min = 0) =>
+  new Date(year, month - 1, day, hour, min)
+
+describe('aggregate', () => {
+  const rows: AggregateRow[] = [
+    { region: 'north', amount: 10, tag: 'a' },
+    { region: 'north', amount: 20, tag: 'a' },
+    { region: 'south', amount: 5, tag: 'b' },
+  ]
+
+  it('groups by one column and computes sum and count', () => {
+    const out = aggregate(rows, {
+      by: 'region',
+      metrics: {
+        total: { field: 'amount', metric: 'sum' },
+        n: { field: 'amount', metric: 'count' },
+      },
+    })
+    expect(out).toEqual([
+      { region: 'north', total: 30, n: 2 },
+      { region: 'south', total: 5, n: 1 },
+    ])
+  })
+
+  it('groups by several columns', () => {
+    const out = aggregate(
+      [
+        { region: 'north', tag: 'a', amount: 1 },
+        { region: 'north', tag: 'b', amount: 2 },
+        { region: 'north', tag: 'a', amount: 3 },
+      ],
+      {
+        by: ['region', 'tag'],
+        metrics: { total: { field: 'amount', metric: 'sum' } },
+      },
+    )
+    expect(out).toEqual([
+      { region: 'north', tag: 'a', total: 4 },
+      { region: 'north', tag: 'b', total: 2 },
+    ])
+  })
+
+  it('computes min, max and extent', () => {
+    const out = aggregate(rows, {
+      by: 'region',
+      metrics: {
+        low: { field: 'amount', metric: 'min' },
+        high: { field: 'amount', metric: 'max' },
+        range: { field: 'amount', metric: 'extent' },
+      },
+    })
+    expect(out[0]).toEqual({
+      region: 'north',
+      low: 10,
+      high: 20,
+      range: [10, 20],
+    })
+  })
+
+  it('computes mean and median over numeric values only', () => {
+    const out = aggregate(
+      [
+        { g: 'x', amount: 10 },
+        { g: 'x', amount: 'nope' },
+        { g: 'x', amount: 30 },
+      ],
+      {
+        by: 'g',
+        metrics: {
+          avg: { field: 'amount', metric: 'mean' },
+          mid: { field: 'amount', metric: 'median' },
+        },
+      },
+    )
+    expect(out[0]?.['avg']).toBe(20)
+    expect(out[0]?.['mid']).toBe(20)
+
+    const even = aggregate(
+      [
+        { g: 'x', amount: 1 },
+        { g: 'x', amount: 3 },
+      ],
+      { by: 'g', metrics: { mid: { field: 'amount', metric: 'median' } } },
+    )
+    expect(even[0]?.['mid']).toBe(2)
+  })
+
+  it('computes unique and uniqueCount in first-seen order', () => {
+    const out = aggregate(rows, {
+      by: 'region',
+      metrics: {
+        tags: { field: 'tag', metric: 'unique' },
+        tagCount: { field: 'tag', metric: 'uniqueCount' },
+      },
+    })
+    expect(out[0]?.['tags']).toEqual(['a'])
+    expect(out[0]?.['tagCount']).toBe(1)
+    const all = aggregate(rows, {
+      by: [],
+      metrics: { tagCount: { field: 'tag', metric: 'uniqueCount' } },
+    })
+    expect(all).toHaveLength(1)
+    expect(all[0]?.['tagCount']).toBe(2)
+  })
+
+  it('takes first and last values in row order', () => {
+    const out = aggregate(rows, {
+      by: 'region',
+      metrics: {
+        opener: { field: 'tag', metric: 'first' },
+        closer: { field: 'tag', metric: 'last' },
+      },
+    })
+    expect(out[0]?.['opener']).toBe('a')
+    expect(out[0]?.['closer']).toBe('a')
+    expect(out[1]?.['opener']).toBe('b')
+  })
+
+  it('groups dates by their instant, not their object identity', () => {
+    const out = aggregate(
+      [
+        { day: date(2026, 1, 5), amount: 1 },
+        { day: new Date(date(2026, 1, 5).getTime()), amount: 2 },
+      ],
+      { by: 'day', metrics: { total: { field: 'amount', metric: 'sum' } } },
+    )
+    expect(out).toHaveLength(1)
+    expect(out[0]?.['total']).toBe(3)
+  })
+
+  it('handles empty input and missing fields', () => {
+    expect(
+      aggregate([], { by: 'a', metrics: { s: { field: 'b', metric: 'sum' } } }),
+    ).toEqual([])
+    const out = aggregate([{ g: 'x' }], {
+      by: 'g',
+      metrics: {
+        sum: { field: 'missing', metric: 'sum' },
+        min: { field: 'missing', metric: 'min' },
+        count: { field: 'missing', metric: 'count' },
+      },
+    })
+    expect(out[0]?.['sum']).toBe(0)
+    expect(out[0]?.['min']).toBeUndefined()
+    expect(out[0]?.['count']).toBe(1)
+  })
+})
+
+describe('timeBucket', () => {
+  const noon = date(2026, 3, 12, 14, 37) // Thursday afternoon
+
+  it('floors to the hour and the day', () => {
+    expect(timeBucket(noon, 'hour')).toEqual(date(2026, 3, 12, 14))
+    expect(timeBucket(noon, 'day')).toEqual(date(2026, 3, 12))
+  })
+
+  it('starts weeks on Monday', () => {
+    expect(timeBucket(noon, 'week')).toEqual(date(2026, 3, 9))
+    expect(timeBucket(date(2026, 3, 8, 3), 'week')).toEqual(date(2026, 3, 2))
+  })
+
+  it('starts months, quarters and years on their first day', () => {
+    expect(timeBucket(noon, 'month')).toEqual(date(2026, 3, 1))
+    expect(timeBucket(noon, 'quarter')).toEqual(date(2026, 1, 1))
+    expect(timeBucket(date(2026, 5, 20), 'quarter')).toEqual(date(2026, 4, 1))
+    expect(timeBucket(noon, 'year')).toEqual(date(2026, 1, 1))
+  })
+
+  it('accepts ISO strings and epochs, and stays Invalid when unparseable', () => {
+    const ms = date(2026, 3, 12, 14).getTime()
+    expect(timeBucket(new Date(ms).toISOString(), 'day')).toEqual(
+      date(2026, 3, 12),
+    )
+    expect(timeBucket(ms, 'day')).toEqual(date(2026, 3, 12))
+    expect(Number.isNaN(timeBucket('not a date', 'day').getTime())).toBe(true)
+  })
+})
+
+describe('rollup', () => {
+  const rows = [
+    { createdAt: date(2026, 1, 3, 9), amount: 5 },
+    { createdAt: date(2026, 1, 3, 17), amount: 10 },
+    { createdAt: date(2026, 1, 4, 9), amount: 7 },
+    { createdAt: 'yesterday', amount: 1 },
+  ]
+
+  it('counts rows per bucket, ascending, skipping unparseable dates', () => {
+    const series = rollup(rows, { field: 'createdAt', unit: 'day' })
+    expect(series).toEqual([
+      { bucket: date(2026, 1, 3), value: 2 },
+      { bucket: date(2026, 1, 4), value: 1 },
+    ])
+  })
+
+  it('applies a value metric per bucket', () => {
+    const series = rollup(rows, {
+      field: 'createdAt',
+      unit: 'day',
+      metric: { field: 'amount', metric: 'sum' },
+    })
+    expect(series.map((point) => point.value)).toEqual([15, 7])
+  })
+
+  it('buckets by month with a bare metric name', () => {
+    const series = rollup(rows, {
+      field: 'createdAt',
+      unit: 'month',
+      metric: 'count',
+    })
+    expect(series).toEqual([{ bucket: date(2026, 1, 1), value: 3 }])
+  })
+
+  it('keeps input order for first and last within a bucket', () => {
+    const series = rollup(
+      [
+        { createdAt: date(2026, 1, 3, 17), label: 'late' },
+        { createdAt: date(2026, 1, 3, 9), label: 'early' },
+      ],
+      {
+        field: 'createdAt',
+        unit: 'day',
+        metric: { field: 'label', metric: 'first' },
+      },
+    )
+    expect(series[0]?.value).toBe('late')
+  })
+})

package/src/aggregate.ts

@@ -0,0 +1,320 @@
+/**
+ * Pure transforms over rows the app already fetched.
+ *
+ * `aggregate` groups rows with the same eleven metric names table aggregation
+ * uses, so one vocabulary spans tables and transforms; `timeBucket` and
+ * `rollup` produce chart series from a time column. These are plain functions
+ * — no signals, no query, no components — and past their size the answer is
+ * server-side, per server mode's own doctrine.
+ */
+
+/** The kit's eleven aggregation metrics, shared with table aggregation. */
+export type Metric =
+  | 'sum'
+  | 'count'
+  | 'min'
+  | 'max'
+  | 'extent'
+  | 'mean'
+  | 'median'
+  | 'unique'
+  | 'uniqueCount'
+  | 'first'
+  | 'last'
+
+/** The bucket units {@link timeBucket} and {@link rollup} understand. */
+export type TimeUnit = 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year'
+
+/** A metric applied to one field. `count` ignores the field's values. */
+export interface MetricSpec {
+  field: string
+  metric: Metric
+}
+
+/** A row is whatever the app's query returned. */
+export type AggregateRow = Record<string, unknown>
+
+/** Options for {@link aggregate}. */
+export interface AggregateOptions {
+  /** Group-by columns. The output row carries each group's value. */
+  by: string | readonly string[]
+  /** Output column → the field to aggregate and the metric to apply. */
+  metrics: Record<string, MetricSpec>
+}
+
+/** A bucketed series point, shaped for a chart's `{ x, y }` series. */
+export interface RollupPoint {
+  /** The start of the bucket, in local time. */
+  bucket: Date
+  value: unknown
+}
+
+/** Options for {@link rollup}. */
+export interface RollupOptions {
+  /** The row's date column. */
+  field: string
+  unit: TimeUnit
+  /**
+   * What to compute per bucket. A bare metric name counts rows (its field is
+   * the date column, which is what a count of a time series means); pass a
+   * {@link MetricSpec} to aggregate a value column per bucket.
+   */
+  metric?: Metric | MetricSpec
+}
+
+/** A date, an ISO string, or an epoch — anything a chart hands back. */
+export type DateLike = Date | string | number
+
+function toDate(value: unknown): Date {
+  if (value instanceof Date) return value
+  if (typeof value === 'number') return new Date(value)
+  if (typeof value === 'string') return new Date(value)
+  return new Date(Number.NaN)
+}
+
+/** Coerce to a number for the numeric metrics; `undefined` when not numeric. */
+function numeric(value: unknown): number | undefined {
+  if (typeof value === 'number') {
+    return Number.isFinite(value) ? value : undefined
+  }
+  if (value instanceof Date) return value.getTime()
+  if (typeof value === 'string' && value.trim() !== '') {
+    const parsed = Number(value)
+    return Number.isFinite(parsed) ? parsed : undefined
+  }
+  return undefined
+}
+
+/** Identity for `unique`/`uniqueCount`: JSON so objects compare by shape. */
+function identity(value: unknown): string {
+  if (value === undefined) return 'undefined'
+  try {
+    return JSON.stringify(value) ?? String(value)
+  } catch {
+    return String(value)
+  }
+}
+
+function uniqueValues(values: readonly unknown[]): unknown[] {
+  const seen = new Set<string>()
+  const out: unknown[] = []
+  for (const value of values) {
+    const key = identity(value)
+    if (!seen.has(key)) {
+      seen.add(key)
+      out.push(value)
+    }
+  }
+  return out
+}
+
+function median(values: readonly number[]): number | undefined {
+  if (values.length === 0) return undefined
+  const sorted = [...values].sort((a, b) => a - b)
+  const middle = Math.floor(sorted.length / 2)
+  if (sorted.length % 2 === 1) return sorted[middle]
+  const low = sorted[middle - 1]
+  const high = sorted[middle]
+  if (low === undefined || high === undefined) return undefined
+  return (low + high) / 2
+}
+
+/** Apply one metric to a group's rows. Internal to the transform module. */
+function applyMetric(rows: readonly AggregateRow[], spec: MetricSpec): unknown {
+  const values = rows.map((row) => row[spec.field])
+  switch (spec.metric) {
+    case 'count':
+      return rows.length
+    case 'sum': {
+      let total = 0
+      for (const value of values) total += numeric(value) ?? 0
+      return total
+    }
+    case 'min': {
+      let best: number | undefined
+      for (const value of values) {
+        const n = numeric(value)
+        if (n !== undefined && (best === undefined || n < best)) best = n
+      }
+      return best
+    }
+    case 'max': {
+      let best: number | undefined
+      for (const value of values) {
+        const n = numeric(value)
+        if (n !== undefined && (best === undefined || n > best)) best = n
+      }
+      return best
+    }
+    case 'extent': {
+      let low: number | undefined
+      let high: number | undefined
+      for (const value of values) {
+        const n = numeric(value)
+        if (n === undefined) continue
+        if (low === undefined || n < low) low = n
+        if (high === undefined || n > high) high = n
+      }
+      if (low === undefined || high === undefined) return undefined
+      return [low, high]
+    }
+    case 'mean': {
+      const numbers = values
+        .map(numeric)
+        .filter((n): n is number => n !== undefined)
+      if (numbers.length === 0) return undefined
+      return (
+        numbers.reduce((total, n) => total + n, 0) / (numbers.length as number)
+      )
+    }
+    case 'median': {
+      const numbers = values
+        .map(numeric)
+        .filter((n): n is number => n !== undefined)
+      return median(numbers)
+    }
+    case 'unique':
+      return uniqueValues(values)
+    case 'uniqueCount':
+      return uniqueValues(values).length
+    case 'first':
+      return values[0]
+    case 'last':
+      return values[values.length - 1]
+  }
+}
+
+/** A stable group key for a row's `by` values; dates group by their instant. */
+function groupKey(row: AggregateRow, by: readonly string[]): string {
+  return JSON.stringify(
+    by.map((key) => {
+      const value = row[key]
+      return value instanceof Date ? value.toISOString() : value
+    }),
+  )
+}
+
+/**
+ * Group rows by `by` and compute each named metric per group.
+ *
+ * The output rows carry the group columns plus one column per metrics key,
+ * in the order groups first appear. `min`/`max`/`extent`/`mean`/`median`
+ * ignore non-numeric values (and empty groups yield `undefined`); `count`
+ * counts rows, and `first`/`last` keep the row's own value.
+ */
+export function aggregate(
+  rows: readonly AggregateRow[],
+  options: AggregateOptions,
+): AggregateRow[] {
+  const by = typeof options.by === 'string' ? [options.by] : options.by
+  const groups = new Map<string, AggregateRow[]>()
+  const groupValues = new Map<string, unknown[]>()
+  for (const row of rows) {
+    const key = groupKey(row, by)
+    const bucket = groups.get(key)
+    if (bucket) {
+      bucket.push(row)
+    } else {
+      groups.set(key, [row])
+      groupValues.set(
+        key,
+        by.map((column) => {
+          const value = row[column]
+          return value instanceof Date ? new Date(value.getTime()) : value
+        }),
+      )
+    }
+  }
+
+  const out: AggregateRow[] = []
+  for (const [key, bucket] of groups) {
+    const result: AggregateRow = {}
+    const values = groupValues.get(key) ?? []
+    by.forEach((column, index) => {
+      result[column] = values[index]
+    })
+    for (const [id, spec] of Object.entries(options.metrics)) {
+      result[id] = applyMetric(bucket, spec)
+    }
+    out.push(result)
+  }
+  return out
+}
+
+/**
+ * Floor a date to the start of its bucket, in local time.
+ *
+ * Weeks start on Monday; months, quarters and years start on their first day
+ * at midnight. An unparseable date yields an Invalid Date, so a chart can
+ * still map it — `rollup` skips those rows instead.
+ */
+export function timeBucket(date: DateLike, unit: TimeUnit): Date {
+  const value = toDate(date)
+  if (Number.isNaN(value.getTime())) return new Date(Number.NaN)
+  const bucket = new Date(value.getTime())
+  switch (unit) {
+    case 'hour':
+      bucket.setMinutes(0, 0, 0)
+      return bucket
+    case 'day':
+      bucket.setHours(0, 0, 0, 0)
+      return bucket
+    case 'week': {
+      bucket.setHours(0, 0, 0, 0)
+      // getDay(): Sunday is 0; shift back to the Monday that starts the week.
+      const shift = (bucket.getDay() + 6) % 7
+      bucket.setDate(bucket.getDate() - shift)
+      return bucket
+    }
+    case 'month':
+      bucket.setHours(0, 0, 0, 0)
+      bucket.setDate(1)
+      return bucket
+    case 'quarter':
+      bucket.setHours(0, 0, 0, 0)
+      bucket.setDate(1)
+      bucket.setMonth(Math.floor(bucket.getMonth() / 3) * 3)
+      return bucket
+    case 'year':
+      bucket.setHours(0, 0, 0, 0)
+      bucket.setDate(1)
+      bucket.setMonth(0)
+      return bucket
+  }
+}
+
+/**
+ * Turn a time column into a chart series: bucket the rows by `unit`, apply
+ * `metric` per bucket, and return the buckets in ascending order.
+ *
+ * Rows whose date column does not parse are skipped — a chart wants the
+ * series it can place, not a hole. Rows are read in the given order, so
+ * `first`/`last` follow the input's order, not the timeline's.
+ */
+export function rollup(
+  rows: readonly AggregateRow[],
+  options: RollupOptions,
+): RollupPoint[] {
+  const spec: MetricSpec =
+    typeof options.metric === 'string'
+      ? { field: options.field, metric: options.metric }
+      : (options.metric ?? { field: options.field, metric: 'count' })
+  const groups = new Map<number, AggregateRow[]>()
+  for (const row of rows) {
+    const bucket = timeBucket(row[options.field] as DateLike, options.unit)
+    if (Number.isNaN(bucket.getTime())) continue
+    const key = bucket.getTime()
+    const bucketRows = groups.get(key)
+    if (bucketRows) {
+      bucketRows.push(row)
+    } else {
+      groups.set(key, [row])
+    }
+  }
+  return [...groups.entries()]
+    .sort((a, b) => a[0] - b[0])
+    .map(([time, bucketRows]) => ({
+      bucket: new Date(time),
+      value: applyMetric(bucketRows, spec),
+    }))
+}