@volter/world-core 2.0.37 → 3.0.0

package/src/clickhouse/values.ts

@@ -0,0 +1,697 @@
+// ClickHouse® VALUES as the kernel's ClickHouse engine carries them (Tinybird's pipes, PostHog's HogQL): the runtime value classes
+// (DateTime, Date, Interval, aggregate states), time-zone arithmetic, the type grammar's defaults,
+// ordering/equality, and the conversions between an engine value, its stored JSON form and its
+// JSON-output form.
+//
+// Semantics transcribed from ClickHouse's documentation:
+//   • DateTime / DateTime64 are an instant plus a display time zone; the server zone (Tinybird's)
+//     is UTC. Calendar functions (`toStartOfDay(t, tz)`, `+ INTERVAL n DAY`, `dateDiff('month', …)`)
+//     work on the wall clock of the zone; `toStartOfMonth` returns a `Date`.
+//   • JSON output renders DateTime as `YYYY-MM-DD hh:mm:ss` and DateTime64(n) with n fractional
+//     digits, in the value's zone; Date as `YYYY-MM-DD`.
+//   • A non-Nullable column's default (what `join_use_nulls = 0` fills for a non-matching side and
+//     what an un-grouped aggregate over no rows answers) is 0 / '' / epoch / [].
+
+/**
+ * A DateTime or DateTime64 value: an instant held as exact integer ticks — whole milliseconds since
+ * the epoch (`ms`) plus the nanoseconds inside that millisecond (`ns`, 0…999 999) — its scale (the
+ * `p` of `DateTime64(p)`, 0 for DateTime) and a display zone. ClickHouse stores DateTime64(p) as an
+ * Int64 count of 10^-p second ticks, so every value is truncated to its scale on construction
+ * (`makeDT`) and compares exactly.
+ */
+export class DT {
+  constructor(readonly ms: number, readonly scale: number, readonly tz?: string, readonly ns = 0) {}
+  withTz(tz: string | undefined, scale = this.scale): DT {
+    return makeDT(this.ms, this.ns, scale, tz);
+  }
+}
+
+/** A DT truncated to `scale` digits (ClickHouse's DateTime64 tick), with `ns` normalised into `ms`. */
+export function makeDT(ms: number, ns: number, scale: number, tz?: string): DT {
+  let m = Math.floor(ms);
+  let n = Math.round(ns + (ms - m) * 1e6);
+  if (n >= 1e6 || n < 0) { const carry = Math.floor(n / 1e6); m += carry; n -= carry * 1e6; }
+  if (scale >= 9) return new DT(m, scale, tz, n);
+  if (scale >= 3) { const step = 10 ** (9 - scale); return new DT(m, scale, tz, n - (n % step)); }
+  const step = 10 ** (3 - scale);
+  return new DT(Math.floor(m / step) * step, scale, tz, 0);
+}
+
+/** Two instants' order, exactly (ms first, then the sub-millisecond nanoseconds). */
+export function compareDT(a: DT, b: DT): number {
+  if (a.ms !== b.ms) return a.ms < b.ms ? -1 : 1;
+  return a.ns === b.ns ? 0 : a.ns < b.ns ? -1 : 1;
+}
+
+// ── exact integers ──────────────────────────────────────────────────────────────────────────
+// ClickHouse's Int64/UInt64/Int128/… are exact. An engine integer is a JS number while it is a
+// safe integer and a bigint beyond ±2^53 (one representation per value, so keys and equality
+// agree); stored JSON carries such a value as its decimal string.
+
+/** A bigint as the engine carries it: a number when safe, else the bigint. */
+export function normInt(v: bigint): number | bigint {
+  return v >= -9007199254740991n && v <= 9007199254740991n ? Number(v) : v;
+}
+const INT_WIDTH: Record<string, { bits: bigint; signed: boolean }> = {
+  Int8: { bits: 8n, signed: true }, Int16: { bits: 16n, signed: true }, Int32: { bits: 32n, signed: true },
+  Int64: { bits: 64n, signed: true }, Int128: { bits: 128n, signed: true }, Int256: { bits: 256n, signed: true },
+  UInt8: { bits: 8n, signed: false }, UInt16: { bits: 16n, signed: false }, UInt32: { bits: 32n, signed: false },
+  UInt64: { bits: 64n, signed: false }, UInt128: { bits: 128n, signed: false }, UInt256: { bits: 256n, signed: false },
+};
+/** Wrap an integer into a sized ClickHouse integer type (two's complement, as CAST/toIntN do). */
+export function wrapInt(v: bigint, type: string): number | bigint {
+  const w = INT_WIDTH[typeHead(type)];
+  if (!w) return normInt(v);
+  const mod = 1n << w.bits;
+  let r = ((v % mod) + mod) % mod;
+  if (w.signed && r >= mod >> 1n) r -= mod;
+  return normInt(r);
+}
+/** An engine value as an exact integer (a float truncates toward zero, as ClickHouse's cast does). */
+export function toBigInt(v: unknown, type = 'Int64'): bigint {
+  if (typeof v === 'bigint') return v;
+  if (typeof v === 'boolean') return v ? 1n : 0n;
+  if (typeof v === 'number') return Number.isFinite(v) ? BigInt(Math.trunc(v)) : 0n;
+  if (v instanceof DT) return BigInt(Math.floor(v.ms / 1000));
+  if (v instanceof DDate) return BigInt(v.days);
+  if (typeof v === 'string') {
+    if (/^[+-]?\d+$/.test(v)) return BigInt(v);
+    throw new ClickHouseError(`Cannot parse string '${v}' as ${baseType(type).base}: syntax error`);
+  }
+  throw new ClickHouseError(`Cannot convert ${valueToText(v)} to ${baseType(type).base}`);
+}
+/** An integer as stored JSON: a number when safe, its decimal string otherwise. */
+function storedInt(v: number | bigint): number | string {
+  return typeof v === 'bigint' ? v.toString() : v;
+}
+
+/** Is this an integer type of 64 bits or more (the ones `output_format_json_quote_64bit_integers` quotes)? */
+export function isWideIntegerType(type: string): boolean {
+  return /^U?Int(64|128|256)$/.test(typeHead(type));
+}
+
+const BIG_MARK = '\u0000tb-bigint:';
+/** JSON.stringify that writes a bigint as a bare JSON number (ClickHouse's unquoted 64-bit output). */
+export function stringifyJson(value: unknown): string {
+  const text = JSON.stringify(value, (_k, v) => (typeof v === 'bigint' ? `${BIG_MARK}${v.toString()}` : v));
+  return text === undefined ? text as unknown as string : text.replace(/"\\u0000tb-bigint:(-?\d+)"/g, '$1');
+}
+
+/** A Date value: a calendar day (days since 1970-01-01), no zone. */
+export class DDate {
+  constructor(readonly days: number) {}
+}
+
+export type IntervalUnit = 'millisecond' | 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year';
+export class Interval {
+  constructor(readonly unit: IntervalUnit, readonly n: number) {}
+}
+
+/** A `Map(K, V)` value: its entries in the order they were written (a ClickHouse Map keeps its entries as an array of
+ *  pairs, https://clickhouse.com/docs/sql-reference/data-types/map). Stored as a JSON object of the entries. */
+export class ChMap {
+  constructor(readonly entries: Array<[unknown, unknown]>) {}
+}
+
+/** An aggregate function's intermediate state (`minState(x)` → `AggregateFunction(min, T)`). */
+export class AggState {
+  constructor(readonly fn: string, readonly value: unknown) {}
+}
+
+export const SERVER_TZ = 'UTC';
+
+// ── time zones ──────────────────────────────────────────────────────────────────────────────
+
+const FORMATTERS = new Map<string, Intl.DateTimeFormat>();
+function formatter(tz: string): Intl.DateTimeFormat {
+  let f = FORMATTERS.get(tz);
+  if (!f) {
+    try {
+      f = new Intl.DateTimeFormat('en-US', {
+        timeZone: tz, hourCycle: 'h23', year: 'numeric', month: '2-digit', day: '2-digit',
+        hour: '2-digit', minute: '2-digit', second: '2-digit',
+      });
+    } catch {
+      throw new ClickHouseError(`Invalid time zone: ${tz}`);
+    }
+    FORMATTERS.set(tz, f);
+  }
+  return f;
+}
+
+/** A query the twin refuses, carrying the status + message the vendor answers with. */
+export class ClickHouseError extends Error {
+  readonly status: number;
+  constructor(message: string, status = 400) {
+    super(message);
+    this.name = 'ClickHouseError';
+    this.status = status;
+  }
+}
+export { ClickHouseError as EngineError };
+
+export type Wall = { y: number; mo: number; d: number; h: number; mi: number; s: number; ms: number };
+
+/** The wall-clock fields of an instant in a zone. */
+export function wallOf(ms: number, tz = SERVER_TZ): Wall {
+  const floorSec = Math.floor(ms / 1000);
+  const frac = ms - floorSec * 1000;
+  if (tz === 'UTC' || tz === 'Etc/UTC' || tz === 'GMT') {
+    const d = new Date(floorSec * 1000);
+    return { y: d.getUTCFullYear(), mo: d.getUTCMonth() + 1, d: d.getUTCDate(), h: d.getUTCHours(), mi: d.getUTCMinutes(), s: d.getUTCSeconds(), ms: frac };
+  }
+  const parts = formatter(tz).formatToParts(new Date(floorSec * 1000));
+  const get = (t: string) => Number(parts.find((p) => p.type === t)?.value ?? '0');
+  return { y: get('year'), mo: get('month'), d: get('day'), h: get('hour'), mi: get('minute'), s: get('second'), ms: frac };
+}
+
+/** The zone's UTC offset at an instant, in ms (local − UTC). */
+export function offsetMs(ms: number, tz = SERVER_TZ): number {
+  const w = wallOf(ms, tz);
+  const asUtc = Date.UTC(w.y, w.mo - 1, w.d, w.h, w.mi, w.s, w.ms);
+  return asUtc - ms;
+}
+
+/**
+ * The instant a wall-clock time names in a zone. ClickHouse's DateLUT (`makeDateTime`) adds the
+ * time of day to the day's start and, from the offset-change moment on, subtracts the change: a
+ * wall time inside a spring-forward gap therefore lands BEFORE the gap (`2024-03-10 02:30:00` in
+ * America/New_York reads back as `01:30:00` — clickhouse-local 26.10 answers the same), and a
+ * repeated fall-back hour resolves to its first occurrence. Two offset probes reproduce both.
+ */
+export function instantOf(w: Wall, tz = SERVER_TZ): number {
+  const local = Date.UTC(w.y, w.mo - 1, w.d, w.h, w.mi, w.s, w.ms);
+  let guess = local - offsetMs(local, tz);
+  guess = local - offsetMs(guess, tz);
+  return guess;
+}
+
+export function daysFromCivil(y: number, mo: number, d: number): number {
+  return Math.floor(Date.UTC(y, mo - 1, d) / 86_400_000);
+}
+export function civilFromDays(days: number): { y: number; mo: number; d: number } {
+  const dt = new Date(days * 86_400_000);
+  return { y: dt.getUTCFullYear(), mo: dt.getUTCMonth() + 1, d: dt.getUTCDate() };
+}
+
+// ── parsing ─────────────────────────────────────────────────────────────────────────────────
+
+const DT_TEXT = /^\s*(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?)?\s*(Z|[+-]\d{2}(?::?\d{2})?)?\s*$/i;
+
+/** Parse ClickHouse datetime text (and the ISO forms Tinybird's ingest accepts) in a zone, exactly:
+ *  whole milliseconds plus the nanoseconds a DateTime64(4…9) fraction carries. */
+export function parseDateTimeInstant(text: string, tz = SERVER_TZ): { ms: number; ns: number } | undefined {
+  const m = DT_TEXT.exec(text);
+  if (!m) {
+    if (/^\s*\d{9,11}(\.\d+)?\s*$/.test(text)) {
+      const [whole = '0', frac = ''] = text.trim().split('.');
+      const f9 = frac.padEnd(9, '0').slice(0, 9);
+      return { ms: Number(whole) * 1000 + Number(f9.slice(0, 3)), ns: Number(f9.slice(3)) };
+    }
+    return undefined;
+  }
+  const f9 = (m[7] ?? '').padEnd(9, '0');
+  const ns = Number(f9.slice(3));
+  const w: Wall = { y: Number(m[1]), mo: Number(m[2]), d: Number(m[3]), h: Number(m[4] ?? 0), mi: Number(m[5] ?? 0), s: Number(m[6] ?? 0), ms: Number(f9.slice(0, 3)) };
+  if (m[8]) {
+    const zone = m[8].toUpperCase();
+    const base = Date.UTC(w.y, w.mo - 1, w.d, w.h, w.mi, w.s, w.ms);
+    if (zone === 'Z') return { ms: base, ns };
+    const zm = /^([+-])(\d{2}):?(\d{2})?$/.exec(zone)!;
+    const off = (Number(zm[2]) * 60 + Number(zm[3] ?? 0)) * 60_000 * (zm[1] === '-' ? -1 : 1);
+    return { ms: base - off, ns };
+  }
+  return { ms: instantOf(w, tz), ns };
+}
+
+export function parseDateText(text: string): number | undefined {
+  const m = /^\s*(\d{4})-(\d{1,2})-(\d{1,2})/.exec(text);
+  if (!m) return undefined;
+  return daysFromCivil(Number(m[1]), Number(m[2]), Number(m[3]));
+}
+
+// ── formatting ──────────────────────────────────────────────────────────────────────────────
+
+const pad = (n: number, w = 2) => String(Math.trunc(Math.abs(n))).padStart(w, '0');
+
+export function formatDate(d: DDate): string {
+  const c = civilFromDays(d.days);
+  return `${pad(c.y, 4)}-${pad(c.mo)}-${pad(c.d)}`;
+}
+
+export function formatDT(v: DT): string {
+  const w = wallOf(v.ms, v.tz);
+  const base = `${pad(w.y, 4)}-${pad(w.mo)}-${pad(w.d)} ${pad(w.h)}:${pad(w.mi)}:${pad(w.s)}`;
+  if (v.scale <= 0) return base;
+  const frac = `${String(Math.floor(w.ms)).padStart(3, '0')}${String(v.ns).padStart(6, '0')}`.padEnd(v.scale, '0').slice(0, v.scale);
+  return `${base}.${frac}`;
+}
+
+/** A DT as a Data Source stores it: wall-clock text in the column's zone, with the zone's offset
+ *  when that zone is not UTC (so an instant inside a repeated DST hour survives a round trip). */
+export function formatStoredDT(v: DT): string {
+  const text = formatDT(v);
+  const zone = v.tz ?? SERVER_TZ;
+  if (zone === 'UTC' || zone === 'Etc/UTC' || zone === 'GMT') return text;
+  const off = offsetMs(v.ms, zone) / 60_000;
+  return `${text}${off < 0 ? '-' : '+'}${pad(Math.floor(Math.abs(off) / 60))}:${pad(Math.abs(off) % 60)}`;
+}
+
+/** `formatDateTime(t, fmt)` — ClickHouse's MySQL-style specifiers. */
+export function formatDateTimePattern(v: DT | DDate, fmt: string, tz?: string): string {
+  const dt = v instanceof DDate ? new DT(instantOf({ ...civilFromDays(v.days), h: 0, mi: 0, s: 0, ms: 0 }, tz ?? SERVER_TZ), 0, tz ?? SERVER_TZ) : v;
+  const zone = tz ?? dt.tz ?? SERVER_TZ;
+  const w = wallOf(dt.ms, zone);
+  const off = offsetMs(dt.ms, zone) / 60_000;
+  const days = daysFromCivil(w.y, w.mo, w.d);
+  const dow = ((days % 7) + 7 + 4) % 7; // 1970-01-01 was a Thursday
+  const doy = days - daysFromCivil(w.y, 1, 1) + 1;
+  const MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
+  const DAYS = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];
+  let out = '';
+  for (let i = 0; i < fmt.length; i++) {
+    const ch = fmt[i]!;
+    if (ch !== '%' || i + 1 >= fmt.length) { out += ch; continue; }
+    const f = fmt[++i]!;
+    switch (f) {
+      case 'Y': out += pad(w.y, 4); break;
+      case 'y': out += pad(w.y % 100); break;
+      case 'C': out += pad(Math.floor(w.y / 100)); break;
+      case 'm': out += pad(w.mo); break;
+      case 'd': out += pad(w.d); break;
+      case 'e': out += String(w.d).padStart(2, ' '); break;
+      case 'F': out += `${pad(w.y, 4)}-${pad(w.mo)}-${pad(w.d)}`; break;
+      case 'D': out += `${pad(w.mo)}/${pad(w.d)}/${pad(w.y % 100)}`; break;
+      case 'H': out += pad(w.h); break;
+      case 'I': case 'h': out += pad(((w.h + 11) % 12) + 1); break;
+      case 'i': out += pad(w.mi); break;
+      case 'M': out += MONTHS[w.mo - 1]; break;
+      case 'S': case 's': out += pad(w.s); break;
+      case 'T': out += `${pad(w.h)}:${pad(w.mi)}:${pad(w.s)}`; break;
+      case 'R': out += `${pad(w.h)}:${pad(w.mi)}`; break;
+      case 'p': out += w.h < 12 ? 'AM' : 'PM'; break;
+      case 'j': out += pad(doy, 3); break;
+      case 'u': out += String(dow === 0 ? 7 : dow); break;
+      case 'w': out += String(dow); break;
+      case 'a': out += DAYS[dow]!.slice(0, 3); break;
+      case 'b': out += MONTHS[w.mo - 1]!.slice(0, 3); break;
+      case 'f': out += String(Math.floor(w.ms)).padStart(3, '0').padEnd(6, '0'); break;
+      case 'z': out += `${off < 0 ? '-' : '+'}${pad(Math.floor(Math.abs(off) / 60))}${pad(Math.abs(off) % 60)}`; break;
+      case 'n': out += '\n'; break;
+      case 't': out += '\t'; break;
+      case '%': out += '%'; break;
+      default: throw new ClickHouseError(`formatDateTime: unsupported format specifier %${f}`);
+    }
+  }
+  return out;
+}
+
+// ── types ───────────────────────────────────────────────────────────────────────────────────
+
+/** Peel `LowCardinality(…)` and `Nullable(…)`. */
+export function baseType(type: string): { base: string; nullable: boolean } {
+  let t = type.trim();
+  let nullable = false;
+  for (let guard = 0; guard < 4; guard++) {
+    const low = /^LowCardinality\((.*)\)$/s.exec(t);
+    if (low) { t = low[1]!.trim(); continue; }
+    const nul = /^Nullable\((.*)\)$/s.exec(t);
+    if (nul) { nullable = true; t = nul[1]!.trim(); continue; }
+    break;
+  }
+  return { base: t, nullable };
+}
+
+export function typeHead(type: string): string {
+  return baseType(type).base.replace(/\(.*$/s, '').trim();
+}
+
+/** Split a parameterised type's top-level arguments: `Array(Nullable(String))` → [`Nullable(String)`]. */
+export function typeArgs(type: string): string[] {
+  const { base } = baseType(type);
+  const open = base.indexOf('(');
+  if (open < 0 || !base.endsWith(')')) return [];
+  const inner = base.slice(open + 1, -1);
+  const out: string[] = [];
+  let depth = 0;
+  let cur = '';
+  let quote = false;
+  for (const ch of inner) {
+    if (ch === "'") quote = !quote;
+    if (!quote && ch === '(') depth++;
+    if (!quote && ch === ')') depth--;
+    if (!quote && ch === ',' && depth === 0) { out.push(cur.trim()); cur = ''; continue; }
+    cur += ch;
+  }
+  if (cur.trim()) out.push(cur.trim());
+  return out;
+}
+
+export function isNullableType(type: string): boolean {
+  return baseType(type).nullable;
+}
+
+const INTS = new Set(['Int8', 'Int16', 'Int32', 'Int64', 'Int128', 'Int256']);
+const UINTS = new Set(['UInt8', 'UInt16', 'UInt32', 'UInt64', 'UInt128', 'UInt256', 'Bool', 'Boolean']);
+const FLOATS = new Set(['Float32', 'Float64']);
+
+export function isIntegerType(type: string): boolean {
+  const h = typeHead(type);
+  return INTS.has(h) || UINTS.has(h);
+}
+export function isUnsignedType(type: string): boolean {
+  return UINTS.has(typeHead(type));
+}
+export function isFloatType(type: string): boolean {
+  const h = typeHead(type);
+  return FLOATS.has(h) || h.startsWith('Decimal');
+}
+export function isNumberType(type: string): boolean {
+  return isIntegerType(type) || isFloatType(type);
+}
+
+/** The zone a `DateTime('tz')` / `DateTime64(n, 'tz')` type names, if any. */
+export function typeZone(type: string): string | undefined {
+  const args = typeArgs(type);
+  const zone = args.find((a) => /^'.*'$/.test(a));
+  return zone ? zone.slice(1, -1) : undefined;
+}
+export function typeScale(type: string): number {
+  const h = typeHead(type);
+  if (h === 'DateTime64') {
+    const first = typeArgs(type)[0];
+    return first !== undefined && /^\d+$/.test(first) ? Number(first) : 3;
+  }
+  return 0;
+}
+
+/** A non-Nullable column's default value. */
+export function defaultOf(type: string): unknown {
+  const { base, nullable } = baseType(type);
+  if (nullable) return null;
+  const h = base.replace(/\(.*$/s, '').trim();
+  if (INTS.has(h) || UINTS.has(h) || FLOATS.has(h) || h.startsWith('Decimal')) return h === 'Bool' || h === 'Boolean' ? false : 0;
+  if (h === 'String' || h === 'FixedString' || h === 'UUID' || h === 'IPv4' || h === 'IPv6' || h.startsWith('Enum')) {
+    if (h === 'UUID') return '00000000-0000-0000-0000-000000000000';
+    return '';
+  }
+  if (h === 'DateTime' || h === 'DateTime64') return new DT(0, typeScale(base), typeZone(base));
+  if (h === 'Date' || h === 'Date32') return new DDate(0);
+  if (h === 'Array') return [];
+  if (h === 'Map') return new ChMap([]);
+  if (h === 'AggregateFunction' || h === 'SimpleAggregateFunction') return null;
+  if (h === 'Nothing') return null;
+  return '';
+}
+
+// ── aggregate states ────────────────────────────────────────────────────────────────────────
+// An `AggregateFunction(fn, T…)` cell holds what `fnMerge` needs to finish the aggregate over any
+// union of states: the running value for min/max/any/anyLast/sum/count, [sum, count] for avg, the
+// distinct values for uniq*/groupUniqArray, every value for groupArray/quantile*/median*/topK and
+// [arg, key] for argMin/argMax. Stored JSON is `{"$agg": fn, "value": …}` with T's stored form.
+
+export type AggKind = 'scalar' | 'sum' | 'avg' | 'set' | 'list' | 'arg';
+export function aggKind(fn: string): AggKind | undefined {
+  const base = fn.replace(/\(.*$/s, '').toLowerCase();
+  if (['min', 'max', 'any', 'anylast'].includes(base)) return 'scalar';
+  if (base === 'sum' || base === 'count') return 'sum';
+  if (base === 'avg') return 'avg';
+  if (['uniq', 'uniqexact', 'uniqcombined', 'uniqcombined64', 'uniqhll12', 'groupuniqarray', 'countdistinct'].includes(base)) return 'set';
+  if (base === 'grouparray' || base === 'grouparraysample' || base === 'topk' || base.startsWith('quantile') || base.startsWith('median')) return 'list';
+  if (base === 'argmin' || base === 'argmax') return 'arg';
+  return undefined;
+}
+
+function stateFromStored(v: { $agg: string; value: unknown }, base: string): AggState {
+  const args = typeArgs(base);
+  const t1 = args[1] ?? 'String';
+  const t2 = args[2] ?? 'String';
+  const kind = aggKind(v.$agg);
+  const raw = v.value;
+  if (raw === null || raw === undefined) return new AggState(v.$agg, null);
+  switch (kind) {
+    case 'sum': return new AggState(v.$agg, typeof raw === 'string' ? normInt(BigInt(raw)) : raw);
+    case 'avg': return new AggState(v.$agg, Array.isArray(raw) ? [Number(raw[0]), Number(raw[1])] : null);
+    case 'set': case 'list': return new AggState(v.$agg, Array.isArray(raw) ? raw.map((x) => fromStored(x, t1)) : []);
+    case 'arg': return new AggState(v.$agg, Array.isArray(raw) ? [fromStored(raw[0], t1), fromStored(raw[1], t2)] : null);
+    default: return new AggState(v.$agg, fromStored(raw, t1));
+  }
+}
+function stateToStored(s: AggState, base: string): unknown {
+  const args = typeArgs(base);
+  const t1 = args[1] ?? 'String';
+  const t2 = args[2] ?? 'String';
+  const v = s.value;
+  if (v === null || v === undefined) return { $agg: s.fn, value: null };
+  switch (aggKind(s.fn)) {
+    case 'sum': return { $agg: s.fn, value: typeof v === 'bigint' ? v.toString() : v };
+    case 'avg': return { $agg: s.fn, value: v };
+    case 'set': case 'list': return { $agg: s.fn, value: (v as unknown[]).map((x) => toStored(x, t1)) };
+    case 'arg': { const [a, k] = v as [unknown, unknown]; return { $agg: s.fn, value: [toStored(a, t1), toStored(k, t2)] }; }
+    default: return { $agg: s.fn, value: toStored(v, t1) };
+  }
+}
+
+/** A stored (JSON) cell → the engine value for its column type. */
+export function fromStored(value: unknown, type: string): unknown {
+  if (value === undefined) return defaultOf(type);
+  if (value === null) return baseType(type).nullable ? null : defaultOf(type);
+  const { base } = baseType(type);
+  const h = base.replace(/\(.*$/s, '').trim();
+  if (h === 'DateTime' || h === 'DateTime64') {
+    if (value instanceof DT) return value;
+    const scale = typeScale(base);
+    const zone = typeZone(base);
+    // Text without an explicit offset is wall-clock time in the COLUMN's zone (ClickHouse parses
+    // a `DateTime('America/New_York')` value in that zone).
+    const at = typeof value === 'number' ? { ms: value * 1000, ns: 0 } : parseDateTimeInstant(String(value), zone ?? SERVER_TZ);
+    return makeDT(at?.ms ?? 0, at?.ns ?? 0, scale, zone);
+  }
+  if (h === 'Date' || h === 'Date32') {
+    if (value instanceof DDate) return value;
+    return new DDate(parseDateText(String(value)) ?? 0);
+  }
+  if (h === 'Array') {
+    const inner = typeArgs(base)[0] ?? 'String';
+    return Array.isArray(value) ? value.map((v) => fromStored(v, inner)) : [];
+  }
+  if (h === 'Map') {
+    if (value instanceof ChMap) return value;
+    const [kt, vt] = typeArgs(base);
+    const pairs: Array<[unknown, unknown]> = Array.isArray(value) ? (value as Array<[unknown, unknown]>) : typeof value === 'object' ? Object.entries(value as Record<string, unknown>) : [];
+    return new ChMap(pairs.map(([k, v]) => [fromStored(k, kt ?? 'String'), fromStored(v, vt ?? 'String')]));
+  }
+  if (h === 'AggregateFunction') {
+    if (value instanceof AggState) return value;
+    if (typeof value === 'object' && value !== null && '$agg' in (value as Record<string, unknown>)) {
+      return stateFromStored(value as { $agg: string; value: unknown }, base);
+    }
+    return null;
+  }
+  if (h === 'Bool' || h === 'Boolean') return value === true || value === 1 || value === 'true';
+  if (INTS.has(h) || UINTS.has(h)) {
+    if (typeof value === 'number') return value;
+    if (typeof value === 'bigint') return normInt(value);
+    if (typeof value === 'string' && /^[+-]?\d+$/.test(value.trim())) return normInt(BigInt(value.trim()));
+    return Number(value);
+  }
+  if (FLOATS.has(h)) return typeof value === 'number' ? value : Number(value);
+  return value;
+}
+
+/**
+ * An engine value → the JSON a Data Source stores for its column type (a materialized view's
+ * insert). The value is CAST to the column type the way ClickHouse converts an INSERT … SELECT
+ * block: integers wrap to the column's width (`CAST(300 AS UInt8)` = 44) and a string that is not
+ * an integer is ClickHouse's parse error, which fails the insert.
+ */
+export function toStored(value: unknown, type: string): unknown {
+  const { base, nullable } = baseType(type);
+  const h = base.replace(/\(.*$/s, '').trim();
+  if (value === null || value === undefined) return nullable ? null : toStored(defaultOf(type), base);
+  if (h === 'DateTime' || h === 'DateTime64') {
+    const v = castDateTime(value, typeScale(base), typeZone(base));
+    return formatStoredDT(new DT(v.ms, v.scale, typeZone(base) ?? SERVER_TZ, v.ns));
+  }
+  if (h === 'Date' || h === 'Date32') {
+    if (value instanceof DDate) return formatDate(value);
+    if (value instanceof DT) { const w = wallOf(value.ms, value.tz); return formatDate(new DDate(daysFromCivil(w.y, w.mo, w.d))); }
+    return String(value);
+  }
+  if (h === 'Array') {
+    const inner = typeArgs(base)[0] ?? 'String';
+    return Array.isArray(value) ? value.map((v) => toStored(v, inner)) : [];
+  }
+  if (h === 'Map') {
+    const vt = typeArgs(base)[1] ?? 'String';
+    return value instanceof ChMap ? Object.fromEntries(value.entries.map(([k, v]) => [valueToText(k), toStored(v, vt)])) : {};
+  }
+  if (h === 'AggregateFunction') {
+    if (value instanceof AggState) return stateToStored(value, base);
+    return null;
+  }
+  if (h === 'Bool' || h === 'Boolean') return value === true || value === 1;
+  if (INTS.has(h) || UINTS.has(h)) return storedInt(wrapInt(toBigInt(typeof value === 'string' ? value.trim() : value, base), base));
+  if (FLOATS.has(h)) return typeof value === 'number' ? value : Number(value);
+  if (h === 'String' || h === 'FixedString') return valueToText(value);
+  return value;
+}
+
+export function castDateTime(value: unknown, scale: number, tz?: string): DT {
+  if (value instanceof DT) return makeDT(value.ms, value.ns, scale, tz ?? value.tz);
+  if (value instanceof DDate) {
+    const c = civilFromDays(value.days);
+    return new DT(instantOf({ ...c, h: 0, mi: 0, s: 0, ms: 0 }, tz ?? SERVER_TZ), scale, tz);
+  }
+  if (typeof value === 'bigint') return makeDT(Number(value) * 1000, 0, scale, tz);
+  if (typeof value === 'number') return makeDT(value * 1000, 0, scale, tz);
+  if (typeof value === 'string') {
+    const at = parseDateTimeInstant(value, tz ?? SERVER_TZ);
+    if (at === undefined) throw new ClickHouseError(`Cannot parse DateTime from String '${value}'`);
+    return makeDT(at.ms, at.ns, scale, tz);
+  }
+  throw new ClickHouseError(`Cannot convert ${typeof value} to DateTime`);
+}
+
+/** `toString(x)` — ClickHouse's text form of a value. */
+export function valueToText(value: unknown): string {
+  if (value === null || value === undefined) return '';
+  if (typeof value === 'string') return value;
+  if (typeof value === 'boolean') return value ? 'true' : 'false';
+  if (typeof value === 'bigint') return value.toString();
+  if (typeof value === 'number') return Number.isFinite(value) ? String(value) : Number.isNaN(value) ? 'nan' : value > 0 ? 'inf' : '-inf';
+  if (value instanceof DT) return formatDT(value);
+  if (value instanceof DDate) return formatDate(value);
+  if (Array.isArray(value)) return `[${value.map(quotedText).join(',')}]`;
+  if (value instanceof ChMap) return `{${value.entries.map(([k, v]) => `${quotedText(k)}:${quotedText(v)}`).join(',')}}`;
+  if (value instanceof AggState) return '';
+  return JSON.stringify(value);
+}
+/** An element inside an array's text form: strings, dates and times are single-quoted and escaped. */
+export function quotedText(v: unknown): string {
+  if (v === null || v === undefined) return 'NULL';
+  if (typeof v === 'string') return `'${v.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+  if (v instanceof DT || v instanceof DDate) return `'${valueToText(v)}'`;
+  return valueToText(v);
+}
+
+/** An engine value → the JSON a `.json` response carries (a bigint stays exact). */
+export function toOutput(value: unknown): unknown {
+  if (value === null || value === undefined) return null;
+  if (value instanceof DT) return formatDT(value);
+  if (value instanceof DDate) return formatDate(value);
+  if (value instanceof AggState) return '';
+  if (value instanceof Interval) return value.n;
+  if (Array.isArray(value)) return value.map(toOutput);
+  // a Map is a JSON object in a JSON answer (ClickHouse's JSON format writes Map as an object)
+  if (value instanceof ChMap) return Object.fromEntries(value.entries.map(([k, v]) => [valueToText(k), toOutput(v)]));
+  if (typeof value === 'number' && !Number.isFinite(value)) return null;
+  return value;
+}
+
+// ── comparison ──────────────────────────────────────────────────────────────────────────────
+
+function asInstant(v: unknown): DT | undefined {
+  if (v instanceof DT) return v;
+  if (v instanceof DDate) return new DT(v.days * 86_400_000, 0);
+  return undefined;
+}
+
+/**
+ * Compare two non-null values ClickHouse-style. A string meeting a DateTime/Date is parsed in the
+ * other side's zone (ClickHouse converts the literal to the column's type).
+ */
+export function compareValues(a: unknown, b: unknown): number {
+  if (a instanceof DT || b instanceof DT || a instanceof DDate || b instanceof DDate) {
+    let x = asInstant(a);
+    let y = asInstant(b);
+    if (x === undefined) x = coerceToInstant(a, b);
+    if (y === undefined) y = coerceToInstant(b, a);
+    if (a instanceof DDate && b instanceof DT) x = castDateTime(a, 0, b.tz);
+    if (b instanceof DDate && a instanceof DT) y = castDateTime(b, 0, a.tz);
+    return compareDT(x, y);
+  }
+  if (typeof a === 'boolean') a = a ? 1 : 0;
+  if (typeof b === 'boolean') b = b ? 1 : 0;
+  const numA = typeof a === 'number' || typeof a === 'bigint';
+  const numB = typeof b === 'number' || typeof b === 'bigint';
+  if (numA && numB) {
+    if (Number.isNaN(a) || Number.isNaN(b)) return Number.isNaN(a) ? (Number.isNaN(b) ? 0 : 1) : -1;
+    return (a as number) < (b as number) ? -1 : (a as number) > (b as number) ? 1 : 0;
+  }
+  if (numA && typeof b === 'string') { const n = parseNumberText(b); if (n !== undefined) return compareValues(a, n); }
+  if (typeof a === 'string' && numB) { const n = parseNumberText(a); if (n !== undefined) return compareValues(n, b); }
+  if (a instanceof ChMap && b instanceof ChMap) return compareValues(a.entries.flat(), b.entries.flat());
+  if (Array.isArray(a) && Array.isArray(b)) {
+    for (let i = 0; i < Math.min(a.length, b.length); i++) {
+      const c = a[i] === null || b[i] === null ? (a[i] === null ? (b[i] === null ? 0 : 1) : -1) : compareValues(a[i], b[i]);
+      if (c !== 0) return c;
+    }
+    return a.length - b.length;
+  }
+  const sa = typeof a === 'string' ? a : valueToText(a);
+  const sb = typeof b === 'string' ? b : valueToText(b);
+  // ClickHouse compares strings bytewise.
+  const ba = Buffer.from(sa, 'utf8');
+  const bb = Buffer.from(sb, 'utf8');
+  return Buffer.compare(ba, bb);
+}
+/** Numeric text as an exact engine number (an integer beyond 2^53 as a bigint). */
+function parseNumberText(s: string): number | bigint | undefined {
+  const t = s.trim();
+  if (t === '') return undefined;
+  if (/^[+-]?\d+$/.test(t)) return normInt(BigInt(t));
+  const n = Number(t);
+  return Number.isNaN(n) ? undefined : n;
+}
+
+function coerceToInstant(v: unknown, other: unknown): DT {
+  const tz = other instanceof DT ? other.tz : undefined;
+  if (typeof v === 'string') {
+    if (other instanceof DDate) {
+      const d = parseDateText(v);
+      if (d === undefined) throw new ClickHouseError(`Cannot parse Date from String '${v}'`);
+      return new DT(d * 86_400_000, 0);
+    }
+    const at = parseDateTimeInstant(v, tz ?? SERVER_TZ);
+    if (at === undefined) throw new ClickHouseError(`Cannot parse DateTime from String '${v}'`);
+    return new DT(at.ms, 9, tz, at.ns);
+  }
+  if (typeof v === 'number' || typeof v === 'bigint') return other instanceof DDate ? new DT(Number(v) * 86_400_000, 0) : makeDT(Number(v) * 1000, 0, 9);
+  throw new ClickHouseError(`Cannot compare ${valueToText(v)} with a DateTime`);
+}
+
+export function valuesEqual(a: unknown, b: unknown): boolean {
+  return compareValues(a, b) === 0;
+}
+
+/** A hash key under which equal values collide (for GROUP BY, JOIN, IN, uniq). */
+export function valueKey(v: unknown): string {
+  if (v === null || v === undefined) return 'N';
+  if (typeof v === 'boolean') return `n:${v ? 1 : 0}`;
+  if (typeof v === 'number' || typeof v === 'bigint') return `n:${v}`;
+  if (typeof v === 'string') return `s:${v}`;
+  if (v instanceof DT) return v.ns ? `t:${v.ms}.${v.ns}` : `t:${v.ms}`;
+  if (v instanceof DDate) return `t:${v.days * 86_400_000}`;
+  if (Array.isArray(v)) return `a:[${v.map(valueKey).join(',')}]`;
+  if (v instanceof ChMap) return `m:{${v.entries.map(([k, x]) => `${valueKey(k)}=${valueKey(x)}`).join(',')}}`;
+  if (v instanceof AggState) return `g:${v.fn}:${valueKey(v.value)}`;
+  return `o:${JSON.stringify(v)}`;
+}
+
+/** A string literal meeting a DateTime in a hash key must hash as the instant. */
+export function valueKeyAs(v: unknown, like: unknown): string {
+  if (typeof v === 'string' && (like instanceof DT || like instanceof DDate)) {
+    try {
+      const at = coerceToInstant(v, like);
+      return at.ns ? `t:${at.ms}.${at.ns}` : `t:${at.ms}`;
+    } catch {
+      return valueKey(v);
+    }
+  }
+  if (typeof v === 'string' && (typeof like === 'number' || typeof like === 'bigint')) {
+    const n = parseNumberText(v);
+    if (n !== undefined) return valueKey(n);
+  }
+  return valueKey(v);
+}