@clickhouse/client 1.22.0 → 1.23.0-head.c8dc8d8.1

package/skills/clickhouse-js-node-rowbinary-parser/src/composite.ts

@@ -0,0 +1,181 @@
+import { type Reader } from "./core.js";
+import { readUInt8 } from "./integers.js";
+import { readUVarint } from "./varint.js";
+
+/**
+ * Read a `Nullable(T)`: a 1-byte null flag (0 = present, non-zero = NULL).
+ * Curried: pass the inner reader, get a `Reader<T | null>`.
+ *
+ * GOTCHA: the inner value bytes follow ONLY when the flag is 0. A NULL is the
+ * single `0x01` flag byte with nothing after it — so do NOT read the inner value
+ * when the flag is set, or the cursor desyncs.
+ *
+ * `readValue` decodes the inner `T`. This generic combinator is the reference
+ * shape; when generating code, MONOMORPHIZE — emit a dedicated `readNullableX`
+ * that inlines the inner read:
+ *
+ *   const readNullableUInt32 = (s) => readUInt8(s) !== 0 ? null : readUInt32(s);
+ */
+export function readNullable<T>(readValue: Reader<T>): Reader<T | null> {
+  return (state) => (readUInt8(state) !== 0 ? null : readValue(state));
+}
+
+/**
+ * Read an `Array(T)`: a LEB128 element count, then that many `T` values
+ * back-to-back. An empty array is just the count byte `0x00`. Curried: pass the
+ * element reader, get a `Reader<T[]>`.
+ *
+ * ARRAY LAYOUT: the count is known up front (the LEB128 prefix), so a generated
+ * reader can pre-size. Pick by how the result is used:
+ *  - small / consumed-as-is (the common case) → DEFAULT to `new Array(n)` +
+ *    index assignment; it skips `push`'s repeated capacity growth. A clean-room
+ *    benchmark found this edged out `push` on the small composite arrays here
+ *    (`baseline/README.md`).
+ *  - large + iterated/computed-over downstream → `[]` + `push` keeps it a PACKED
+ *    elements kind (faster to traverse; a pre-sized array is HOLEY), or use a
+ *    typed array (`Float64Array`…) for numeric elements.
+ * This generic combinator uses `push` for simplicity; the monomorphized
+ * `readArrayX` below should choose per the rule above.
+ *
+ * `readElement` decodes one element. This generic combinator is the reference
+ * shape; when generating code, MONOMORPHIZE — emit a dedicated `readArrayX` that
+ * inlines the element read in the loop (and pre-sizes for the common small case):
+ *
+ *   function readArrayUInt32(s) {
+ *     const n = readUVarint(s);
+ *     const out = new Array(n);
+ *     for (let i = 0; i < n; i++) out[i] = readUInt32(s);
+ *     return out;
+ *   }
+ */
+export function readArray<T>(readElement: Reader<T>): Reader<T[]> {
+  return (state) => {
+    const n = readUVarint(state);
+    const out: T[] = [];
+    for (let i = 0; i < n; i++) out.push(readElement(state));
+    return out;
+  };
+}
+
+/**
+ * Read a `QBit(element_type, dimension)` vector. `QBit` is a vector-search type
+ * whose ON-DISK layout is quantized and bit-transposed — but that is a STORAGE /
+ * Native-format concern. In RowBinary a `QBit` is fully TRANSPARENT: it is the
+ * plain vector, encoded byte-for-byte like `Array(element_type)` (a LEB128
+ * length, then `dimension` element values). So this is just {@link readArray}.
+ *
+ * `element_type` is one of `BFloat16` / `Float32` / `Float64`, so `readElement`
+ * is the matching float reader. When generating code, MONOMORPHIZE — inline the
+ * element read in the loop.
+ */
+export function readQBit<T>(readElement: Reader<T>): Reader<T[]> {
+  return readArray(readElement);
+}
+
+/**
+ * Read a `Tuple(...)` into a positional array: each element's value back-to-back,
+ * with NO count and NO delimiter. Curried: pass one reader per element (in
+ * order), get a `Reader` of the tuple. For a named tuple as an object, use
+ * {@link readTupleNamed} (identical wire).
+ *
+ * Reference shape; when generating code, MONOMORPHIZE — emit the inline sequence
+ * with no array-of-readers and no loop:
+ *
+ *   [readUInt32(s), readString(s)]
+ */
+export function readTuple<T extends readonly unknown[]>(readers: {
+  [K in keyof T]: Reader<T[K]>;
+}): Reader<T> {
+  return (state) => {
+    const out: unknown[] = [];
+    for (const read of readers as ReadonlyArray<Reader<unknown>>) {
+      out.push(read(state));
+    }
+    return out as unknown as T;
+  };
+}
+
+/**
+ * Read a named `Tuple(name1 T1, ...)` into an object. The wire is identical to
+ * an unnamed tuple — values back-to-back, no count or delimiter — so the
+ * `readers` object's keys MUST be listed in the tuple's declared field order
+ * (JS iterates string keys in insertion order), and each reader runs in that
+ * order. Curried: pass the readers object, get a `Reader` of the result object.
+ *
+ * Reference shape; when generating code, MONOMORPHIZE — emit the inline object
+ * literal instead of looping over entries:
+ *
+ *   { id: readUInt32(s), name: readString(s) }
+ */
+export function readTupleNamed<T extends Record<string, unknown>>(readers: {
+  [K in keyof T]: Reader<T[K]>;
+}): Reader<T> {
+  const fns = readers as Record<string, Reader<unknown>>;
+  const keys = Object.keys(fns);
+  return (state) => {
+    const out: Record<string, unknown> = {};
+    for (const key of keys) out[key] = fns[key]!(state);
+    return out as T;
+  };
+}
+
+/**
+ * Read a `Map(K, V)`: a LEB128 pair count, then that many key/value pairs with
+ * key and value interleaved (k, v, k, v, ...) — a flattened `Array(Tuple(K, V))`.
+ * An empty map is just the count byte `0x00`. Curried: pass the key and value
+ * readers, get a `Reader<Map<K, V>>`.
+ *
+ * The key is read BEFORE the value in each pair. Returns a JS `Map`, which keeps
+ * insertion order and accepts any key type.
+ *
+ * Reference shape; when generating code, MONOMORPHIZE — inline both reads in the
+ * loop.
+ */
+export function readMap<K, V>(
+  readKey: Reader<K>,
+  readValue: Reader<V>,
+): Reader<Map<K, V>> {
+  return (state) => {
+    const n = readUVarint(state);
+    const out = new Map<K, V>();
+    for (let i = 0; i < n; i++) {
+      const key = readKey(state);
+      out.set(key, readValue(state));
+    }
+    return out;
+  };
+}
+
+/**
+ * Read a `Variant(T1, ..., Tn)`: a 1-byte discriminant selecting the active
+ * alternative, then that alternative's value. Discriminant `0xFF` means NULL.
+ * Curried: pass the alternative readers (in sorted-type-name order), get a
+ * `Reader`.
+ *
+ * GOTCHA: the discriminant indexes the alternatives sorted by type NAME
+ * (ClickHouse globally sorts them), NOT their declaration order. So `readers`
+ * MUST be ordered by sorted type name. E.g. `Variant(UInt8, String)` sorts to
+ * ["String", "UInt8"], so discriminant 0 = String and 1 = UInt8.
+ *
+ * Reference shape; when generating code, MONOMORPHIZE — emit a `switch` over the
+ * discriminant with each branch inlined, alternatives in sorted order, `0xFF`
+ * -> null.
+ */
+export function readVariant<T extends readonly unknown[]>(readers: {
+  [K in keyof T]: Reader<T[K]>;
+}): Reader<T[number] | null> {
+  const fns = readers as ReadonlyArray<Reader<T[number]>>;
+  return (state) => {
+    const discriminant = readUInt8(state);
+    if (discriminant === 0xff) return null;
+    const fn = fns[discriminant];
+    if (fn === undefined) {
+      // Out-of-range discriminant (corrupted/truncated input): fail loudly
+      // instead of throwing a cryptic "fns[discriminant] is not a function".
+      throw new RangeError(
+        `RowBinary Variant: discriminant ${discriminant} out of range (${fns.length} alternatives)`,
+      );
+    }
+    return fn(state);
+  };
+}

package/skills/clickhouse-js-node-rowbinary-parser/src/core.ts

@@ -0,0 +1,77 @@
+/**
+ * Thrown by {@link advance} when the buffer lacks the bytes a read needs — the
+ * "need more bytes" signal for incremental decoding over a still-filling buffer.
+ * A driver catches it (`err === NeedMoreData`), waits for more input, and retries
+ * the row from its last committed position.
+ *
+ * A bare sentinel, NOT an `Error` subclass, on purpose: constructing an Error
+ * captures a stack trace — the expensive part of throwing — and on a path that
+ * starves once per chunk that cost is pure waste. Throwing a constant skips it,
+ * which is why throw + restart beats a generator's yield for realistic chunks
+ * (see `streamingRow.bench.ts`).
+ */
+export const NeedMoreData = Symbol("RowBinary.NeedMoreData");
+
+/**
+ * The cursor state every reader threads through: the input `Buffer`, the current
+ * position, and a `DataView` over the same bytes.
+ *
+ * Deliberately STATE only — no read methods. Decoding lives in the free
+ * `readX(state, ...)` functions in the sibling modules, so a generated parser
+ * pulls in only the per-type readers a result needs. `view`/`buf` are public so
+ * those free functions can reach them.
+ */
+export class Cursor {
+  pos = 0;
+
+  /**
+   * Node-only skill, so the input is a `Buffer`: number reads go through
+   * {@link Cursor.view} (DataView), while `String`/`FixedString` use the
+   * fast `buf.toString("utf8", ...)`.
+   */
+  readonly buf: Buffer;
+
+  /**
+   * `DataView` over the same bytes, for fixed-width integer/float reads. Built
+   * with the buffer's own `byteOffset`/`byteLength`: a `Buffer` is often a window
+   * into a larger pooled `ArrayBuffer`, so `new DataView(buf.buffer)` alone would
+   * point at the wrong bytes.
+   */
+  readonly view: DataView;
+
+  constructor(buf: Buffer) {
+    this.buf = buf;
+    this.view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  }
+}
+
+/**
+ * A `Reader<T>` decodes one value of type `T` from the cursor, advancing it. Leaf
+ * readers (e.g. `readUInt32`) are `Reader`s directly; combinators (e.g.
+ * `readArray`) take sub-`Reader`s and return a `Reader`, so types compose with no
+ * per-element closures.
+ */
+export type Reader<T> = (state: Cursor) => T;
+
+/**
+ * Reserve `n` bytes for the next read: bounds-check them, advance the cursor past
+ * them, and return the offset the read starts at (the value BEFORE advancing).
+ * Every fixed-width read goes through this, so the length check and cursor
+ * bookkeeping live in one place:
+ *
+ *   function readInt32(s) { return s.view.getInt32(advance(s, 4), true); }
+ *
+ * Throws {@link NeedMoreData} when fewer than `n` bytes remain, WITHOUT moving the
+ * cursor, so a driver can rewind to its last committed row and retry.
+ *
+ * SAFE TO TOGGLE: for a complete in-memory buffer the check never fires — a parser
+ * for that case can drop `advance` and read against `state.pos` directly, trading
+ * streaming tolerance for one fewer compare per read.
+ */
+export function advance(state: Cursor, n: number): number {
+  const start = state.pos;
+  const next = start + n;
+  if (next > state.buf.length) throw NeedMoreData;
+  state.pos = next;
+  return start;
+}

package/skills/clickhouse-js-node-rowbinary-parser/src/datetime.ts

@@ -0,0 +1,113 @@
+import { type Reader, Cursor } from "./core.js";
+import { readInt32, readInt64, readUInt16, readUInt32 } from "./integers.js";
+
+/**
+ * Semantic aliases for `number` that mark the unit of a temporal value in a
+ * return type. They are plain `number`s (no runtime brand) — purely to make
+ * `[Date, Nanoseconds]` etc. self-documenting at the call site.
+ */
+export type Milliseconds = number;
+export type Microseconds = number;
+export type Nanoseconds = number;
+
+/**
+ * Read a `Date`: 2-byte `UInt16` count of days since 1970-01-01 (UTC), returned
+ * as a JS `Date` at UTC midnight. A ClickHouse `Date` has no time or timezone;
+ * `.toISOString().slice(0, 10)` gives "YYYY-MM-DD".
+ *
+ * SAFE TO TOGGLE: a `Date` is an object allocation per value. On a hot path that
+ * only needs the calendar number, read the raw `UInt16` (days) instead.
+ */
+export function readDate(state: Cursor): Date {
+  return new Date(readUInt16(state) * 86_400_000);
+}
+
+/**
+ * Read a `Date32`: 4-byte signed `Int32` count of days since 1970-01-01 (UTC),
+ * returned as a JS `Date` at UTC midnight (pre-1970 dates are negative day
+ * counts, which `Date` handles).
+ */
+export function readDate32(state: Cursor): Date {
+  return new Date(readInt32(state) * 86_400_000);
+}
+
+/**
+ * Read a `DateTime` (and `DateTime(tz)`): 4-byte `UInt32` Unix seconds, returned
+ * as a JS `Date` (exact at second resolution). The instant is UTC-based; a
+ * column's timezone is display metadata, not in the bytes.
+ */
+export function readDateTime(state: Cursor): Date {
+  return new Date(readUInt32(state) * 1000);
+}
+
+/**
+ * Read a `DateTime64(P)` (and `DateTime64(P, tz)`): 8-byte signed `Int64` count
+ * of `10^-P`-second ticks since the epoch. Curried: `readDateTime64(P)` returns
+ * the reader.
+ *
+ * Returns a pair `[date, nanoseconds]`: `date` is a JS `Date` truncated to whole
+ * seconds, and `nanoseconds` is the sub-second remainder in nanoseconds
+ * (0..999_999_999). The split keeps full precision a `Date` alone (millisecond
+ * resolution) can't hold. `nanoseconds` is always in ns regardless of P. Timezone
+ * is metadata.
+ *
+ * For the typical precisions, prefer the specialized variants
+ * {@link readDateTime64P3} (ms — returns a plain `Date`),
+ * {@link readDateTime64P6} (µs), and {@link readDateTime64P9} (ns).
+ */
+export function readDateTime64(precision: number): Reader<[Date, Nanoseconds]> {
+  return (state) => {
+    const ticks = readInt64(state);
+    const scale = 10n ** BigInt(precision);
+    let sec = ticks / scale;
+    let frac = ticks % scale;
+    if (frac < 0n) {
+      // Floor toward -inf so the fractional remainder stays in [0, scale).
+      frac += scale;
+      sec -= 1n;
+    }
+    return [new Date(Number(sec) * 1000), Number(frac) * 10 ** (9 - precision)];
+  };
+}
+
+/**
+ * Read a `DateTime64(3)` ({@link Milliseconds}) — the most common precision — as
+ * a plain JS `Date`. P=3 is exactly a `Date`'s own millisecond resolution, so the
+ * instant is represented losslessly with no separate fraction. Specialized
+ * variant of {@link readDateTime64} with the scale baked in.
+ */
+export function readDateTime64P3(state: Cursor): Date {
+  return new Date(Number(readInt64(state)));
+}
+
+/**
+ * Read a `DateTime64(6)` (microseconds) as `[date, microseconds]`: a JS `Date`
+ * truncated to whole seconds plus the sub-second remainder in microseconds.
+ * Specialized variant of {@link readDateTime64}.
+ */
+export function readDateTime64P6(state: Cursor): [Date, Microseconds] {
+  const ticks = readInt64(state);
+  let sec = ticks / 1_000_000n;
+  let frac = ticks % 1_000_000n; // microseconds within the second
+  if (frac < 0n) {
+    frac += 1_000_000n;
+    sec -= 1n;
+  }
+  return [new Date(Number(sec) * 1000), Number(frac)];
+}
+
+/**
+ * Read a `DateTime64(9)` (nanoseconds) as `[date, nanoseconds]`: a JS `Date`
+ * truncated to whole seconds plus the sub-second remainder in nanoseconds.
+ * Specialized variant of {@link readDateTime64} with the scale baked in.
+ */
+export function readDateTime64P9(state: Cursor): [Date, Nanoseconds] {
+  const ticks = readInt64(state);
+  let sec = ticks / 1_000_000_000n;
+  let frac = ticks % 1_000_000_000n; // nanoseconds within the second
+  if (frac < 0n) {
+    frac += 1_000_000_000n;
+    sec -= 1n;
+  }
+  return [new Date(Number(sec) * 1000), Number(frac)];
+}

package/skills/clickhouse-js-node-rowbinary-parser/src/decimals.ts

@@ -0,0 +1,57 @@
+import { type Reader } from "./core.js";
+import { readInt32, readInt64, readInt128, readInt256 } from "./integers.js";
+
+/**
+ * A decimal kept lossless as its raw parts: `value = unscaled / 10 ** scale`.
+ * The `readDecimal*` readers return this so no precision or scale information is
+ * thrown away at decode time.
+ */
+export type DecimalValue = readonly [unscaled: bigint, scale: number];
+
+/**
+ * Format a {@link DecimalValue} as a fixed-point decimal string with `scale`
+ * fractional digits (e.g. `[15000n, 4]` -> `"1.5000"`). Plug in only when you
+ * need a string.
+ *
+ * Trailing zeros are preserved to reflect the declared scale, deliberately unlike
+ * ClickHouse's text output, which trims them (`"1.5"`) and drops the point for
+ * integers (`"10"`).
+ */
+export function formatDecimal([unscaled, scale]: DecimalValue): string {
+  if (scale === 0) return unscaled.toString();
+  if (unscaled < 0n) {
+    const digits = (-unscaled).toString().padStart(scale + 1, "0");
+    const point = digits.length - scale;
+    return `-${digits.slice(0, point)}.${digits.slice(point)}`;
+  }
+  const digits = unscaled.toString().padStart(scale + 1, "0");
+  const point = digits.length - scale;
+  return `${digits.slice(0, point)}.${digits.slice(point)}`;
+}
+
+/**
+ * Read a `Decimal32(P, S)`: a 4-byte little-endian signed integer (same wire
+ * shape as `Int32`) scaled by 10^S. Pass the column's scale `S`; returns a
+ * `Reader` of the raw `[unscaled, scale]` pair (see {@link formatDecimal}).
+ *
+ * `Decimal(P, S)` is an alias: pick the width reader by precision P — P<=9 ->
+ * Decimal32, <=18 -> Decimal64, <=38 -> Decimal128, <=76 -> Decimal256.
+ */
+export function readDecimal32(scale: number): Reader<DecimalValue> {
+  return (state) => [BigInt(readInt32(state)), scale];
+}
+
+/** Read a `Decimal64(P, S)`: 8-byte LE signed integer scaled by 10^S. Returns `[unscaled, scale]`; see {@link formatDecimal}. */
+export function readDecimal64(scale: number): Reader<DecimalValue> {
+  return (state) => [readInt64(state), scale];
+}
+
+/** Read a `Decimal128(P, S)`: 16-byte LE signed integer scaled by 10^S. Returns `[unscaled, scale]`; see {@link formatDecimal}. */
+export function readDecimal128(scale: number): Reader<DecimalValue> {
+  return (state) => [readInt128(state), scale];
+}
+
+/** Read a `Decimal256(P, S)`: 32-byte LE signed integer scaled by 10^S. Returns `[unscaled, scale]`; see {@link formatDecimal}. */
+export function readDecimal256(scale: number): Reader<DecimalValue> {
+  return (state) => [readInt256(state), scale];
+}