@clickhouse/client 1.22.0 → 1.23.0-head.70ad405.1

package/skills/clickhouse-js-node-rowbinary/src/readers/ip.ts

@@ -0,0 +1,93 @@
+import { Cursor, advance } from "./core.js";
+
+/**
+ * Read an `IPv4`: stored as a 4-byte little-endian `UInt32`. Returns the raw
+ * 32-bit value (the little-endian load already orders the octets); pass it to
+ * {@link formatIPv4} for the dotted-quad string.
+ */
+export function readIPv4(state: Cursor): number {
+  return state.view.getUint32(advance(state, 4), true);
+}
+
+/**
+ * Read an `IPv6`: 16 bytes in network (big-endian) order. Returns the raw bytes
+ * as a zero-copy view; pass them to {@link formatIPv6} for the canonical string.
+ *
+ * The view shares memory with the response buffer, so keeping it alive pins the
+ * whole response chunk in memory. If the value must outlive the row/response,
+ * copy it with `Buffer.from(...)`.
+ */
+export function readIPv6(state: Cursor): Buffer {
+  const start = advance(state, 16);
+  return state.buf.subarray(start, start + 16);
+}
+
+/**
+ * Format an `IPv4` (the raw 32-bit value from {@link readIPv4}) as a dotted-quad
+ * string. Kept aside so the hot read path can skip building a string when the
+ * numeric value is all the caller needs.
+ */
+export function formatIPv4(value: number): string {
+  return `${(value >>> 24) & 0xff}.${(value >>> 16) & 0xff}.${(value >>> 8) & 0xff}.${value & 0xff}`;
+}
+
+/**
+ * Join groups `[from, to)` as colon-separated lowercase hex, by concatenating
+ * into a string in a loop. Benchmarks faster than `slice().map().join(":")`,
+ * which allocates an intermediate array. Returns `""` for an empty range.
+ */
+function joinGroupsHex(g: number[], from: number, to: number): string {
+  if (from >= to) return "";
+  let s = g[from]!.toString(16);
+  for (let i = from + 1; i < to; i++) s += ":" + g[i]!.toString(16);
+  return s;
+}
+
+/**
+ * Format an `IPv6` (the raw 16 bytes from {@link readIPv6}) as the canonical
+ * RFC 5952 string: lowercase, no leading zeros, the longest run of zero groups
+ * (>= 2) collapsed to `::` (leftmost on a tie), and the `::ffff:a.b.c.d` form
+ * for IPv4-mapped addresses (matching ClickHouse).
+ *
+ * Kept aside from the read so the hot path only formats when a string is
+ * actually needed.
+ */
+export function formatIPv6(b: Buffer): string {
+  // IPv4-mapped (::ffff:a.b.c.d): first 10 bytes zero, then 0xffff.
+  let mapped = b[10] === 0xff && b[11] === 0xff;
+  for (let i = 0; mapped && i < 10; i++) {
+    if (b[i] !== 0) mapped = false;
+  }
+  if (mapped) {
+    return `::ffff:${b[12]}.${b[13]}.${b[14]}.${b[15]}`;
+  }
+
+  // Eight 16-bit groups, big-endian.
+  const g: number[] = [];
+  for (let i = 0; i < 8; i++) {
+    g.push((b[2 * i]! << 8) | b[2 * i + 1]!);
+  }
+
+  // Longest run of >= 2 zero groups becomes "::" (leftmost wins on a tie).
+  let bestStart = -1;
+  let bestLen = 0;
+  let curStart = -1;
+  let curLen = 0;
+  for (let i = 0; i < 8; i++) {
+    if (g[i] === 0) {
+      if (curStart < 0) curStart = i;
+      curLen++;
+      if (curLen > bestLen) {
+        bestLen = curLen;
+        bestStart = curStart;
+      }
+    } else {
+      curStart = -1;
+      curLen = 0;
+    }
+  }
+  if (bestLen < 2) {
+    return joinGroupsHex(g, 0, 8);
+  }
+  return `${joinGroupsHex(g, 0, bestStart)}::${joinGroupsHex(g, bestStart + bestLen, 8)}`;
+}

package/skills/clickhouse-js-node-rowbinary/src/readers/json.ts

@@ -0,0 +1,33 @@
+import { Cursor } from "./core.js";
+import { readUVarint } from "./varint.js";
+import { readString } from "./strings.js";
+import { readDynamic } from "./dynamic.js";
+
+/**
+ * Read a `JSON` value. ClickHouse's `JSON` is NOT JSON text and NOT BSON — it is
+ * a list of (path, value) pairs built on the same machinery as `Dynamic`:
+ *
+ *   <varuint pathCount>  then pathCount x ( <String path> <Dynamic value> )
+ *
+ * Nested objects are FLATTENED to dotted paths (`{a:{b:2}}` -> path `"a.b"`), and
+ * each leaf value is a self-describing `Dynamic`, so this just loops
+ * {@link readString} + {@link readDynamic}. Returns a `Map` keyed by the flat
+ * dotted path. Path order on the wire is not significant.
+ *
+ * GOTCHA: a null-valued path is NOT stored at all — `{"a":null}` serializes as
+ * zero paths, identical to `{}`. JSON arrays come back as `Array(Nullable(T))`.
+ *
+ * LIMITATION — typed paths only. This reads a plain `JSON` column, where every
+ * path is dynamic (tagged). A `JSON(a T, ...)` with DECLARED typed paths
+ * serializes those paths' values WITHOUT a type tag, so they cannot be decoded
+ * without the schema; read each typed path with its known `T` reader instead.
+ */
+export function readJSON(state: Cursor): Map<string, unknown> {
+  const n = readUVarint(state);
+  const out = new Map<string, unknown>();
+  for (let i = 0; i < n; i++) {
+    const path = readString(state);
+    out.set(path, readDynamic(state));
+  }
+  return out;
+}

package/skills/clickhouse-js-node-rowbinary/src/readers/lowCardinality.ts

@@ -0,0 +1,18 @@
+import { type Reader } from "./core.js";
+
+/**
+ * `LowCardinality(T)` is TRANSPARENT in RowBinary: it is encoded byte-for-byte
+ * the same as `T`, with NO dictionary/index layer. (The dictionary encoding
+ * exists only in the Native format — do not look for it here.) So there is
+ * nothing to decode at this level: use `T`'s own reader directly.
+ *
+ * This identity combinator exists only to document that, and to let a generated
+ * parser name the wrapper at the call site if it wants the type to read
+ * literally — it returns the inner reader unchanged:
+ *
+ *   readLowCardinality(readString) === readString
+ *
+ * Prefer just calling the inner reader.
+ */
+export const readLowCardinality = <T>(readValue: Reader<T>): Reader<T> =>
+  readValue;

package/skills/clickhouse-js-node-rowbinary/src/readers/nested.ts

@@ -0,0 +1,23 @@
+import { readArray, readTupleNamed } from "./composite.js";
+import { type Reader } from "./core.js";
+
+/**
+ * `Nested(a T1, b T2, …)` has NO wire format of its own:
+ *  - `flatten_nested = 1` (the default): the column expands into separate
+ *    columns `a Array(T1)`, `b Array(T2)`, … — decode each with `readArray`.
+ *  - `flatten_nested = 0`: the column is `Array(Tuple(a T1, b T2, …))` — decode
+ *    with `readArray` + `readTupleNamed`.
+ *
+ * Either way it reuses existing readers; there is no dedicated Nested wire. This
+ * thin alias just composes the two for the `flatten_nested = 0` shape, as
+ * documentation that "Nested === Array(Tuple(...))":
+ *
+ *   readNested({ a: readUInt8, b: readString })
+ *     === readArray(readTupleNamed({ a: readUInt8, b: readString }))
+ *
+ * When generating code, prefer inlining (monomorphize the array + tuple) over
+ * this generic composition.
+ */
+export const readNested = <T extends Record<string, unknown>>(fields: {
+  [K in keyof T]: Reader<T[K]>;
+}): Reader<T[]> => readArray(readTupleNamed(fields));

package/skills/clickhouse-js-node-rowbinary/src/readers/nothing.ts

@@ -0,0 +1,29 @@
+import { type Reader } from "./core.js";
+
+/**
+ * `Nothing` is the empty type: it has NO values and occupies ZERO bytes. It is
+ * never a column on its own (you cannot materialize a value of it) — it only
+ * appears wrapped, as the inferred element of an untyped literal:
+ *
+ *   []   -> Array(Nothing)     -> always the empty array (varint length 0x00)
+ *   NULL -> Nullable(Nothing)  -> always NULL (lone flag byte 0x01)
+ *
+ * So a `Nothing` value is NEVER read: `readArray`'s element reader and
+ * `readNullable`'s inner reader are not called in those cases (the array is
+ * empty / the value is NULL). There is nothing to decode.
+ *
+ * Wire this in as the inner reader to make that invariant loud: it throws if it
+ * is ever actually invoked, which would mean a `Nothing` reader was placed where
+ * a real element/inner type was expected.
+ *
+ *   readArray(readNothing)      // [] — readNothing never runs
+ *   readNullable(readNothing)   // null — readNothing never runs
+ */
+export const readNothing: Reader<never> = () => {
+  throw new Error(
+    "RowBinary: Nothing is zero-width and is never decoded — it only appears as " +
+      "an empty Array(Nothing) or a NULL Nullable(Nothing), where the inner reader " +
+      "is not called. Reaching here means a Nothing reader was wired where a real " +
+      "element/inner type was expected.",
+  );
+};

package/skills/clickhouse-js-node-rowbinary/src/readers/reader.ts

@@ -0,0 +1,68 @@
+/**
+ * Barrel re-export of the RowBinary reader, split by type family into the
+ * sibling modules. Import from here for everything in one place, or from a
+ * specific module (e.g. `./integers.js`, `./strings.js`) to pull in only the
+ * sub-parsers a given result actually needs — the latter is what a generated
+ * parser should do, copying just the modules its column types require.
+ *
+ * - core      — Cursor, Reader<T>, advance, NeedMoreData
+ * - varint    — readUVarint
+ * - integers  — readUInt8..readUInt256, readInt8..readInt256
+ * - bool / enums / floats
+ * - decimals  — DecimalValue, formatDecimal, readDecimal32..256
+ * - strings   — readString, readFixedString, readFixedStringBytes
+ * - uuid      — readUUID(+BigInt/HiLo), formatUUID(+Table)
+ * - ip        — readIPv4/6, formatIPv4/6
+ * - datetime / time / interval
+ * - composite — readArray/Map/Tuple/TupleNamed/Nullable/Variant/QBit
+ * - rows      — readRows
+ * - geo       — Point, readPoint/Ring/LineString/Polygon/MultiLineString/MultiPolygon/Geometry
+ * - dynamic   — readDynamic, readDynamicType
+ * - json      — readJSON
+ * - stream    — streamRowBatches, coalesceChunks
+ * - transparent / special wrappers (mostly documentation; see each file):
+ *   lowCardinality (readLowCardinality), simpleAggregateFunction
+ *   (readSimpleAggregateFunction), nested (readNested), nothing (readNothing),
+ *   aggregateFunction (readAggregateFunction)
+ *
+ * Runtime schema path — compile a reader from the type STRINGS rather than
+ * hand-/code-generating one. Use this when the column types are not known until
+ * the response arrives (or you just want a generic decoder); for a fixed,
+ * known schema the specialized straight-line reader is faster.
+ * - header   — readHeader: the RowBinaryWithNamesAndTypes preamble (column
+ *              names + type strings) off the cursor
+ * - compile  — astToReader: fold one parsed type AST (from
+ *              `@clickhouse/datatype-parser`) into a value Reader. AST in,
+ *              reader out — the type-to-combinator mapping, nothing else
+ * - rowBinaryWithNamesAndTypes — typeStringToReader (parse a type string +
+ *              fold) and compileRowBinaryWithNamesAndTypes (read the header,
+ *              compile every column, return a `readRows` driver for the rest of
+ *              the stream): the end-to-end runtime entry point
+ */
+export * from "./core.js";
+export * from "./varint.js";
+export * from "./integers.js";
+export * from "./bool.js";
+export * from "./enums.js";
+export * from "./floats.js";
+export * from "./decimals.js";
+export * from "./strings.js";
+export * from "./uuid.js";
+export * from "./ip.js";
+export * from "./datetime.js";
+export * from "./time.js";
+export * from "./interval.js";
+export * from "./composite.js";
+export * from "./rows.js";
+export * from "./geo.js";
+export * from "./dynamic.js";
+export * from "./json.js";
+export * from "./stream.js";
+export * from "./lowCardinality.js";
+export * from "./simpleAggregateFunction.js";
+export * from "./nested.js";
+export * from "./nothing.js";
+export * from "./aggregateFunction.js";
+export * from "./header.js";
+export * from "./compile.js";
+export * from "./rowBinaryWithNamesAndTypes.js";

package/skills/clickhouse-js-node-rowbinary/src/readers/rowBinaryWithNamesAndTypes.ts

@@ -0,0 +1,155 @@
+/**
+ * The `RowBinaryWithNamesAndTypes` entry point: read the header off a cursor,
+ * compile each column's type string into a reader, and hand back a driver that
+ * decodes the rest of the stream.
+ *
+ * This ties together the pieces: {@link readHeader} (wire), the parser
+ * (`@clickhouse/datatype-parser`), and {@link astToReader} (the AST → reader
+ * fold in `compile.ts`), then assembles a named-tuple row reader over the
+ * columns and a {@link readRows} driver for the row data.
+ */
+
+import { parseDataType } from "@clickhouse/datatype-parser";
+
+import type { Reader, Cursor } from "./core.js";
+import { readHeader } from "./header.js";
+import { readRows } from "./rows.js";
+import { astToReader, RowBinaryTypeError } from "./compile.js";
+
+/** One decoded row, keyed by column name. */
+export type Row = Record<string, unknown>;
+
+/**
+ * The product of compiling a `RowBinaryWithNamesAndTypes` header: the column
+ * metadata, the per-column readers, and — the headline — `readRows`, the
+ * {@link Reader} that decodes every remaining row of the stream.
+ */
+export interface CompiledStream {
+  /** Column names, in stream order (from the header). */
+  names: string[];
+  /** Column type strings, in stream order (from the header). */
+  types: string[];
+  /** One folded reader per column, in stream order. */
+  columnReaders: Reader<unknown>[];
+  /** Reads exactly one row into a `{ [name]: value }` object. */
+  readRow: Reader<Row>;
+  /**
+   * Reads the REST of the stream (all rows after the header) into an array.
+   * Streaming-aware via {@link readRows}: on a partial trailing row it rewinds
+   * to the last complete row and returns what it has.
+   */
+  readRows: Reader<Row[]>;
+}
+
+/**
+ * Parse one ClickHouse type string and fold it into a {@link Reader}. Throws a
+ * {@link RowBinaryTypeError} if the parser rejects the string (e.g. the
+ * deliberately unsupported `AggregateFunction` / `SimpleAggregateFunction`) —
+ * carrying the `typeString` and the parse `position`.
+ */
+export function typeStringToReader(typeStr: string): Reader<unknown> {
+  const result = parseDataType(typeStr);
+  if (!result.ok()) {
+    const err = result.error!;
+    throw new RowBinaryTypeError(
+      `cannot compile type ${JSON.stringify(typeStr)}: ${err.message}`,
+      { typeString: typeStr, position: err.position },
+    );
+  }
+  return astToReader(result.ast!);
+}
+
+/** Resolves a ClickHouse type string to a reader — `typeStringToReader` or a cache wrapping it. */
+export type TypeReaderResolver = (typeStr: string) => Reader<unknown>;
+
+/**
+ * Build an LRU-cached {@link typeStringToReader}. The full ClickHouse type
+ * STRING is a perfect cache key: two columns of the same type compile to the
+ * same reader, and a reader is stateless (it only ever touches the cursor it is
+ * handed), so one instance is safe to share across columns and across streams —
+ * a cache hit skips the parse + AST fold entirely.
+ *
+ * Worth it when you decode many `RowBinaryWithNamesAndTypes` responses whose
+ * schemas overlap (e.g. the same query run repeatedly): keep one cache and pass
+ * it to {@link compileRowBinaryWithNamesAndTypes}, so a recurring type is
+ * compiled once rather than once per response. A single response rarely repeats
+ * a type across its own columns, so the win is across calls, not within one.
+ *
+ * Classic Map-based LRU: a `Map` iterates in insertion order, so on a HIT we
+ * delete + re-set the entry to move it to the most-recently-used end, and on
+ * overflow we evict the oldest key (the first the `Map` yields). `maxSize` caps
+ * memory. A parse FAILURE is never cached — {@link typeStringToReader} throws
+ * before anything is stored — so fixing a bad type is not shadowed by a cached
+ * error.
+ */
+export function createTypeReaderCache(maxSize = 256): TypeReaderResolver {
+  const cache = new Map<string, Reader<unknown>>();
+  return (typeStr) => {
+    const cached = cache.get(typeStr);
+    if (cached !== undefined) {
+      // Touch on hit. A Map iterates in INSERTION order, not usage order — so on
+      // its own `keys().next()` would give the oldest-added key, not the
+      // least-recently-USED one. Deleting and re-inserting moves this key to the
+      // tail, which is what turns insertion order INTO recency order: every
+      // access (hit here, or miss below) lands the key at the tail, leaving the
+      // head as the genuine least-recently-used entry.
+      cache.delete(typeStr);
+      cache.set(typeStr, cached);
+      return cached;
+    }
+    const reader = typeStringToReader(typeStr); // may throw — then nothing is cached
+    cache.set(typeStr, reader);
+    if (cache.size > maxSize) {
+      // The head is the least-recently-used key (see touch-on-hit above), so it
+      // is the correct one to evict.
+      const lru = cache.keys().next().value;
+      if (lru !== undefined) cache.delete(lru);
+    }
+    return reader;
+  };
+}
+
+/**
+ * The headline entry point. Reads the `RowBinaryWithNamesAndTypes` header off
+ * `state`, compiles each column type into a combinator reader, and returns the
+ * column metadata plus the readers — including `readRows`, the reader for the
+ * REST of the stream. After this call the cursor sits at the first row, so:
+ *
+ *   const s = new Cursor(buf);
+ *   const { names, readRows } = compileRowBinaryWithNamesAndTypes(s);
+ *   const rows = readRows(s);   // decode every remaining row
+ *
+ * Pass `resolveType` to reuse readers across calls — e.g. a shared
+ * {@link createTypeReaderCache}. It defaults to {@link typeStringToReader}
+ * (compile every column afresh).
+ */
+export function compileRowBinaryWithNamesAndTypes(
+  state: Cursor,
+  resolveType: TypeReaderResolver = typeStringToReader,
+): CompiledStream {
+  const { names, types } = readHeader(state);
+  const columnReaders = types.map((t) => resolveType(t));
+
+  // Build the row reader POSITIONALLY — by column index, NOT by keying the
+  // readers on column name and handing them to `readTupleNamed`. The header is
+  // an ordered list and RowBinary has no row delimiter, so every row MUST read
+  // exactly these readers, in exactly this order. Keying readers by name first
+  // would corrupt the stream on legal-but-awkward headers:
+  //   - duplicate column names (e.g. two `SELECT 1 AS x, 2 AS x`) collapse to a
+  //     single entry in a `Record`, so fewer readers run than there are columns;
+  //   - integer-like names (`0`, `1`, …) are reordered ahead of string keys by
+  //     `Object.keys()`, so the readers would run out of header order.
+  // Either desyncs the cursor and misreads every subsequent row. Reading by
+  // index sidesteps both. The row OBJECT is still keyed by name; on a duplicate
+  // name the last column with that name wins in the object, but every column is
+  // still consumed off the wire in order, so the cursor stays in sync.
+  const readRow: Reader<Row> = (s) => {
+    const row: Row = {};
+    for (let i = 0; i < columnReaders.length; i++) {
+      row[names[i]!] = columnReaders[i]!(s);
+    }
+    return row;
+  };
+
+  return { names, types, columnReaders, readRow, readRows: readRows(readRow) };
+}

package/skills/clickhouse-js-node-rowbinary/src/readers/rows.ts

@@ -0,0 +1,58 @@
+import { NeedMoreData, type Reader } from "./core.js";
+
+/**
+ * Drive `readRow` over every row of a plain `RowBinary` result into an array.
+ * Curried: `readRows(readRow)` returns a `Reader<T[]>`. Rows are concatenated on
+ * the wire with no count, length prefix, or delimiter, so the result is exhausted
+ * only when the cursor reaches the buffer end.
+ *
+ * `readRow` must consume EXACTLY one row's bytes — a byte short or long compounds
+ * across rows and the cursor overshoots or never lands on `buf.length`. Returns
+ * `[]` for an empty buffer. When generating code, inline the per-column reads
+ * into the loop body:
+ *
+ *   function readRowsUser(s) {
+ *     const out = [];
+ *     while (s.pos < s.buf.length) {
+ *       out.push({ id: readUInt64(s), name: readString(s) });
+ *     }
+ *     return out;
+ *   }
+ *
+ * STREAMING (partial trailing row): a chunk of a still-arriving response may end
+ * mid-row. `pos` is committed only AFTER a row reads cleanly, so when a row
+ * starves and `readRow` throws {@link NeedMoreData}, this catches it, rewinds
+ * `pos` to the last complete row boundary, and returns the rows so far — never a
+ * half-built row. The cursor is left at the straddling row, a commit point the
+ * driver carries forward:
+ *
+ *   const drive = readRows(readRow);
+ *   let committed = 0;
+ *   for (const chunk of chunks) {              // chunk = growing prefix
+ *     const s = new Cursor(chunk);
+ *     s.pos = committed;
+ *     emit(drive(s));                          // complete rows in this chunk
+ *     committed = s.pos;                        // start of the straddling row
+ *   }
+ *
+ * On a complete buffer no read starves, so the catch never runs. Errors other
+ * than {@link NeedMoreData} are real decode faults and propagate. See also
+ * `streamRowBatches`, the async driver built on this.
+ */
+export function readRows<T>(readRow: Reader<T>): Reader<T[]> {
+  return (state) => {
+    const out: T[] = [];
+    let committed = state.pos;
+    try {
+      while (state.pos < state.buf.length) {
+        const row = readRow(state);
+        committed = state.pos; // row read cleanly — advance the commit point
+        out.push(row);
+      }
+    } catch (e) {
+      if (e !== NeedMoreData) throw e;
+      state.pos = committed; // drop the partial trailing row; resume next chunk
+    }
+    return out;
+  };
+}

package/skills/clickhouse-js-node-rowbinary/src/readers/simpleAggregateFunction.ts

@@ -0,0 +1,20 @@
+import { type Reader } from "./core.js";
+
+/**
+ * `SimpleAggregateFunction(func, T)` is TRANSPARENT in RowBinary: the column
+ * already holds a finished value of the underlying type `T` (the partial
+ * aggregate of a "simple" function — sum / min / max / groupArrayArray / … — is
+ * just a value of `T`), so it is encoded byte-for-byte the same as `T`. Decode
+ * the inner `T` directly.
+ *
+ * Do NOT confuse it with `AggregateFunction(func, T)`, whose value is an opaque
+ * serialized aggregation STATE with a function-specific binary layout — see
+ * `./aggregateFunction.js`.
+ *
+ * Identity combinator, documentation only:
+ *
+ *   readSimpleAggregateFunction(readUInt64) === readUInt64
+ */
+export const readSimpleAggregateFunction = <T>(
+  readValue: Reader<T>,
+): Reader<T> => readValue;