ch_connect 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8625111f390ba7adf290bf889a5684c34003d1f318f421c62939ae13a310f569
-  data.tar.gz: 512cc470c691018ac249647c3dc46918490705392667bee72ebb5f49bb604b44
+  metadata.gz: 5856233f30844a931e61ac5aca11af740c7a876a95ae19e497a3157bca9097b2
+  data.tar.gz: 6588fea1311f26c52e6b13885316741e47a98a284751456d9a5c3e022d036c04
 SHA512:
-  metadata.gz: 7157352591d8de2ff4f07228e2e57c9df16f0eb8ff4d75e68499bd07d402a082e938bbcaec2975ffba6dfdd697229f5c929925db13cffb643cd3001a275e7010
-  data.tar.gz: a5c4988858f866533fb27784af8c2a0f109e2432078d4ec819e644b55a45896d47c5fb07a0dd2225b9f2660e0f6cfe9ce880cad3b95dae59feb003b25d1c0bca
+  metadata.gz: 8a2650b889220c5b2e66803f0da23022357fd6c6997e7d09841b983768d099bb487b4e6ad51c8eadb18e255e966afa318d7652a176dd01a44819a43989bb1aef
+  data.tar.gz: 0310c5b3de97b1388d85eab045c1c21ff8ab5ab7354cc092683e1c383532c93c77ca6cdffd3bc86b0bdcd6d8ae943b26c3eb00616e996f732dd9f658ce892485

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## Unreleased
+
+## [0.4.0] - 2026-10-06
+
+- Raised the minimum supported ClickHouse version to 23.3; connections to older servers are rejected during the handshake
+- Updated vendored clickhouse-c headers to `916cbf5`, adding sparse column decoding
+- Improved decimal decoding performance while preserving exact values across all decimal widths
+- Fixed decoding of geometry types, including Point, Ring, Polygon, MultiPolygon, LineString, and MultiLineString
+- Preserved complete UTF-8 server error messages during connection establishment
+
 ## [0.3.1] - 2026-08-19
 
 - Fixed LZ4 compression negotiation for ClickHouse users configured with `readonly = 1`

data/README.md

@@ -52,7 +52,7 @@ Add the gem to your Gemfile:
 gem "ch_connect"
 ```
 
-The gem contains a native extension and currently requires MRI Ruby. LZ4 and ZSTD support are detected when the extension is built.
+The gem contains a native extension and requires MRI Ruby and ClickHouse 23.3 or newer. LZ4 and ZSTD support are detected when the extension is built.
 
 ## Configuration
 

data/ext/ch_connect_native/ch_connect_native.c

@@ -16,6 +16,7 @@
 #include <ruby/encoding.h>
 #include <ruby/intern.h>
 
+#include <inttypes.h>
 #include <limits.h>
 #include <string.h>
 #include <time.h>
@@ -40,8 +41,6 @@ static VALUE vAF_INET6;
 
 static ID id_jd;
 static ID id_new;
-static ID id_pow;
-static ID id_div;
 static ID id_BigDecimal;
 
 static VALUE sym_read_rows;
@@ -136,6 +135,39 @@ load_i64le(const uint8_t *p)
     return v;
 }
 
+/* Little-endian two's complement value of 4, 8, 16 or 32 bytes as int64, when it fits. */
+static int
+load_decimal_i64(const uint8_t *p, size_t nbytes, int64_t *out)
+{
+    if (nbytes == 4) {
+        *out = load_i32le(p);
+        return 1;
+    }
+    int64_t v = load_i64le(p);
+    uint8_t sign = v < 0 ? 0xFF : 0x00;
+    for (size_t b = 8; b < nbytes; b++) {
+        if (p[b] != sign) return 0;
+    }
+    *out = v;
+    return 1;
+}
+
+/* "<unscaled>e-<scale>" for a decimal value. BigDecimal parses it exactly,
+ * without the intermediate BigDecimals of dividing by 10**scale. Only values
+ * outside the int64 range go through a Ruby Integer. */
+static VALUE
+decimal_literal(const uint8_t *p, size_t nbytes, int scale)
+{
+    int64_t v;
+    if (load_decimal_i64(p, nbytes, &v)) {
+        char buf[48];
+        int len = snprintf(buf, sizeof(buf), "%" PRId64 "e-%d", v, scale);
+        return rb_usascii_str_new(buf, len);
+    }
+    VALUE unscaled = rb_integer_unpack(p, nbytes, 1, 0, INTEGER_PACK_LITTLE_ENDIAN | INTEGER_PACK_2COMP);
+    return rb_str_catf(rb_big2str(unscaled, 10), "e-%d", scale);
+}
+
 static inline float
 load_f32le(const uint8_t *p)
 {
@@ -334,23 +366,9 @@ decode_fixed(const chc_column *col, const chc_type *t, long n_rows, native_state
     case CHC_DECIMAL128:
     case CHC_DECIMAL256: {
         int scale = chc_type_decimal_scale(t);
-        VALUE divisor = rb_funcall(INT2FIX(10), id_pow, 1, INT2FIX(scale));
+        size_t nbytes = kind == CHC_DECIMAL32 ? 4 : kind == CHC_DECIMAL64 ? 8 : kind == CHC_DECIMAL128 ? 16 : 32;
         for (long i = 0; i < n_rows; i++) {
-            VALUE unscaled;
-            if (kind == CHC_DECIMAL32) {
-                int32_t v = load_i32le(data + i * 4);
-                unscaled = INT2NUM(v);
-            } else if (kind == CHC_DECIMAL64) {
-                int64_t v = load_i64le(data + i * 8);
-                unscaled = LL2NUM(v);
-            } else {
-                size_t nbytes = (kind == CHC_DECIMAL128) ? 16 : 32;
-                unscaled = rb_integer_unpack(data + i * nbytes, nbytes, 1, 0,
-                                             INTEGER_PACK_LITTLE_ENDIAN |
-                                             INTEGER_PACK_2COMP);
-            }
-            VALUE bd = rb_funcall(rb_mKernel, id_BigDecimal, 1, unscaled);
-            rb_ary_push(ary, rb_funcall(bd, id_div, 1, divisor));
+            rb_ary_push(ary, rb_funcall(rb_mKernel, id_BigDecimal, 1, decimal_literal(data + i * nbytes, nbytes, scale)));
         }
         break;
     }
@@ -377,9 +395,85 @@ decode_string_column(const chc_column *col, long n_rows)
     return ary;
 }
 
+static void
+raise_invalid_geometry(native_state *state, const char *detail)
+{
+    *state = NATIVE_BROKEN;
+    rb_raise(eConnectionError, "Invalid native geometry column: %s", detail);
+}
+
+/* clickhouse-c exposes geometry aliases as their wire-level Array/Tuple
+ * column tree, while the logical geometry type has no synthetic children. */
+static VALUE
+decode_geometry(const chc_column *col, long n_rows, int depth, native_state *state)
+{
+    if (n_rows == 0) return rb_ary_new();
+
+    if (depth == 0) {
+        if (chc_column_layout(col) != CHC_COL_TUPLE ||
+            chc_column_tuple_arity(col) != 2) {
+            raise_invalid_geometry(state, "Point must be a two-coordinate tuple");
+        }
+
+        const chc_column *x_col = chc_column_tuple_child(col, 0);
+        const chc_column *y_col = chc_column_tuple_child(col, 1);
+        size_t x_size = 0;
+        size_t y_size = 0;
+        const uint8_t *x = (const uint8_t *)chc_column_fixed_data(x_col, &x_size);
+        const uint8_t *y = (const uint8_t *)chc_column_fixed_data(y_col, &y_size);
+        if (!x || !y || x_size != 8 || y_size != 8) {
+            raise_invalid_geometry(state, "Point coordinates must be Float64 columns");
+        }
+
+        VALUE points = rb_ary_new_capa(n_rows);
+        for (long i = 0; i < n_rows; i++) {
+            VALUE point = rb_ary_new_capa(2);
+            rb_ary_push(point, DBL2NUM(load_f64le(x + i * 8)));
+            rb_ary_push(point, DBL2NUM(load_f64le(y + i * 8)));
+            rb_ary_push(points, point);
+        }
+        return points;
+    }
+
+    if (chc_column_layout(col) != CHC_COL_ARRAY) {
+        raise_invalid_geometry(state, "expected an Array layer");
+    }
+
+    const uint64_t *offsets = chc_column_array_offsets(col);
+    const chc_column *values_col = chc_column_array_values(col);
+    long total = (long)chc_column_n_rows(values_col);
+    VALUE elements = decode_geometry(values_col, total, depth - 1, state);
+    VALUE ary = rb_ary_new_capa(n_rows);
+    uint64_t start = 0;
+    for (long i = 0; i < n_rows; i++) {
+        uint64_t end = offsets[i];
+        if (end < start || end > (uint64_t)RARRAY_LEN(elements)) {
+            raise_invalid_geometry(state, "Array offset is out of bounds");
+        }
+        rb_ary_push(ary, rb_ary_subseq(elements, (long)start, (long)(end - start)));
+        start = end;
+    }
+    return ary;
+}
+
 static VALUE
 decode_column(const chc_column *col, const chc_type *t, long n_rows, native_state *state)
 {
+    switch (chc_type_kind(t)) {
+    case CHC_POINT:
+        return decode_geometry(col, n_rows, 0, state);
+    case CHC_RING:
+    case CHC_LINE_STRING:
+        return decode_geometry(col, n_rows, 1, state);
+    case CHC_POLYGON:
+    case CHC_MULTI_LINE_STRING:
+        return decode_geometry(col, n_rows, 2, state);
+    case CHC_MULTI_POLYGON:
+        return decode_geometry(col, n_rows, 3, state);
+    default:
+        break;
+    }
+
     if (n_rows == 0) {
         chc_kind kind = chc_type_kind(t);
         if (kind == CHC_STRING) return rb_ary_new();
@@ -638,7 +732,8 @@ native_client_handshake_step(VALUE self)
 
     nc->state = NATIVE_ACTIVE;
     chc_err err = {0};
-    int rc = chc_async_handshake(nc->ac, &err);
+    chc_exception *exception = NULL;
+    int rc = chc_async_handshake(nc->ac, &exception, &err);
     if (rc == CHC_OK) {
         nc->state = NATIVE_READY;
         return sym_done;
@@ -646,6 +741,16 @@ native_client_handshake_step(VALUE self)
     if (rc == CHC_WOULD_BLOCK) return sym_want_read;
 
     nc->state = NATIVE_BROKEN;
+    if (exception) {
+        /* Anchor the server exception before allocating Ruby objects so close
+         * or GC can still release it if message construction raises. */
+        nc->pending_pkt.kind = CHC_PKT_EXCEPTION;
+        nc->pending_pkt.exception = exception;
+        VALUE msg = rb_utf8_str_new(exception->display_text,
+                                    (long)exception->display_text_len);
+        clear_pending(nc);
+        rb_exc_raise(rb_exc_new_str(eConnectionError, msg));
+    }
     rb_raise(eConnectionError, "%s", err.msg);
 }
 
@@ -880,7 +985,7 @@ native_client_recv_step(VALUE self)
             nc->state = NATIVE_READY;
             return sym_done;
         }
-        /* PONG / TOTALS / EXTREMES / LOG / PROFILE_EVENTS — skip. */
+        /* PONG / TOTALS / EXTREMES / LOG / PROFILE_EVENTS / TIMEZONE_UPDATE — skip. */
     }
 }
 
@@ -969,8 +1074,6 @@ Init_ch_connect_native(void)
 
     id_jd = rb_intern("jd");
     id_new = rb_intern("new");
-    id_pow = rb_intern("**");
-    id_div = rb_intern("/");
     id_BigDecimal = rb_intern("BigDecimal");
 
     sym_read_rows = ID2SYM(rb_intern("read_rows"));

data/lib/ch_connect/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChConnect
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/vendor/clickhouse-c/VENDOR.md

@@ -1,9 +1,12 @@
 # Vendored: clickhouse-c
 
 - Source: https://github.com/ClickHouse/clickhouse-c
-- Commit: dd47d2882646aeba5a1946c62a229d57abea6c28 (cloned 2026-07-14)
+- Commit: 916cbf5ff41a218978bc073bca10f5b609a00081 (updated 2026-10-05)
 - License: Apache-2.0 (see LICENSE)
 
+The native client requires ClickHouse 23.3 or newer and advertises protocol
+revision 54465, including sparse column serialization support.
+
 Only the headers the extension includes are vendored: `clickhouse.h`,
 `clickhouse-compression.h`, `clickhouse-client.h`, `clickhouse-async.h`
 (plus LICENSE). Upstream's tests, docs, tools, and the `clickhouse-posix-io.h`
@@ -12,7 +15,7 @@ ioless (Ruby owns the socket and TLS).
 
 ## Local patches
 
-- `clickhouse.h` — `chc__col_read`, `case CHC_TUPLE`: ClickHouse serializes the
+- `clickhouse.h` — `chc__col_read_tuple`: ClickHouse serializes the
   empty `Tuple()` as one UInt8 (zero) per row (ClickHouse/ClickHouse#55061);
   upstream reads nothing and desyncs the stream. Marked with
   `LOCAL PATCH (ch_connect)`. Should be upstreamed.

data/vendor/clickhouse-c/clickhouse-async.h

@@ -47,8 +47,15 @@ int  chc_async_submit(chc_async_client *c, const void *buf, size_t len,
 void chc_async_pending_out(chc_async_client *c, const uint8_t **buf, size_t *len);
 void chc_async_consume_out(chc_async_client *c, size_t n);
 
-/* drive -- never block; CHC_WOULD_BLOCK = submit more / drain out */
-int  chc_async_handshake(chc_async_client *c, chc_err *err);
+/* Drive calls never block. CHC_WOULD_BLOCK requests more input or output
+ * draining
+ *
+ * Handshake rejection returns CHC_ERR_SERVER and leaves err->msg empty. When
+ * exc is not NULL, caller owns *exc and frees it with chc_exception_free.
+ * Other results leave *exc unchanged. Query exceptions still arrive as
+ * packets, but handshake rejection ends connection setup */
+int  chc_async_handshake(chc_async_client *c, chc_exception **exc,
+                         chc_err *err);
 int  chc_async_send_query(chc_async_client *c, const char *sql, size_t sql_len,
                           const char *query_id, size_t query_id_len, chc_err *err);
 int  chc_async_send_data(chc_async_client *c, const chc_block_builder *bb,
@@ -122,16 +129,7 @@ chc_async_client_init(chc_async_client **out, const chc_client_opts *opts,
     if (!c) return CHC_ERR_OOM;
 
     c->cli.al = al;
-    c->cli.client_version_major = opts->client_version_major;
-    c->cli.client_version_minor = opts->client_version_minor;
-    c->cli.client_version_patch = opts->client_version_patch;
-    c->cli.client_revision = opts->client_revision ? opts->client_revision
-                                                   : CHC_CLIENT_DEFAULT_REVISION;
-    c->cli.compression = opts->codec ? opts->compression : CHC_COMP_NONE;
-    c->cli.codec       = opts->codec;
-    /* Seed server.revision so block/packet framing is well-defined before the
-     * handshake completes; recv after handshake uses min(client, server). */
-    c->cli.server.revision = c->cli.client_revision;
+    chc__client_setup(&c->cli, opts);
 
     chc__mem_sink_init(&c->out, &c->out_io, al);
     c->cli.io = &c->out_io;
@@ -196,7 +194,7 @@ chc_async_consume_out(chc_async_client *c, size_t n)
 }
 
 int
-chc_async_handshake(chc_async_client *c, chc_err *err)
+chc_async_handshake(chc_async_client *c, chc_exception **exc, chc_err *err)
 {
     int rc;
     chc_client *cli = &c->cli;
@@ -209,7 +207,6 @@ chc_async_handshake(chc_async_client *c, chc_err *err)
                 .client_version_major = cli->client_version_major,
                 .client_version_minor = cli->client_version_minor,
                 .client_version_patch = cli->client_version_patch,
-                .client_revision = cli->client_revision,
                 .database = c->database,
                 .user = c->user,
                 .password = c->password,
@@ -222,27 +219,23 @@ chc_async_handshake(chc_async_client *c, chc_err *err)
 
         case CHC__HS_RECV_HELLO:
             chc__in_checkpoint(&cli->in);
-            rc = chc__client_recv_hello(cli, err);
+            rc = chc__client_recv_hello(cli, exc, err);
             if (rc == CHC_WOULD_BLOCK) { chc__in_rewind(&cli->in); return rc; }
             if (rc != CHC_OK) return rc;  /* server exception / protocol */
             chc_in_reset(&cli->in);
-            if (cli->server.revision > cli->client_revision)
-                cli->server.revision = cli->client_revision;
             c->hs_phase = CHC__HS_POST_HELLO;
             continue;
 
         case CHC__HS_POST_HELLO:
-            if (cli->server.revision >= CHC__REV_ADDENDUM) {
-                rc = chc__write_string(cli->io, "", 0, err);  /* quota_key */
-                if (rc != CHC_OK) return rc;
-            }
+            rc = chc__write_string(cli->io, "", 0, err);  /* quota_key */
+            if (rc != CHC_OK) return rc;
             rc = chc_client_send_ping(cli, err);
             if (rc != CHC_OK) return rc;
             c->hs_phase = CHC__HS_RECV_PONG;
             continue;
 
         case CHC__HS_RECV_PONG:
-            rc = chc__recv_pong(cli, err);  /* owns checkpoint/rewind */
+            rc = chc__recv_pong(cli, exc, err);  /* owns checkpoint/rewind */
             if (rc != CHC_OK) return rc;
             chc_in_reset(&cli->in);
             c->hs_phase = CHC__HS_DONE;