ch_connect 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5856233f30844a931e61ac5aca11af740c7a876a95ae19e497a3157bca9097b2
-  data.tar.gz: 6588fea1311f26c52e6b13885316741e47a98a284751456d9a5c3e022d036c04
+  metadata.gz: 88e326f103cde2be1cd0ed39c0615ff2861020fb2f5d334fcb08ff012196cd3b
+  data.tar.gz: 0c6b12e11fdfec5626fefa1cbbcf81da8ed384f4983d1caf0466758adcb617db
 SHA512:
-  metadata.gz: 8a2650b889220c5b2e66803f0da23022357fd6c6997e7d09841b983768d099bb487b4e6ad51c8eadb18e255e966afa318d7652a176dd01a44819a43989bb1aef
-  data.tar.gz: 0310c5b3de97b1388d85eab045c1c21ff8ab5ab7354cc092683e1c383532c93c77ca6cdffd3bc86b0bdcd6d8ae943b26c3eb00616e996f732dd9f658ce892485
+  metadata.gz: fda2b6e230f5a66d66a0cd4189dc324f76713756c458bd67e06f1dc0873ec589c53980e497eaa9d37a5dd985ba0f8ef60264bafe8750b2d331ce33e93d12b574
+  data.tar.gz: eaa8b14dd4da0af03c2c3ab66825ab27c4770d0f7a7dd596deb4daa06a6b0fc3191a9454b4bd0f3411b3cbe1f85dd5eaf1ff2546a2e20c9160d7f94ae1572b43

data/CHANGELOG.md

@@ -1,4 +1,8 @@
-## Unreleased
+## [0.4.1] - 2026-10-12
+
+- Improved large result decoding speed (about 1.9x faster for the 100K-row benchmark query) and reduced its allocations from 97 MB to 55 MB
+- Fixed quadratic re-parsing of large uncompressed result blocks (about 5x faster for multi-megabyte String columns)
+- Faster decoding of repeated `Date`/`Date32` values, `Nullable(String)`, `UUID`, and `Map` with String keys, and faster `Response#each`
 
 ## [0.4.0] - 2026-10-06
 

data/README.md

@@ -22,27 +22,27 @@ Speed (iterations/second, higher is better):
 
 | Scenario | ch_connect | click_house | clickhouse | click_house-client |
 |----------|------------|-------------|------------|--------------------|
-| Small queries (10 rows) | **962 i/s** | 363 i/s (2.7x slower) | 317 i/s (3.0x slower) | 372 i/s (2.6x slower) |
-| Large queries (100K rows) | **8.2 i/s** | 1.1 i/s (7.2x slower) | 0.6 i/s (14.1x slower) | 1.5 i/s (5.3x slower) |
+| Small queries (10 rows) | **1961 i/s** | 718 i/s (2.7x slower) | 524 i/s (3.7x slower) | 796 i/s (2.5x slower) |
+| Large queries (100K rows) | **14.2 i/s** | 1.4 i/s (10.5x slower) | 0.5 i/s (26.4x slower) | 1.8 i/s (8.0x slower) |
 
 Multi-threaded throughput (10K-row query, total queries/second):
 
 | Threads | ch_connect | click_house | clickhouse | click_house-client |
 |---------|------------|-------------|------------|--------------------|
-| 1 | **292 q/s** | 69 q/s | 117 q/s | 48 q/s |
-| 4 | **598 q/s** | 162 q/s | 319 q/s | 82 q/s |
-| 8 | **606 q/s** | 158 q/s | 321 q/s | 80 q/s |
+| 1 | **523 q/s** | 74 q/s | 140 q/s | 56 q/s |
+| 4 | **755 q/s** | 148 q/s | 306 q/s | 78 q/s |
+| 8 | **637 q/s** | 150 q/s | 326 q/s | 82 q/s |
 
 Memory allocated for a large query (lower is better):
 
 | Gem | Allocated |
 |-----|-----------|
-| ch_connect | **97 MB** |
+| ch_connect | **55 MB** |
 | click_house | 198 MB |
 | click_house-client | 210 MB |
 | clickhouse | 436 MB |
 
-A small query allocates approximately **3.2 KB / 32 Ruby objects**. See the [`benchmark/`](benchmark/) directory for the comparison suite and methodology.
+A small query allocates approximately **3.6 KB / 40 Ruby objects**. See the [`benchmark/`](benchmark/) directory for the comparison suite and methodology.
 
 ## Installation
 

data/ext/ch_connect_native/ch_connect_native.c

@@ -58,6 +58,9 @@ static VALUE sym_zstd;
 /* Julian day number of 1970-01-01 (Date.jd(2440588) == Date.new(1970, 1, 1)) */
 #define UNIX_EPOCH_JD 2440588
 
+/* Per-column Date cache slots; must be a power of two */
+#define DATE_CACHE_SIZE 256
+
 /* One immutable codec shared by all connections and threads: the built-in
  * adapters are stateless wrappers over the one-shot lz4/zstd functions. Both
  * slot pairs are filled (when available) so decode survives a server-side
@@ -291,17 +294,23 @@ decode_fixed(const chc_column *col, const chc_type *t, long n_rows, native_state
         }
         break;
     }
-    case CHC_DATE: {
-        for (long i = 0; i < n_rows; i++) {
-            uint16_t days = load_u16le(data + i * 2);
-            rb_ary_push(ary, rb_funcall(cDate, id_jd, 1, LONG2NUM(UNIX_EPOCH_JD + (long)days)));
-        }
-        break;
-    }
+    case CHC_DATE:
     case CHC_DATE32: {
+        /* Date construction is a Ruby method call and columns usually repeat
+         * a handful of days, so reuse (immutable) Dates through a small
+         * direct-mapped cache. Every cached Date is already anchored by ary. */
+        long cache_days[DATE_CACHE_SIZE];
+        VALUE cache_vals[DATE_CACHE_SIZE];
+        for (int k = 0; k < DATE_CACHE_SIZE; k++) cache_vals[k] = Qundef;
         for (long i = 0; i < n_rows; i++) {
-            int32_t days = load_i32le(data + i * 4);
-            rb_ary_push(ary, rb_funcall(cDate, id_jd, 1, LONG2NUM(UNIX_EPOCH_JD + (long)days)));
+            long days = kind == CHC_DATE ? (long)load_u16le(data + i * 2)
+                                         : (long)load_i32le(data + i * 4);
+            unsigned long slot = (unsigned long)days & (DATE_CACHE_SIZE - 1);
+            if (cache_vals[slot] == Qundef || cache_days[slot] != days) {
+                cache_days[slot] = days;
+                cache_vals[slot] = rb_funcall(cDate, id_jd, 1, LONG2NUM(UNIX_EPOCH_JD + days));
+            }
+            rb_ary_push(ary, cache_vals[slot]);
         }
         break;
     }
@@ -331,18 +340,23 @@ decode_fixed(const chc_column *col, const chc_type *t, long n_rows, native_state
         break;
     }
     case CHC_UUID: {
-        char buf[37];
+        /* Each 8-byte half is little-endian on the wire; emit both halves
+         * most significant nibble first, with dashes after hex digits
+         * 8, 12, 16 and 20. */
+        static const char hex[] = "0123456789abcdef";
         for (long i = 0; i < n_rows; i++) {
             const uint8_t *p = data + i * 16;
-            uint64_t hi = load_u64le(p);
-            uint64_t lo = load_u64le(p + 8);
-            snprintf(buf, sizeof(buf), "%08llx-%04llx-%04llx-%04llx-%012llx",
-                     (unsigned long long)(hi >> 32),
-                     (unsigned long long)((hi >> 16) & 0xFFFF),
-                     (unsigned long long)(hi & 0xFFFF),
-                     (unsigned long long)(lo >> 48),
-                     (unsigned long long)(lo & 0xFFFFFFFFFFFFULL));
-            rb_ary_push(ary, rb_utf8_str_new(buf, 36));
+            uint64_t halves[2] = { load_u64le(p), load_u64le(p + 8) };
+            VALUE str = rb_utf8_str_new(NULL, 36);
+            char *out = RSTRING_PTR(str);
+            int o = 0;
+            for (int h = 0; h < 2; h++) {
+                for (int shift = 60; shift >= 0; shift -= 4) {
+                    if (o == 8 || o == 13 || o == 18 || o == 23) out[o++] = '-';
+                    out[o++] = hex[(halves[h] >> shift) & 0xF];
+                }
+            }
+            rb_ary_push(ary, str);
         }
         break;
     }
@@ -379,17 +393,37 @@ decode_fixed(const chc_column *col, const chc_type *t, long n_rows, native_state
     return ary;
 }
 
+/* null_map (optional) marks rows that a Nullable wrapper turns into nil, so
+ * their placeholder strings are never allocated. */
+static VALUE
+decode_string_column(const chc_column *col, long n_rows, const uint8_t *null_map)
+{
+    const uint8_t *data = chc_column_string_data(col);
+    const uint64_t *offsets = chc_column_string_offsets(col);
+    VALUE ary = rb_ary_new_capa(n_rows);
+
+    uint64_t start = 0;
+    for (long i = 0; i < n_rows; i++) {
+        uint64_t end = offsets[i];
+        if (null_map && null_map[i] == 1) rb_ary_push(ary, Qnil);
+        else rb_ary_push(ary, rb_utf8_str_new((const char *)data + start, (long)(end - start)));
+        start = end;
+    }
+    return ary;
+}
+
 static VALUE
-decode_string_column(const chc_column *col, long n_rows)
+decode_interned_string_column(const chc_column *col, long n_rows)
 {
     const uint8_t *data = chc_column_string_data(col);
     const uint64_t *offsets = chc_column_string_offsets(col);
+    rb_encoding *utf8 = rb_utf8_encoding();
     VALUE ary = rb_ary_new_capa(n_rows);
 
     uint64_t start = 0;
     for (long i = 0; i < n_rows; i++) {
         uint64_t end = offsets[i];
-        rb_ary_push(ary, rb_utf8_str_new((const char *)data + start, (long)(end - start)));
+        rb_ary_push(ary, rb_enc_interned_str((const char *)data + start, (long)(end - start), utf8));
         start = end;
     }
     return ary;
@@ -492,12 +526,14 @@ decode_column(const chc_column *col, const chc_type *t, long n_rows, native_stat
         return decode_fixed(col, t, n_rows, state);
 
     case CHC_COL_STRING:
-        return decode_string_column(col, n_rows);
+        return decode_string_column(col, n_rows, NULL);
 
     case CHC_COL_NULLABLE: {
         const uint8_t *null_map = chc_column_null_map(col);
         const chc_column *inner = chc_column_nullable_inner(col);
         const chc_type *inner_t = chc_type_child(t, 0);
+        if (chc_column_layout(inner) == CHC_COL_STRING)
+            return decode_string_column(inner, n_rows, null_map);
         VALUE vals = decode_column(inner, inner_t, n_rows, state);
         for (long i = 0; i < n_rows; i++) {
             if (null_map[i] == 1) rb_ary_store(vals, i, Qnil);
@@ -516,7 +552,12 @@ decode_column(const chc_column *col, const chc_type *t, long n_rows, native_stat
             long total = (long)chc_column_n_rows(values_col);
             const chc_column *keys_col = chc_column_tuple_child(values_col, 0);
             const chc_column *vals_col = chc_column_tuple_child(values_col, 1);
-            VALUE keys = decode_column(keys_col, kt, total, state);
+            /* Hash#[]= dups and freezes unfrozen String keys; decoding them
+             * as interned (frozen, deduplicated) strings skips that copy, and
+             * map keys typically repeat across rows. */
+            VALUE keys = chc_column_layout(keys_col) == CHC_COL_STRING
+                ? decode_interned_string_column(keys_col, total)
+                : decode_column(keys_col, kt, total, state);
             VALUE vals = decode_column(vals_col, vt, total, state);
 
             VALUE ary = rb_ary_new_capa(n_rows);
@@ -529,7 +570,7 @@ decode_column(const chc_column *col, const chc_type *t, long n_rows, native_stat
                              "Map offset out of bounds at row %ld: %llu", i,
                              (unsigned long long)end);
                 }
-                VALUE hash = rb_hash_new();
+                VALUE hash = rb_hash_new_capa((long)(end - start));
                 for (uint64_t j = start; j < end; j++) {
                     rb_hash_aset(hash, RARRAY_AREF(keys, (long)j), RARRAY_AREF(vals, (long)j));
                 }
@@ -913,13 +954,16 @@ native_client_decode_block(VALUE self, native_client_t *nc, chc_block *block)
                                 (long)n_rows, &nc->state);
         rb_ary_push(col_vals, cols[i]);
     }
+    /* row_vals only holds references that cols[] already anchors */
+    VALUE row_buf;
+    VALUE *row_vals = ALLOCV_N(VALUE, row_buf, n_cols);
     for (size_t r = 0; r < n_rows; r++) {
-        VALUE row = rb_ary_new_capa((long)n_cols);
         for (size_t c = 0; c < n_cols; c++) {
-            rb_ary_push(row, RARRAY_AREF(cols[c], (long)r));
+            row_vals[c] = RARRAY_AREF(cols[c], (long)r);
         }
-        rb_ary_push(rows, row);
+        rb_ary_push(rows, rb_ary_new_from_values((long)n_cols, row_vals));
     }
+    ALLOCV_END(row_buf);
     ALLOCV_END(cols_buf);
     RB_GC_GUARD(col_vals);
 }

data/lib/ch_connect/connection.rb

@@ -27,6 +27,7 @@ module ChConnect
 
     READ_CHUNK = 64 * 1024
     WRITE_CHUNK = 64 * 1024
+    DRAIN_LIMIT = 16 * 1024 * 1024
     PARAM_ESCAPES = {
       "\0" => "\\0", "\a" => "\\a", "\b" => "\\b", "\e" => "\\e",
       "\f" => "\\f", "\n" => "\\n", "\r" => "\\r", "\t" => "\\t",
@@ -60,7 +61,7 @@ module ChConnect
         loop do
           case @client.recv_step
           when :done then break
-          when :want_read then @client.feed(read_chunk)
+          when :want_read then feed_available
           end
         end
         @client.take_result
@@ -191,6 +192,23 @@ module ChConnect
         end
       end
 
+      # Blocks for one chunk, then drains whatever else the socket already has
+      # buffered (up to DRAIN_LIMIT). An uncompressed Data block that ends
+      # mid-column is re-parsed from that column's start on the next
+      # recv_step, so retrying once per burst instead of once per chunk keeps
+      # large blocks from going quadratic.
+      def feed_available
+        @client.feed(read_chunk)
+        drained = 0
+        while drained < DRAIN_LIMIT
+          chunk = @socket.read_nonblock(READ_CHUNK, @read_buf, exception: false)
+          break unless chunk.is_a?(String)
+
+          @client.feed(chunk)
+          drained += chunk.bytesize
+        end
+      end
+
       # Reads into a reusable buffer: feed copies the bytes synchronously,
       # so the buffer never needs to survive past the next read.
       def read_chunk(timeout = @config.read_timeout)

data/lib/ch_connect/response.rb

@@ -17,7 +17,18 @@ module ChConnect
     def each
       return to_enum(:each) unless block_given?
 
-      rows.each { |row| yield columns.zip(row).to_h }
+      # An index loop avoids zip's per-row pair arrays (~3x faster under YJIT).
+      names = columns
+      count = names.size
+      rows.each do |row|
+        hash = {}
+        index = 0
+        while index < count
+          hash[names[index]] = row[index]
+          index += 1
+        end
+        yield hash
+      end
     end
   end
 end

data/lib/ch_connect/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChConnect
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ch_connect
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Karol Bąk