clickhouse-native 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/clickhouse_native/client.cpp +213 -9
- data/ext/clickhouse_native/extconf.rb +53 -14
- data/lib/clickhouse_native/client.rb +37 -21
- data/lib/clickhouse_native/errors.rb +2 -0
- data/lib/clickhouse_native/logging.rb +64 -0
- data/lib/clickhouse_native/pool.rb +6 -4
- data/lib/clickhouse_native/version.rb +3 -1
- data/lib/clickhouse_native.rb +5 -0
- metadata +11 -63
- data/lib/clickhouse_native/clickhouse_native.bundle +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c874b72697637755a30f9a8a22edad7c35050878e7e7d1e9260f2cd7dc06c137
|
|
4
|
+
data.tar.gz: 4df29895e45102db083fc6f21718f4845fcd944fba72bea95f1c3b2dc6549c0a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 55a54773a8f42cfea9029973966fc245442c8ccf732808750d61374b8f661aaba0bc9869567dd37599a1128367ff83161a73b21c33987f9d88138cf467c08571
|
|
7
|
+
data.tar.gz: ffea3e3c143e8bf311573f7c7864b8c2794efebcdf9dcdc090e4ec3a54436d4f6d5df63bd76119e42301138e887ce9ebbe536009bc9518a9d8067a67042f4f34
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
#include <clickhouse/client.h>
|
|
5
5
|
#include <clickhouse/columns/array.h>
|
|
6
6
|
#include <clickhouse/columns/date.h>
|
|
7
|
+
#include <clickhouse/columns/decimal.h>
|
|
7
8
|
#include <clickhouse/columns/enum.h>
|
|
8
9
|
#include <clickhouse/columns/factory.h>
|
|
9
10
|
#include <clickhouse/columns/lowcardinality.h>
|
|
@@ -99,6 +100,55 @@ static VALUE value_at(const ColumnRef& col, size_t idx) {
|
|
|
99
100
|
case Type::Float32: return DBL2NUM(col->As<ColumnFloat32>()->At(idx));
|
|
100
101
|
case Type::Float64: return DBL2NUM(col->As<ColumnFloat64>()->At(idx));
|
|
101
102
|
|
|
103
|
+
case Type::Decimal:
|
|
104
|
+
case Type::Decimal32:
|
|
105
|
+
case Type::Decimal64:
|
|
106
|
+
case Type::Decimal128: {
|
|
107
|
+
// ClickHouse stores decimals as scaled integers. clickhouse-cpp
|
|
108
|
+
// returns them as Int128. To preserve precision, we format the
|
|
109
|
+
// unscaled int into a string at the column's scale and construct
|
|
110
|
+
// a BigDecimal from it.
|
|
111
|
+
auto dec = col->As<ColumnDecimal>();
|
|
112
|
+
auto unscaled = dec->At(idx); // Int128 (absl::int128)
|
|
113
|
+
size_t scale = type->As<DecimalType>()->GetScale();
|
|
114
|
+
|
|
115
|
+
bool neg = unscaled < 0;
|
|
116
|
+
absl::uint128 mag = neg
|
|
117
|
+
? absl::uint128(-static_cast<absl::int128>(unscaled))
|
|
118
|
+
: absl::uint128(static_cast<absl::int128>(unscaled));
|
|
119
|
+
|
|
120
|
+
// Render magnitude in base 10, then place the decimal point.
|
|
121
|
+
char buf[48];
|
|
122
|
+
int len = 0;
|
|
123
|
+
if (mag == absl::uint128(0)) {
|
|
124
|
+
buf[len++] = '0';
|
|
125
|
+
} else {
|
|
126
|
+
while (mag > absl::uint128(0)) {
|
|
127
|
+
absl::uint128 q = mag / absl::uint128(10);
|
|
128
|
+
absl::uint128 r = mag - q * absl::uint128(10);
|
|
129
|
+
buf[len++] = '0' + static_cast<int>(absl::Uint128Low64(r));
|
|
130
|
+
mag = q;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Pad with leading zeros so we have at least scale+1 digits.
|
|
134
|
+
while (static_cast<size_t>(len) <= scale) buf[len++] = '0';
|
|
135
|
+
|
|
136
|
+
std::string out;
|
|
137
|
+
out.reserve(len + 3);
|
|
138
|
+
if (neg) out.push_back('-');
|
|
139
|
+
for (int i = len - 1; i >= 0; i--) {
|
|
140
|
+
if (scale > 0 && static_cast<size_t>(i) == scale - 1) {
|
|
141
|
+
// Insert decimal point before the last `scale` digits.
|
|
142
|
+
out.push_back('.');
|
|
143
|
+
}
|
|
144
|
+
out.push_back(buf[i]);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
VALUE rb_cBigDecimal = rb_const_get(rb_cObject, rb_intern("BigDecimal"));
|
|
148
|
+
return rb_funcall(rb_cBigDecimal, rb_intern("BigDecimal"), 1,
|
|
149
|
+
rb_utf8_str_new(out.data(), out.size()));
|
|
150
|
+
}
|
|
151
|
+
|
|
102
152
|
case Type::String: {
|
|
103
153
|
auto sv = col->As<ColumnString>()->At(idx);
|
|
104
154
|
return rb_utf8_str_new(sv.data(), sv.size());
|
|
@@ -149,7 +199,11 @@ static VALUE value_at(const ColumnRef& col, size_t idx) {
|
|
|
149
199
|
|
|
150
200
|
case Type::LowCardinality: {
|
|
151
201
|
auto lc = col->As<ColumnLowCardinality>();
|
|
152
|
-
|
|
202
|
+
ItemView item = lc->GetItem(idx);
|
|
203
|
+
// For LowCardinality(Nullable(T)), clickhouse-cpp's GetItem returns
|
|
204
|
+
// a default-constructed ItemView (type == Void) for null rows.
|
|
205
|
+
if (item.type == Type::Void) return Qnil;
|
|
206
|
+
auto sv = item.AsBinaryData();
|
|
153
207
|
return rb_utf8_str_new(sv.data(), sv.size());
|
|
154
208
|
}
|
|
155
209
|
|
|
@@ -222,6 +276,26 @@ static void append_default(const ColumnRef& col) {
|
|
|
222
276
|
case Type::Date32: col->As<ColumnDate32>()->Append(0); return;
|
|
223
277
|
case Type::DateTime: col->As<ColumnDateTime>()->Append(0); return;
|
|
224
278
|
case Type::DateTime64: col->As<ColumnDateTime64>()->Append(0); return;
|
|
279
|
+
case Type::Decimal:
|
|
280
|
+
case Type::Decimal32:
|
|
281
|
+
case Type::Decimal64:
|
|
282
|
+
case Type::Decimal128: col->As<ColumnDecimal>()->Append(std::string("0")); return;
|
|
283
|
+
case Type::LowCardinality: {
|
|
284
|
+
auto nested_type = type->As<LowCardinalityType>()->GetNestedType();
|
|
285
|
+
bool nullable = nested_type->GetCode() == Type::Nullable;
|
|
286
|
+
if (!nullable) {
|
|
287
|
+
auto lct = col->AsStrict<ColumnLowCardinalityT<ColumnString>>();
|
|
288
|
+
lct->Append(std::string_view());
|
|
289
|
+
} else {
|
|
290
|
+
auto tmp_inner = std::make_shared<ColumnString>();
|
|
291
|
+
auto tmp_nulls = std::make_shared<ColumnUInt8>();
|
|
292
|
+
tmp_inner->Append(std::string_view());
|
|
293
|
+
tmp_nulls->Append(1);
|
|
294
|
+
auto tmp = std::make_shared<ColumnNullable>(tmp_inner, tmp_nulls);
|
|
295
|
+
col->As<ColumnLowCardinality>()->Append(tmp);
|
|
296
|
+
}
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
225
299
|
case Type::Array: {
|
|
226
300
|
auto inner_type = type->As<ArrayType>()->GetItemType();
|
|
227
301
|
auto inner_col = CreateColumnByType(inner_type->GetName());
|
|
@@ -234,6 +308,51 @@ static void append_default(const ColumnRef& col) {
|
|
|
234
308
|
}
|
|
235
309
|
}
|
|
236
310
|
|
|
311
|
+
// Detect a trailing timezone indicator: "Z", "+HHMM", "+HH:MM", or the
|
|
312
|
+
// same with '-' (anywhere after the date portion, so the '-' inside
|
|
313
|
+
// "YYYY-MM-DD" doesn't match). Used to decide whether a naked timestamp
|
|
314
|
+
// string should be forced to UTC.
|
|
315
|
+
static bool has_trailing_tz(const char* s, size_t len) {
|
|
316
|
+
if (len == 0) return false;
|
|
317
|
+
if (s[len - 1] == 'Z' || s[len - 1] == 'z') return true;
|
|
318
|
+
// walk backwards for a '+' or '-' within the last 6 chars (HH:MM / HHMM)
|
|
319
|
+
size_t limit = len > 6 ? len - 6 : 0;
|
|
320
|
+
for (size_t i = len; i-- > limit; ) {
|
|
321
|
+
if (i < 11) break; // position must be past "YYYY-MM-DD "
|
|
322
|
+
if (s[i] == '+' || s[i] == '-') return true;
|
|
323
|
+
}
|
|
324
|
+
return false;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// Accepts a Time, a String (parsed via Time.parse), or a numeric (epoch
|
|
328
|
+
// seconds). Mirrors how the HTTP gem's JSONEachRow coerced date cells.
|
|
329
|
+
//
|
|
330
|
+
// Naked timestamp strings (no trailing TZ indicator) are parsed as UTC —
|
|
331
|
+
// CH's DateTime64(N, 'UTC') convention. Strings with explicit zones are
|
|
332
|
+
// respected.
|
|
333
|
+
static VALUE coerce_to_time(VALUE value) {
|
|
334
|
+
if (rb_obj_is_kind_of(value, rb_cTime)) return value;
|
|
335
|
+
if (RB_TYPE_P(value, T_STRING)) {
|
|
336
|
+
const char* p = RSTRING_PTR(value);
|
|
337
|
+
long n = RSTRING_LEN(value);
|
|
338
|
+
VALUE to_parse = value;
|
|
339
|
+
if (!has_trailing_tz(p, n)) {
|
|
340
|
+
std::string with_utc(p, n);
|
|
341
|
+
with_utc.append(" UTC");
|
|
342
|
+
to_parse = rb_utf8_str_new(with_utc.data(), with_utc.size());
|
|
343
|
+
}
|
|
344
|
+
return rb_funcall(rb_cTime, rb_intern("parse"), 1, to_parse);
|
|
345
|
+
}
|
|
346
|
+
if (RB_INTEGER_TYPE_P(value) || RB_FLOAT_TYPE_P(value)) {
|
|
347
|
+
return rb_funcall(rb_cTime, rb_intern("at"), 1, value);
|
|
348
|
+
}
|
|
349
|
+
// Date / DateTime / other responds to to_time
|
|
350
|
+
if (rb_respond_to(value, rb_intern("to_time"))) {
|
|
351
|
+
return rb_funcall(value, rb_intern("to_time"), 0);
|
|
352
|
+
}
|
|
353
|
+
throw chn::EncoderFailure("cannot coerce value to Time");
|
|
354
|
+
}
|
|
355
|
+
|
|
237
356
|
static int64_t time_to_datetime64_ticks(VALUE value, size_t prec) {
|
|
238
357
|
VALUE to_i = rb_funcall(value, rb_intern("to_i"), 0);
|
|
239
358
|
VALUE nsec = rb_funcall(value, rb_intern("nsec"), 0);
|
|
@@ -246,6 +365,29 @@ static int64_t time_to_datetime64_ticks(VALUE value, size_t prec) {
|
|
|
246
365
|
|
|
247
366
|
static void append_value(const ColumnRef& col, VALUE value) {
|
|
248
367
|
auto type = col->Type();
|
|
368
|
+
// Graceful nil handling for non-Nullable columns — mirrors how the HTTP
|
|
369
|
+
// gem's JSONEachRow insert silently coerced nil to the column default.
|
|
370
|
+
// Structural types (Nullable/Array/Map/Tuple/LowCardinality) have their
|
|
371
|
+
// own nil semantics handled below.
|
|
372
|
+
if (NIL_P(value)) {
|
|
373
|
+
auto code = type->GetCode();
|
|
374
|
+
// Structural types handle nil via their own semantics below.
|
|
375
|
+
if (code != Type::Nullable && code != Type::Array &&
|
|
376
|
+
code != Type::Map && code != Type::Tuple) {
|
|
377
|
+
append_default(col);
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
// Bool columns come over the wire as UInt8. Ruby true/false are the
|
|
382
|
+
// natural inputs for Bool, so coerce here before the numeric cases run.
|
|
383
|
+
if (value == Qtrue) value = INT2FIX(1);
|
|
384
|
+
else if (value == Qfalse) value = INT2FIX(0);
|
|
385
|
+
|
|
386
|
+
// Symbol -> String: common for LowCardinality/String columns where
|
|
387
|
+
// callers naturally use :pt, :eur, etc. Matches how JSON serialization
|
|
388
|
+
// in the HTTP path rendered them.
|
|
389
|
+
if (SYMBOL_P(value)) value = rb_funcall(value, rb_intern("to_s"), 0);
|
|
390
|
+
|
|
249
391
|
switch (type->GetCode()) {
|
|
250
392
|
case Type::Int8: col->As<ColumnInt8>()->Append(NUM2INT(value)); return;
|
|
251
393
|
case Type::Int16: col->As<ColumnInt16>()->Append(NUM2INT(value)); return;
|
|
@@ -272,23 +414,24 @@ static void append_value(const ColumnRef& col, VALUE value) {
|
|
|
272
414
|
}
|
|
273
415
|
|
|
274
416
|
case Type::Date: {
|
|
275
|
-
VALUE
|
|
276
|
-
col->As<ColumnDate>()->Append(static_cast<std::time_t>(NUM2LL(to_i)));
|
|
417
|
+
VALUE t = coerce_to_time(value);
|
|
418
|
+
col->As<ColumnDate>()->Append(static_cast<std::time_t>(NUM2LL(rb_funcall(t, rb_intern("to_i"), 0))));
|
|
277
419
|
return;
|
|
278
420
|
}
|
|
279
421
|
case Type::Date32: {
|
|
280
|
-
VALUE
|
|
281
|
-
col->As<ColumnDate32>()->Append(static_cast<std::time_t>(NUM2LL(to_i)));
|
|
422
|
+
VALUE t = coerce_to_time(value);
|
|
423
|
+
col->As<ColumnDate32>()->Append(static_cast<std::time_t>(NUM2LL(rb_funcall(t, rb_intern("to_i"), 0))));
|
|
282
424
|
return;
|
|
283
425
|
}
|
|
284
426
|
case Type::DateTime: {
|
|
285
|
-
VALUE
|
|
286
|
-
col->As<ColumnDateTime>()->Append(static_cast<std::time_t>(NUM2LL(to_i)));
|
|
427
|
+
VALUE t = coerce_to_time(value);
|
|
428
|
+
col->As<ColumnDateTime>()->Append(static_cast<std::time_t>(NUM2LL(rb_funcall(t, rb_intern("to_i"), 0))));
|
|
287
429
|
return;
|
|
288
430
|
}
|
|
289
431
|
case Type::DateTime64: {
|
|
290
432
|
size_t prec = type->As<DateTime64Type>()->GetPrecision();
|
|
291
|
-
|
|
433
|
+
VALUE t = coerce_to_time(value);
|
|
434
|
+
col->As<ColumnDateTime64>()->Append(time_to_datetime64_ticks(t, prec));
|
|
292
435
|
return;
|
|
293
436
|
}
|
|
294
437
|
|
|
@@ -321,6 +464,63 @@ static void append_value(const ColumnRef& col, VALUE value) {
|
|
|
321
464
|
return;
|
|
322
465
|
}
|
|
323
466
|
|
|
467
|
+
case Type::Decimal:
|
|
468
|
+
case Type::Decimal32:
|
|
469
|
+
case Type::Decimal64:
|
|
470
|
+
case Type::Decimal128: {
|
|
471
|
+
// clickhouse-cpp's Decimal::Append(string) only scales up when a
|
|
472
|
+
// decimal point is present (see columns/decimal.cpp). "1" at
|
|
473
|
+
// scale 8 is parsed as unscaled 1 => 1e-8, not 1. Normalise
|
|
474
|
+
// through BigDecimal#to_s("F") so Integer / Float / BigDecimal
|
|
475
|
+
// all land as a fixed-point string.
|
|
476
|
+
VALUE bd = rb_funcall(rb_cObject, rb_intern("BigDecimal"), 1,
|
|
477
|
+
rb_funcall(value, rb_intern("to_s"), 0));
|
|
478
|
+
VALUE str = rb_funcall(bd, rb_intern("to_s"), 1,
|
|
479
|
+
rb_utf8_str_new_cstr("F"));
|
|
480
|
+
col->As<ColumnDecimal>()->Append(
|
|
481
|
+
std::string(RSTRING_PTR(str), RSTRING_LEN(str)));
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
case Type::LowCardinality: {
|
|
486
|
+
// Only LowCardinality(String) and LowCardinality(Nullable(String))
|
|
487
|
+
// are supported for insert in v1 — this covers the profile-service
|
|
488
|
+
// use case. Numeric LC dictionaries are rare and can wait.
|
|
489
|
+
auto nested_type = type->As<LowCardinalityType>()->GetNestedType();
|
|
490
|
+
bool nullable = nested_type->GetCode() == Type::Nullable;
|
|
491
|
+
auto inner_type = nullable
|
|
492
|
+
? nested_type->As<NullableType>()->GetNestedType()
|
|
493
|
+
: nested_type;
|
|
494
|
+
auto inner_code = inner_type->GetCode();
|
|
495
|
+
if (inner_code != Type::String && inner_code != Type::FixedString) {
|
|
496
|
+
throw chn::EncoderFailure(
|
|
497
|
+
"LowCardinality(" + nested_type->GetName() + ") insert not supported");
|
|
498
|
+
}
|
|
499
|
+
if (!nullable) {
|
|
500
|
+
auto lct = col->AsStrict<ColumnLowCardinalityT<ColumnString>>();
|
|
501
|
+
StringValue(value);
|
|
502
|
+
lct->Append(std::string_view(RSTRING_PTR(value), RSTRING_LEN(value)));
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
// LowCardinality(Nullable(T)): the factory returns base
|
|
506
|
+
// ColumnLowCardinality wrapping ColumnNullable, not the templated
|
|
507
|
+
// T form. Build a one-row ColumnNullable<String> and Append it —
|
|
508
|
+
// the base class handles merging into the LC dictionary.
|
|
509
|
+
auto tmp_inner = std::make_shared<ColumnString>();
|
|
510
|
+
auto tmp_nulls = std::make_shared<ColumnUInt8>();
|
|
511
|
+
if (NIL_P(value)) {
|
|
512
|
+
tmp_inner->Append(std::string_view());
|
|
513
|
+
tmp_nulls->Append(1);
|
|
514
|
+
} else {
|
|
515
|
+
StringValue(value);
|
|
516
|
+
tmp_inner->Append(std::string_view(RSTRING_PTR(value), RSTRING_LEN(value)));
|
|
517
|
+
tmp_nulls->Append(0);
|
|
518
|
+
}
|
|
519
|
+
auto tmp = std::make_shared<ColumnNullable>(tmp_inner, tmp_nulls);
|
|
520
|
+
col->As<ColumnLowCardinality>()->Append(tmp);
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
|
|
324
524
|
default:
|
|
325
525
|
throw chn::EncoderFailure(
|
|
326
526
|
"cannot insert into column of type " + type->GetName());
|
|
@@ -415,6 +615,9 @@ static VALUE ch_client_initialize(int argc, VALUE* argv, VALUE self) {
|
|
|
415
615
|
rb_ivar_set(self, rb_intern("@host"), rb_utf8_str_new(host.data(), host.size()));
|
|
416
616
|
rb_ivar_set(self, rb_intern("@port"), UINT2NUM(port));
|
|
417
617
|
rb_ivar_set(self, rb_intern("@database"), rb_utf8_str_new(database.data(), database.size()));
|
|
618
|
+
|
|
619
|
+
VALUE logger = rb_hash_lookup2(kwargs, ID2SYM(rb_intern("logger")), Qnil);
|
|
620
|
+
rb_ivar_set(self, rb_intern("@logger"), logger);
|
|
418
621
|
return self;
|
|
419
622
|
}
|
|
420
623
|
|
|
@@ -466,7 +669,8 @@ static VALUE ch_client_execute(VALUE self, VALUE rb_sql) {
|
|
|
466
669
|
}
|
|
467
670
|
|
|
468
671
|
// ------------------------------------------------------------------
|
|
469
|
-
// query() —
|
|
672
|
+
// query() — buffers all rows into an array; GVL is held for the duration.
|
|
673
|
+
// See query_each below for a streaming, GVL-releasing variant.
|
|
470
674
|
// ------------------------------------------------------------------
|
|
471
675
|
|
|
472
676
|
static VALUE ch_client_query(VALUE self, VALUE rb_sql) {
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "mkmf"
|
|
2
4
|
require "fileutils"
|
|
3
5
|
require "etc"
|
|
@@ -5,7 +7,11 @@ require "shellwords"
|
|
|
5
7
|
|
|
6
8
|
EXT_DIR = __dir__
|
|
7
9
|
VENDOR = File.expand_path("vendor/clickhouse-cpp", EXT_DIR)
|
|
8
|
-
|
|
10
|
+
# Scope the cmake build dir by target arch so host and cross-compile
|
|
11
|
+
# builds don't share a CMakeCache.txt (which burns in absolute paths
|
|
12
|
+
# like the host cmake binary).
|
|
13
|
+
ARCH = RbConfig::CONFIG["arch"] || "unknown"
|
|
14
|
+
BUILD_DIR = File.expand_path("../../tmp/cpp-build-#{ARCH}", EXT_DIR)
|
|
9
15
|
|
|
10
16
|
def fatal(msg)
|
|
11
17
|
warn
|
|
@@ -46,6 +52,7 @@ unless find_executable("cmake")
|
|
|
46
52
|
end
|
|
47
53
|
|
|
48
54
|
cxx = ENV["CXX"] || RbConfig::CONFIG["CXX"] || "c++"
|
|
55
|
+
cc = ENV["CC"] || RbConfig::CONFIG["CC"] || "cc"
|
|
49
56
|
unless find_executable(cxx.split.first)
|
|
50
57
|
fatal <<~MSG
|
|
51
58
|
A C++17-capable compiler is required.
|
|
@@ -59,23 +66,49 @@ end
|
|
|
59
66
|
|
|
60
67
|
FileUtils.mkdir_p(BUILD_DIR)
|
|
61
68
|
|
|
69
|
+
darwin_cross = ARCH.include?("darwin") && Dir.exist?("/opt/osxcross")
|
|
70
|
+
|
|
71
|
+
build_env = {}
|
|
72
|
+
configure_args = [
|
|
73
|
+
"cmake",
|
|
74
|
+
"-S", VENDOR,
|
|
75
|
+
"-B", BUILD_DIR,
|
|
76
|
+
"-DCMAKE_C_COMPILER=#{cc.split.first}",
|
|
77
|
+
"-DCMAKE_CXX_COMPILER=#{cxx.split.first}",
|
|
78
|
+
"-DBUILD_SHARED_LIBS=OFF",
|
|
79
|
+
"-DBUILD_BENCHMARK=OFF",
|
|
80
|
+
"-DBUILD_TESTS=OFF",
|
|
81
|
+
"-DWITH_OPENSSL=OFF",
|
|
82
|
+
"-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
|
|
83
|
+
"-DCMAKE_BUILD_TYPE=Release"
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
if darwin_cross
|
|
87
|
+
# osxcross defaults MACOSX_DEPLOYMENT_TARGET to 10.13, but
|
|
88
|
+
# clickhouse-cpp calls std::optional::value() which libc++ marks
|
|
89
|
+
# unavailable below 10.14. Force the deployment target and disable
|
|
90
|
+
# libc++'s availability annotations.
|
|
91
|
+
build_env["MACOSX_DEPLOYMENT_TARGET"] = "10.15"
|
|
92
|
+
darwin_cxxflags = "-mmacosx-version-min=10.15 -D_LIBCPP_DISABLE_AVAILABILITY"
|
|
93
|
+
configure_args << "-DCMAKE_C_FLAGS=#{darwin_cxxflags}"
|
|
94
|
+
configure_args << "-DCMAKE_CXX_FLAGS=#{darwin_cxxflags}"
|
|
95
|
+
# Mach-O static archives need the osxcross ar/ranlib; the host's
|
|
96
|
+
# GNU ar leaves them without a table of contents that Apple's ld
|
|
97
|
+
# can read. cmake resolves relative FILEPATH to CWD, so use the
|
|
98
|
+
# absolute osxcross bindir.
|
|
99
|
+
prefix = cxx.split.first.sub(/-clang\+\+.*\z/, "")
|
|
100
|
+
osxcross_bin = "/opt/osxcross/target/bin"
|
|
101
|
+
configure_args << "-DCMAKE_AR=#{osxcross_bin}/#{prefix}-ar"
|
|
102
|
+
configure_args << "-DCMAKE_RANLIB=#{osxcross_bin}/#{prefix}-ranlib"
|
|
103
|
+
end
|
|
104
|
+
|
|
62
105
|
unless File.exist?(File.join(BUILD_DIR, "CMakeCache.txt"))
|
|
63
|
-
configure_args
|
|
64
|
-
"cmake"
|
|
65
|
-
"-S", VENDOR,
|
|
66
|
-
"-B", BUILD_DIR,
|
|
67
|
-
"-DBUILD_SHARED_LIBS=OFF",
|
|
68
|
-
"-DBUILD_BENCHMARK=OFF",
|
|
69
|
-
"-DBUILD_TESTS=OFF",
|
|
70
|
-
"-DWITH_OPENSSL=OFF",
|
|
71
|
-
"-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
|
|
72
|
-
"-DCMAKE_BUILD_TYPE=Release",
|
|
73
|
-
]
|
|
74
|
-
system(*configure_args) or fatal("cmake configure failed. See #{BUILD_DIR}/CMakeFiles/CMakeOutput.log")
|
|
106
|
+
system(build_env, *configure_args) or
|
|
107
|
+
fatal("cmake configure failed. See #{BUILD_DIR}/CMakeFiles/CMakeOutput.log")
|
|
75
108
|
end
|
|
76
109
|
|
|
77
110
|
jobs = ENV.fetch("MAKE_JOBS") { Etc.nprocessors.to_s }
|
|
78
|
-
system("cmake", "--build", BUILD_DIR, "--parallel", jobs) or fatal("cmake build failed")
|
|
111
|
+
system(build_env, "cmake", "--build", BUILD_DIR, "--parallel", jobs) or fatal("cmake build failed")
|
|
79
112
|
|
|
80
113
|
inc_dirs = [
|
|
81
114
|
VENDOR,
|
|
@@ -93,6 +126,12 @@ $CXXFLAGS = "#{$CXXFLAGS} -std=c++17 #{inc_dirs.map { |d| "-I#{d}" }.join(' ')}"
|
|
|
93
126
|
$CPPFLAGS = "#{$CPPFLAGS} #{inc_dirs.map { |d| "-I#{d}" }.join(' ')}"
|
|
94
127
|
$LDFLAGS = "#{$LDFLAGS} #{ordered_libs.map(&:shellescape).join(' ')}"
|
|
95
128
|
|
|
129
|
+
if darwin_cross
|
|
130
|
+
darwin_flags = "-mmacosx-version-min=10.15 -D_LIBCPP_DISABLE_AVAILABILITY"
|
|
131
|
+
$CXXFLAGS = "#{$CXXFLAGS} #{darwin_flags}"
|
|
132
|
+
$LDFLAGS = "#{$LDFLAGS} -mmacosx-version-min=10.15"
|
|
133
|
+
end
|
|
134
|
+
|
|
96
135
|
have_library("c++") || have_library("stdc++")
|
|
97
136
|
|
|
98
137
|
$objs = ["client.o"]
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module ClickhouseNative
|
|
2
4
|
class Client
|
|
3
5
|
attr_reader :host, :port, :database
|
|
@@ -13,32 +15,20 @@ module ClickhouseNative
|
|
|
13
15
|
# columns defaults to the first hash's keys (for Array<Hash>) or all table
|
|
14
16
|
# columns in DDL order (for Array<Array>).
|
|
15
17
|
# types may be supplied to skip the DESCRIBE lookup.
|
|
18
|
+
#
|
|
19
|
+
# Hash keys not present in the schema raise ArgumentError — if you need
|
|
20
|
+
# to insert a subset, pass `columns:` explicitly.
|
|
16
21
|
def insert(table, rows, columns: nil, db_name: nil, types: nil)
|
|
17
22
|
return 0 if rows.empty?
|
|
18
|
-
fq = db_name ? "#{db_name}.#{table}" : table
|
|
19
|
-
|
|
20
|
-
if types && columns
|
|
21
|
-
raise ArgumentError, "types and columns must have the same length" if columns.size != types.size
|
|
22
|
-
col_pairs = columns.zip(types).map { |n, t| [n.to_s, t] }
|
|
23
|
-
else
|
|
24
|
-
schema = describe_table(table, db_name: db_name)
|
|
25
|
-
type_by_name = schema.to_h { |c| [c[:name], c[:type]] }
|
|
26
|
-
columns ||= rows.first.is_a?(Hash) ? rows.first.keys.map(&:to_s) : schema.map { |c| c[:name] }
|
|
27
|
-
col_pairs = columns.map do |name|
|
|
28
|
-
name_s = name.to_s
|
|
29
|
-
t = type_by_name[name_s] or raise ArgumentError, "unknown column #{name_s.inspect} in #{fq}"
|
|
30
|
-
[name_s, t]
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
23
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
end
|
|
24
|
+
fq = db_name ? "#{db_name}.#{table}" : table
|
|
25
|
+
col_pairs =
|
|
26
|
+
if types && columns
|
|
27
|
+
zip_columns_and_types(columns, types)
|
|
39
28
|
else
|
|
40
|
-
rows
|
|
29
|
+
columns_from_schema(table, rows, columns, db_name, fq)
|
|
41
30
|
end
|
|
31
|
+
row_arrays = rows.first.is_a?(Hash) ? hash_rows_to_arrays(rows, col_pairs) : rows
|
|
42
32
|
|
|
43
33
|
insert_block(fq, col_pairs, row_arrays)
|
|
44
34
|
end
|
|
@@ -46,5 +36,31 @@ module ClickhouseNative
|
|
|
46
36
|
def inspect
|
|
47
37
|
"#<#{self.class} #{host}:#{port}/#{database}>"
|
|
48
38
|
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def zip_columns_and_types(columns, types)
|
|
43
|
+
if columns.size != types.size
|
|
44
|
+
raise ArgumentError, "types and columns must have the same length"
|
|
45
|
+
end
|
|
46
|
+
columns.zip(types).map { |n, t| [n.to_s, t] }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def columns_from_schema(table, rows, columns, db_name, fqn)
|
|
50
|
+
schema = describe_table(table, db_name: db_name)
|
|
51
|
+
type_by_name = schema.to_h { |c| [c[:name], c[:type]] }
|
|
52
|
+
columns ||= rows.first.is_a?(Hash) ? rows.first.keys.map(&:to_s) : schema.map { |c| c[:name] }
|
|
53
|
+
columns.map do |name|
|
|
54
|
+
name_s = name.to_s
|
|
55
|
+
t = type_by_name[name_s] or
|
|
56
|
+
raise ArgumentError, "unknown column #{name_s.inspect} in #{fqn}"
|
|
57
|
+
[name_s, t]
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def hash_rows_to_arrays(rows, col_pairs)
|
|
62
|
+
lookup = col_pairs.map { |n, _| [n.to_sym, n] }
|
|
63
|
+
rows.map { |h| lookup.map { |sym, str| h.fetch(sym) { h[str] } } }
|
|
64
|
+
end
|
|
49
65
|
end
|
|
50
66
|
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickhouseNative
|
|
4
|
+
# Sequel-style logging wrapper. Prepended onto Client so it fires for the
|
|
5
|
+
# raw C-level execute/query/query_value/query_each/insert_block calls.
|
|
6
|
+
#
|
|
7
|
+
# client = Client.new(..., logger: Rails.logger)
|
|
8
|
+
# client.query("SELECT 1")
|
|
9
|
+
# # => DEBUG -- : (0.421ms) SELECT 1
|
|
10
|
+
#
|
|
11
|
+
# Errors are logged at ERROR with the elapsed time and exception class.
|
|
12
|
+
module Logging
|
|
13
|
+
LEVEL = :debug
|
|
14
|
+
|
|
15
|
+
def execute(sql)
|
|
16
|
+
log_sql(sql) { super }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def query(sql)
|
|
20
|
+
log_sql(sql) { super }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def query_value(sql)
|
|
24
|
+
log_sql(sql) { super }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def query_each(sql, &)
|
|
28
|
+
log_sql(sql) { super }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def insert_block(table, columns, rows)
|
|
32
|
+
col_list = columns.map(&:first).join(", ")
|
|
33
|
+
log_sql("INSERT INTO #{table} (#{col_list}) VALUES (#{rows.size} rows)") { super }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def log_sql(sql)
|
|
39
|
+
return yield unless @logger
|
|
40
|
+
|
|
41
|
+
t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
42
|
+
begin
|
|
43
|
+
result = yield
|
|
44
|
+
elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000
|
|
45
|
+
@logger.public_send(LEVEL, format("(%<ms>.3fms) %<sql>s", ms: elapsed_ms, sql: sql))
|
|
46
|
+
result
|
|
47
|
+
rescue => error
|
|
48
|
+
elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000
|
|
49
|
+
first_line = error.message.to_s.lines.first.to_s.strip
|
|
50
|
+
@logger.error(format(
|
|
51
|
+
"(%<ms>.3fms) %<class>s: %<msg>s -- %<sql>s",
|
|
52
|
+
ms: elapsed_ms, class: error.class, msg: first_line, sql: sql,
|
|
53
|
+
))
|
|
54
|
+
raise
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
class Client
|
|
60
|
+
attr_accessor :logger
|
|
61
|
+
|
|
62
|
+
prepend Logging
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -1,17 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require "connection_pool"
|
|
2
4
|
|
|
3
5
|
module ClickhouseNative
|
|
4
6
|
class Pool
|
|
5
7
|
def initialize(host:, port:, database: "default", user: "default", password: "",
|
|
6
|
-
compression: :none, pool_size: 5, pool_timeout: 5)
|
|
7
|
-
client_kwargs = {host:, port:, database:, user:, password:, compression:}
|
|
8
|
+
compression: :none, logger: nil, pool_size: 5, pool_timeout: 5)
|
|
9
|
+
client_kwargs = { host:, port:, database:, user:, password:, compression:, logger: }
|
|
8
10
|
@pool = ConnectionPool.new(size: pool_size, timeout: pool_timeout) do
|
|
9
11
|
Client.new(**client_kwargs)
|
|
10
12
|
end
|
|
11
13
|
end
|
|
12
14
|
|
|
13
|
-
def with(&
|
|
14
|
-
@pool.with(&
|
|
15
|
+
def with(&)
|
|
16
|
+
@pool.with(&)
|
|
15
17
|
end
|
|
16
18
|
|
|
17
19
|
def execute(sql)
|
data/lib/clickhouse_native.rb
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
require "bigdecimal"
|
|
1
5
|
require "clickhouse_native/version"
|
|
2
6
|
require "clickhouse_native/errors"
|
|
3
7
|
require "clickhouse_native/clickhouse_native"
|
|
4
8
|
require "clickhouse_native/client"
|
|
9
|
+
require "clickhouse_native/logging"
|
|
5
10
|
require "clickhouse_native/pool"
|
|
6
11
|
|
|
7
12
|
module ClickhouseNative
|
metadata
CHANGED
|
@@ -1,84 +1,29 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: clickhouse-native
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yuri Smirnov
|
|
8
|
+
autorequire:
|
|
8
9
|
bindir: bin
|
|
9
10
|
cert_chain: []
|
|
10
|
-
date:
|
|
11
|
+
date: 2026-04-22 00:00:00.000000000 Z
|
|
11
12
|
dependencies:
|
|
12
13
|
- !ruby/object:Gem::Dependency
|
|
13
14
|
name: connection_pool
|
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
|
15
16
|
requirements:
|
|
16
|
-
- - "
|
|
17
|
+
- - ">="
|
|
17
18
|
- !ruby/object:Gem::Version
|
|
18
19
|
version: '2.4'
|
|
19
20
|
type: :runtime
|
|
20
21
|
prerelease: false
|
|
21
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
23
|
requirements:
|
|
23
|
-
- - "
|
|
24
|
+
- - ">="
|
|
24
25
|
- !ruby/object:Gem::Version
|
|
25
26
|
version: '2.4'
|
|
26
|
-
- !ruby/object:Gem::Dependency
|
|
27
|
-
name: rake
|
|
28
|
-
requirement: !ruby/object:Gem::Requirement
|
|
29
|
-
requirements:
|
|
30
|
-
- - "~>"
|
|
31
|
-
- !ruby/object:Gem::Version
|
|
32
|
-
version: '13.2'
|
|
33
|
-
type: :development
|
|
34
|
-
prerelease: false
|
|
35
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
-
requirements:
|
|
37
|
-
- - "~>"
|
|
38
|
-
- !ruby/object:Gem::Version
|
|
39
|
-
version: '13.2'
|
|
40
|
-
- !ruby/object:Gem::Dependency
|
|
41
|
-
name: rake-compiler
|
|
42
|
-
requirement: !ruby/object:Gem::Requirement
|
|
43
|
-
requirements:
|
|
44
|
-
- - "~>"
|
|
45
|
-
- !ruby/object:Gem::Version
|
|
46
|
-
version: '1.2'
|
|
47
|
-
type: :development
|
|
48
|
-
prerelease: false
|
|
49
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
-
requirements:
|
|
51
|
-
- - "~>"
|
|
52
|
-
- !ruby/object:Gem::Version
|
|
53
|
-
version: '1.2'
|
|
54
|
-
- !ruby/object:Gem::Dependency
|
|
55
|
-
name: rake-compiler-dock
|
|
56
|
-
requirement: !ruby/object:Gem::Requirement
|
|
57
|
-
requirements:
|
|
58
|
-
- - "~>"
|
|
59
|
-
- !ruby/object:Gem::Version
|
|
60
|
-
version: '1.9'
|
|
61
|
-
type: :development
|
|
62
|
-
prerelease: false
|
|
63
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
64
|
-
requirements:
|
|
65
|
-
- - "~>"
|
|
66
|
-
- !ruby/object:Gem::Version
|
|
67
|
-
version: '1.9'
|
|
68
|
-
- !ruby/object:Gem::Dependency
|
|
69
|
-
name: rspec
|
|
70
|
-
requirement: !ruby/object:Gem::Requirement
|
|
71
|
-
requirements:
|
|
72
|
-
- - "~>"
|
|
73
|
-
- !ruby/object:Gem::Version
|
|
74
|
-
version: '3.13'
|
|
75
|
-
type: :development
|
|
76
|
-
prerelease: false
|
|
77
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
-
requirements:
|
|
79
|
-
- - "~>"
|
|
80
|
-
- !ruby/object:Gem::Version
|
|
81
|
-
version: '3.13'
|
|
82
27
|
description: A high-performance Ruby client for ClickHouse using the native binary
|
|
83
28
|
protocol via a C++ extension wrapping clickhouse-cpp.
|
|
84
29
|
email:
|
|
@@ -341,14 +286,16 @@ files:
|
|
|
341
286
|
- ext/clickhouse_native/vendor/clickhouse-cpp/contrib/zstd/zstd/zstd.h
|
|
342
287
|
- ext/clickhouse_native/vendor/clickhouse-cpp/contrib/zstd/zstd/zstd_errors.h
|
|
343
288
|
- lib/clickhouse_native.rb
|
|
344
|
-
- lib/clickhouse_native/clickhouse_native.bundle
|
|
345
289
|
- lib/clickhouse_native/client.rb
|
|
346
290
|
- lib/clickhouse_native/errors.rb
|
|
291
|
+
- lib/clickhouse_native/logging.rb
|
|
347
292
|
- lib/clickhouse_native/pool.rb
|
|
348
293
|
- lib/clickhouse_native/version.rb
|
|
294
|
+
homepage:
|
|
349
295
|
licenses:
|
|
350
296
|
- Apache-2.0
|
|
351
297
|
metadata: {}
|
|
298
|
+
post_install_message:
|
|
352
299
|
rdoc_options: []
|
|
353
300
|
require_paths:
|
|
354
301
|
- lib
|
|
@@ -356,14 +303,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
356
303
|
requirements:
|
|
357
304
|
- - ">="
|
|
358
305
|
- !ruby/object:Gem::Version
|
|
359
|
-
version: 3.
|
|
306
|
+
version: 3.3.0
|
|
360
307
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
361
308
|
requirements:
|
|
362
309
|
- - ">="
|
|
363
310
|
- !ruby/object:Gem::Version
|
|
364
311
|
version: '0'
|
|
365
312
|
requirements: []
|
|
366
|
-
rubygems_version:
|
|
313
|
+
rubygems_version: 3.5.22
|
|
314
|
+
signing_key:
|
|
367
315
|
specification_version: 4
|
|
368
316
|
summary: ClickHouse Ruby driver over the native TCP protocol
|
|
369
317
|
test_files: []
|
|
Binary file
|