click_house 2.0.1 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +34 -0
- data/Gemfile.lock +1 -1
- data/Gemfile_faraday1 +1 -1
- data/Gemfile_faraday1.lock +3 -4
- data/Gemfile_faraday2.lock +1 -1
- data/README.md +55 -35
- data/lib/click_house/benchmark/map_join.rb +20 -0
- data/lib/click_house/config.rb +29 -3
- data/lib/click_house/connection.rb +16 -1
- data/lib/click_house/errors.rb +1 -0
- data/lib/click_house/extend/connection_inserting.rb +62 -10
- data/lib/click_house/extend/connection_selective.rb +3 -3
- data/lib/click_house/extend/connection_table.rb +6 -1
- data/lib/click_house/response/execution.rb +70 -0
- data/lib/click_house/response/factory.rb +29 -9
- data/lib/click_house/response/result_set.rb +81 -6
- data/lib/click_house/response.rb +1 -1
- data/lib/click_house/serializer/base.rb +32 -0
- data/lib/click_house/serializer/json_oj_serializer.rb +17 -0
- data/lib/click_house/serializer/json_serializer.rb +11 -0
- data/lib/click_house/serializer.rb +9 -0
- data/lib/click_house/type/array_type.rb +4 -0
- data/lib/click_house/type/base_type.rb +4 -0
- data/lib/click_house/type/boolean_type.rb +6 -1
- data/lib/click_house/type/date_time64_type.rb +1 -1
- data/lib/click_house/type/date_time_type.rb +1 -1
- data/lib/click_house/type/decimal_type.rb +5 -4
- data/lib/click_house/type/string_type.rb +1 -1
- data/lib/click_house/version.rb +1 -1
- data/lib/click_house.rb +5 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f271ed936f2d0f66ec2045ec72025e317523dc8d70123c2bcf145ede5b4f1871
|
4
|
+
data.tar.gz: d3e48437b08cbf5eb96315bd46ba88429302bcbebc882d787257c3636c586c8f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3bae70c6d33a93ad298db0d14b91beb4bbfe31117cb51e9a58cdeeb95dc48e506363d3e660e6b9e3cd54398df2065210f3b0582e1891bb76e2bdd900520269b3
|
7
|
+
data.tar.gz: cb202cf091808068383a32bb3a2a078aaacc3e051e5c30ecfc47d0bdfab84bb188f0ace4dbc6dbd2950407e5743deeb5f2ffd82c3b31a27ef6fcea01a3fdab1d
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,36 @@
|
|
1
|
+
# 2.1.0
|
2
|
+
* `ClickHouse.connection.insert` now returns `ClickHouse::Response::Execution` objects
|
3
|
+
with methods `headers`, `summary`, `written_rows`, `written_bytes`, etc...
|
4
|
+
* `ClickHouse.connection.insert(columns: ["id"], values: [1])` now uses `JSONCompactEachRow` by default
|
5
|
+
(to increase JSON serialization speed)
|
6
|
+
* Methods `insert_rows` and `insert_compact` added to `connection`
|
7
|
+
* Added ability to pass object directly to insert like:
|
8
|
+
`ClickHouse.connection.insert("table", {id: 1})` or
|
9
|
+
`ClickHouse.connection.insert("table", [{id: 1})]` (for ruby < 3.0 use `ClickHouse.connection.insert("table", [{id: 1}], {})`)
|
10
|
+
* 🔥 Added config option `json_serializer` (one of `ClickHouse::Serializer::JsonSerializer`, `ClickHouse::Serializer::JsonOjSerializer`)
|
11
|
+
* 🔥 Added config option `symbolize_keys`
|
12
|
+
* 🔥 Added type serialization for INSERT statements, example below:
|
13
|
+
|
14
|
+
```sql
|
15
|
+
CREATE TABLE assets(visible Boolean, tags Array(Nullable(String))) ENGINE Memory
|
16
|
+
```
|
17
|
+
|
18
|
+
```ruby
|
19
|
+
# cache table schema in a class variable
|
20
|
+
@schema = ClickHouse.connection.table_schema('assets')
|
21
|
+
|
22
|
+
# Json each row
|
23
|
+
ClickHouse.connection.insert('assets', @schema.serialize({'visible' => true, 'tags' => ['ruby']}))
|
24
|
+
|
25
|
+
# Json compact
|
26
|
+
ClickHouse.connection.insert('assets', columns: %w[visible tags]) do |buffer|
|
27
|
+
buffer << [
|
28
|
+
@schema.serialize_column("visible", true),
|
29
|
+
@schema.serialize_column("tags", ['ruby']),
|
30
|
+
]
|
31
|
+
end
|
32
|
+
```
|
33
|
+
|
1
34
|
# 2.0.0
|
2
35
|
* Fixed `Bigdecimal` casting with high precision
|
3
36
|
* Added nested `type casting like Array(Array(Array(Nullable(T))))`
|
@@ -5,6 +38,7 @@
|
|
5
38
|
* Added `Tuple(T1, T2)` support
|
6
39
|
* Added support for `Faraday` v1 and v2
|
7
40
|
* Added support for `Oj` parser
|
41
|
+
* Time types return `Time` class instead of `DateTime` for now
|
8
42
|
|
9
43
|
# 1.6.3
|
10
44
|
* [PR](https://github.com/shlima/click_house/pull/38) Add option format for insert
|
data/Gemfile.lock
CHANGED
data/Gemfile_faraday1
CHANGED
data/Gemfile_faraday1.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
click_house (1.
|
4
|
+
click_house (2.1.0)
|
5
5
|
activesupport
|
6
6
|
faraday (>= 1.7, < 3)
|
7
7
|
|
@@ -40,8 +40,6 @@ GEM
|
|
40
40
|
faraday-patron (1.0.0)
|
41
41
|
faraday-rack (1.0.0)
|
42
42
|
faraday-retry (1.0.3)
|
43
|
-
faraday_middleware (1.2.0)
|
44
|
-
faraday (~> 1.0)
|
45
43
|
i18n (1.12.0)
|
46
44
|
concurrent-ruby (~> 1.0)
|
47
45
|
json (2.6.2)
|
@@ -94,12 +92,13 @@ GEM
|
|
94
92
|
unicode-display_width (2.3.0)
|
95
93
|
|
96
94
|
PLATFORMS
|
95
|
+
ruby
|
97
96
|
x86_64-darwin-21
|
98
97
|
|
99
98
|
DEPENDENCIES
|
100
99
|
bundler
|
101
100
|
click_house!
|
102
|
-
|
101
|
+
faraday (< 2)
|
103
102
|
oj
|
104
103
|
pry
|
105
104
|
rake
|
data/Gemfile_faraday2.lock
CHANGED
data/README.md
CHANGED
@@ -52,6 +52,8 @@ ClickHouse.config do |config|
|
|
52
52
|
config.timeout = 60
|
53
53
|
config.open_timeout = 3
|
54
54
|
config.ssl_verify = false
|
55
|
+
# set to true to symbolize keys for SELECT and INSERT statements (type casting)
|
56
|
+
config.symbolize_keys = false
|
55
57
|
config.headers = {}
|
56
58
|
|
57
59
|
# or provide connection options separately
|
@@ -66,10 +68,15 @@ ClickHouse.config do |config|
|
|
66
68
|
# if you want to add settings to all queries
|
67
69
|
config.global_params = { mutations_sync: 1 }
|
68
70
|
|
69
|
-
# choose a ruby JSON parser
|
71
|
+
# choose a ruby JSON parser (default one)
|
70
72
|
config.json_parser = ClickHouse::Middleware::ParseJson
|
71
73
|
# or Oj parser
|
72
74
|
config.json_parser = ClickHouse::Middleware::ParseJsonOj
|
75
|
+
|
76
|
+
# JSON.dump (default one)
|
77
|
+
config.json_serializer = ClickHouse::Serializer::JsonSerializer
|
78
|
+
# or Oj.dump
|
79
|
+
config.json_serializer = ClickHouse::Serializer::JsonOjSerializer
|
73
80
|
end
|
74
81
|
```
|
75
82
|
|
@@ -195,7 +202,8 @@ response.body #=> "\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
|
|
195
202
|
|
196
203
|
## Insert
|
197
204
|
|
198
|
-
When column names and values are transferred separately
|
205
|
+
When column names and values are transferred separately, data sends to the server
|
206
|
+
using `JSONCompactEachRow` format by default.
|
199
207
|
|
200
208
|
```ruby
|
201
209
|
ClickHouse.connection.insert('table', columns: %i[id name]) do |buffer|
|
@@ -203,23 +211,42 @@ ClickHouse.connection.insert('table', columns: %i[id name]) do |buffer|
|
|
203
211
|
buffer << [2, 'Venus']
|
204
212
|
end
|
205
213
|
|
214
|
+
# or
|
206
215
|
ClickHouse.connection.insert('table', columns: %i[id name], values: [[1, 'Mercury'], [2, 'Venus']])
|
207
|
-
#=> true
|
208
216
|
```
|
209
217
|
|
210
|
-
When rows are passed as a
|
211
|
-
|
218
|
+
When rows are passed as an Array or a Hash, data sends to the server
|
219
|
+
using `JSONEachRow` format by default.
|
212
220
|
|
213
221
|
```ruby
|
214
|
-
ClickHouse.connection.insert('table',
|
222
|
+
ClickHouse.connection.insert('table', [{ name: 'Sun', id: 1 }, { name: 'Moon', id: 2 }])
|
223
|
+
|
224
|
+
# or
|
225
|
+
ClickHouse.connection.insert('table', { name: 'Sun', id: 1 })
|
226
|
+
|
227
|
+
# for ruby < 3.0 provide an extra argument
|
228
|
+
ClickHouse.connection.insert('table', { name: 'Sun', id: 1 }, {})
|
215
229
|
|
230
|
+
# or
|
216
231
|
ClickHouse.connection.insert('table') do |buffer|
|
217
232
|
buffer << { name: 'Sun', id: 1 }
|
218
233
|
buffer << { name: 'Moon', id: 2 }
|
219
234
|
end
|
220
|
-
#=> true
|
221
235
|
```
|
222
236
|
|
237
|
+
Sometimes it's needed to use other format than `JSONEachRow` For example if you want to send BigDecimal's
|
238
|
+
you could use `JSONStringsEachRow` format so string representation of `BigDecimal` will be parsed:
|
239
|
+
|
240
|
+
```ruby
|
241
|
+
ClickHouse.connection.insert('table', { name: 'Sun', id: '1' }, format: 'JSONStringsEachRow')
|
242
|
+
# or
|
243
|
+
ClickHouse.connection.insert_rows('table', { name: 'Sun', id: '1' }, format: 'JSONStringsEachRow')
|
244
|
+
# or
|
245
|
+
ClickHouse.connection.insert_compact('table', columns: %w[name id], values: %w[Sun 1], format: 'JSONCompactStringsEachRow')
|
246
|
+
```
|
247
|
+
|
248
|
+
See the [type casting](#type-casting) section to insert the data in a proper way.
|
249
|
+
|
223
250
|
## Create a table
|
224
251
|
### Create table using DSL
|
225
252
|
|
@@ -326,42 +353,35 @@ end
|
|
326
353
|
## Type casting
|
327
354
|
|
328
355
|
By default gem provides all necessary type casting, but you may overwrite or define
|
329
|
-
your own logic
|
356
|
+
your own logic. if you need to redefine all built-in types with your implementation,
|
357
|
+
just clear the default type system:
|
330
358
|
|
331
359
|
```ruby
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
end
|
336
|
-
|
337
|
-
def serialize(value)
|
338
|
-
value.strftime('%Y-%m-%d')
|
339
|
-
end
|
340
|
-
end
|
341
|
-
|
342
|
-
ClickHouse.add_type('Date', DateType.new)
|
360
|
+
ClickHouse.types.clear
|
361
|
+
ClickHouse.types # => {}
|
362
|
+
ClickHouse.types.default #=> #<ClickHouse::Type::UndefinedType>
|
343
363
|
```
|
344
364
|
|
345
|
-
|
346
|
-
argument and *Numeric* type with `%d` argument:
|
365
|
+
Type casting works automatically when fetching data, when inserting data, you must serialize the types yourself
|
347
366
|
|
348
|
-
```
|
349
|
-
|
350
|
-
def cast(value, time_zone)
|
351
|
-
Time.parse("#{value} #{time_zone}")
|
352
|
-
end
|
353
|
-
end
|
354
|
-
|
355
|
-
ClickHouse.add_type('DateTime(%s)', DateTimeType.new)
|
367
|
+
```sql
|
368
|
+
CREATE TABLE assets(visible Boolean, tags Array(Nullable(String))) ENGINE Memory
|
356
369
|
```
|
357
370
|
|
358
|
-
if you need to redefine all built-in types with your implementation,
|
359
|
-
just clear the default type system:
|
360
|
-
|
361
371
|
```ruby
|
362
|
-
|
363
|
-
ClickHouse.
|
364
|
-
|
372
|
+
# cache table schema in a class variable
|
373
|
+
@schema = ClickHouse.connection.table_schema('assets')
|
374
|
+
|
375
|
+
# Json each row
|
376
|
+
ClickHouse.connection.insert('assets', @schema.serialize({'visible' => true, 'tags' => ['ruby']}))
|
377
|
+
|
378
|
+
# Json compact
|
379
|
+
ClickHouse.connection.insert('assets', columns: %w[visible tags]) do |buffer|
|
380
|
+
buffer << [
|
381
|
+
@schema.serialize_column("visible", true),
|
382
|
+
@schema.serialize_column("tags", ['ruby']),
|
383
|
+
]
|
384
|
+
end
|
365
385
|
```
|
366
386
|
|
367
387
|
## Using with a connection pool
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'benchmark'
|
4
|
+
require 'stringio'
|
5
|
+
|
6
|
+
INPUT = Array.new(5_000_000, 'foo bar')
|
7
|
+
|
8
|
+
Benchmark.bm do |x|
|
9
|
+
x.report('map.join') do
|
10
|
+
INPUT.map(&:to_s).join("\n")
|
11
|
+
end
|
12
|
+
|
13
|
+
x.report('StringIO') do
|
14
|
+
out = StringIO.new
|
15
|
+
INPUT.each do |value|
|
16
|
+
out << "#{value}\n"
|
17
|
+
end
|
18
|
+
out.string
|
19
|
+
end
|
20
|
+
end
|
data/lib/click_house/config.rb
CHANGED
@@ -18,18 +18,21 @@ module ClickHouse
|
|
18
18
|
headers: {},
|
19
19
|
global_params: {},
|
20
20
|
json_parser: ClickHouse::Middleware::ParseJson,
|
21
|
+
json_serializer: ClickHouse::Serializer::JsonSerializer,
|
22
|
+
oj_dump_options: {
|
23
|
+
mode: :compat # to be able to dump improper JSON like {1 => 2}
|
24
|
+
},
|
21
25
|
oj_load_options: {
|
22
26
|
mode: :custom,
|
23
27
|
allow_blank: true,
|
24
28
|
bigdecimal_as_decimal: false, # dump BigDecimal as a String
|
25
29
|
bigdecimal_load: :bigdecimal, # convert all decimal numbers to BigDecimal
|
26
|
-
empty_string: false,
|
27
|
-
second_precision: 6,
|
28
|
-
time_format: :ruby,
|
29
30
|
},
|
30
31
|
json_load_options: {
|
31
32
|
decimal_class: BigDecimal,
|
32
33
|
},
|
34
|
+
# should be after json load options
|
35
|
+
symbolize_keys: false,
|
33
36
|
}.freeze
|
34
37
|
|
35
38
|
attr_accessor :adapter
|
@@ -49,6 +52,9 @@ module ClickHouse
|
|
49
52
|
attr_accessor :oj_load_options
|
50
53
|
attr_accessor :json_load_options
|
51
54
|
attr_accessor :json_parser # response middleware
|
55
|
+
attr_accessor :oj_dump_options
|
56
|
+
attr_accessor :json_serializer # [ClickHouse::Serializer::Base]
|
57
|
+
attr_accessor :symbolize_keys # [NilClass, Boolean]
|
52
58
|
|
53
59
|
def initialize(params = {})
|
54
60
|
assign(DEFAULTS.merge(params))
|
@@ -77,5 +83,25 @@ module ClickHouse
|
|
77
83
|
def null_logger
|
78
84
|
@null_logger ||= Logger.new(IO::NULL)
|
79
85
|
end
|
86
|
+
|
87
|
+
# @param klass [ClickHouse::Serializer::Base]
|
88
|
+
def json_serializer=(klass)
|
89
|
+
@json_serializer = klass.new(self)
|
90
|
+
end
|
91
|
+
|
92
|
+
def symbolize_keys=(value)
|
93
|
+
bool = value ? true : false
|
94
|
+
|
95
|
+
# merge to be able to clone a config
|
96
|
+
# prevent overriding default values
|
97
|
+
self.oj_load_options = oj_load_options.merge(symbol_keys: bool)
|
98
|
+
self.json_load_options = json_load_options.merge(symbolize_names: bool)
|
99
|
+
@symbolize_keys = bool
|
100
|
+
end
|
101
|
+
|
102
|
+
# @param name [Symbol, String]
|
103
|
+
def key(name)
|
104
|
+
symbolize_keys ? name.to_sym : name.to_s
|
105
|
+
end
|
80
106
|
end
|
81
107
|
end
|
@@ -47,13 +47,22 @@ module ClickHouse
|
|
47
47
|
end
|
48
48
|
|
49
49
|
# transport should work the same both with Faraday v1 and Faraday v2
|
50
|
+
# rubocop:disable Metrics/AbcSize
|
50
51
|
def transport
|
51
52
|
@transport ||= Faraday.new(config.url!) do |conn|
|
52
53
|
conn.options.timeout = config.timeout
|
53
54
|
conn.options.open_timeout = config.open_timeout
|
54
55
|
conn.headers = config.headers
|
55
56
|
conn.ssl.verify = config.ssl_verify
|
56
|
-
|
57
|
+
|
58
|
+
if config.auth?
|
59
|
+
if faraday_v1?
|
60
|
+
conn.request :basic_auth, config.username, config.password
|
61
|
+
else
|
62
|
+
conn.request :authorization, :basic, config.username, config.password
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
57
66
|
conn.response Middleware::RaiseError
|
58
67
|
conn.response Middleware::Logging, logger: config.logger!
|
59
68
|
conn.response config.json_parser, content_type: %r{application/json}, parser_options: { config: config }
|
@@ -61,10 +70,16 @@ module ClickHouse
|
|
61
70
|
conn.adapter config.adapter
|
62
71
|
end
|
63
72
|
end
|
73
|
+
# rubocop:enable Metrics/AbcSize
|
64
74
|
|
65
75
|
def compose(path, query = {})
|
66
76
|
# without <query.compact> "DB::Exception: Empty query" error will occur
|
67
77
|
"#{path}?#{URI.encode_www_form({ send_progress_in_http_headers: 1 }.merge(query).compact)}"
|
68
78
|
end
|
79
|
+
|
80
|
+
# @return [Boolean]
|
81
|
+
def faraday_v1?
|
82
|
+
Faraday::VERSION.start_with?('1')
|
83
|
+
end
|
69
84
|
end
|
70
85
|
end
|
data/lib/click_house/errors.rb
CHANGED
@@ -3,7 +3,8 @@
|
|
3
3
|
module ClickHouse
|
4
4
|
module Extend
|
5
5
|
module ConnectionInserting
|
6
|
-
|
6
|
+
DEFAULT_JSON_EACH_ROW_FORMAT = 'JSONEachRow'
|
7
|
+
DEFAULT_JSON_COMPACT_EACH_ROW_FORMAT = 'JSONCompactEachRow'
|
7
8
|
|
8
9
|
# @return [Boolean]
|
9
10
|
#
|
@@ -13,20 +14,71 @@ module ClickHouse
|
|
13
14
|
# buffer << ['Moon', 2]
|
14
15
|
# end
|
15
16
|
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
|
19
|
-
|
17
|
+
# @return [Response::Execution]
|
18
|
+
# @param body [Array, Hash]
|
19
|
+
# rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
|
20
|
+
def insert(table, body = [], **opts)
|
21
|
+
# In Ruby < 3.0, if the last argument is a hash, and the method being called
|
22
|
+
# accepts keyword arguments, then it is always converted to keyword arguments.
|
23
|
+
columns = opts.fetch(:columns, [])
|
24
|
+
values = opts.fetch(:values, [])
|
25
|
+
format = opts.fetch(:format, nil)
|
26
|
+
|
27
|
+
yield(body) if block_given?
|
20
28
|
|
21
|
-
|
22
|
-
|
29
|
+
# values: [{id: 1}]
|
30
|
+
if values.any? && columns.empty?
|
31
|
+
return insert_rows(table, values, format: format)
|
32
|
+
end
|
33
|
+
|
34
|
+
# body: [{id: 1}]
|
35
|
+
if body.any? && columns.empty?
|
36
|
+
return insert_rows(table, body, format: format)
|
37
|
+
end
|
38
|
+
|
39
|
+
# body: [1], columns: ["id"]
|
40
|
+
if body.any? && columns.any?
|
41
|
+
return insert_compact(table, columns: columns, values: body, format: format)
|
42
|
+
end
|
43
|
+
|
44
|
+
# columns: ["id"], values: [[1]]
|
45
|
+
if columns.any? && values.any?
|
46
|
+
return insert_compact(table, columns: columns, values: values, format: format)
|
47
|
+
end
|
48
|
+
|
49
|
+
Response::Factory.empty_exec
|
50
|
+
end
|
51
|
+
# rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
|
52
|
+
|
53
|
+
# @param table [String]
|
54
|
+
# @param body [Array, Hash]
|
55
|
+
# @param format [String]
|
56
|
+
# @return [Response::Execution]
|
57
|
+
#
|
58
|
+
# Sometimes it's needed to use other format than JSONEachRow
|
59
|
+
# For example if you want to send BigDecimal's you could use
|
60
|
+
# JSONStringsEachRow format so string representation of BigDecimal will be parsed
|
61
|
+
def insert_rows(table, body, format: nil)
|
62
|
+
format ||= DEFAULT_JSON_EACH_ROW_FORMAT
|
63
|
+
|
64
|
+
case body
|
65
|
+
when Hash
|
66
|
+
Response::Factory.exec(execute("INSERT INTO #{table} FORMAT #{format}", config.json_serializer.dump(body)))
|
67
|
+
when Array
|
68
|
+
Response::Factory.exec(execute("INSERT INTO #{table} FORMAT #{format}", config.json_serializer.dump_each_row(body)))
|
23
69
|
else
|
24
|
-
|
70
|
+
raise ArgumentError, "unknown body class <#{body.class}>"
|
25
71
|
end
|
72
|
+
end
|
26
73
|
|
27
|
-
|
74
|
+
# @return [Response::Execution]
|
75
|
+
def insert_compact(table, columns: [], values: [], format: nil)
|
76
|
+
format ||= DEFAULT_JSON_COMPACT_EACH_ROW_FORMAT
|
77
|
+
|
78
|
+
yield(values) if block_given?
|
28
79
|
|
29
|
-
execute("INSERT INTO #{table} FORMAT #{format}",
|
80
|
+
response = execute("INSERT INTO #{table} (#{columns.join(',')}) FORMAT #{format}", config.json_serializer.dump_each_row(values))
|
81
|
+
Response::Factory.exec(response)
|
30
82
|
end
|
31
83
|
end
|
32
84
|
end
|
@@ -6,17 +6,17 @@ module ClickHouse
|
|
6
6
|
# @return [ResultSet]
|
7
7
|
def select_all(sql)
|
8
8
|
response = get(body: sql, query: { default_format: 'JSON' })
|
9
|
-
Response::Factory
|
9
|
+
Response::Factory.response(response, config)
|
10
10
|
end
|
11
11
|
|
12
12
|
def select_value(sql)
|
13
13
|
response = get(body: sql, query: { default_format: 'JSON' })
|
14
|
-
Array(Response::Factory
|
14
|
+
Array(Response::Factory.response(response, config).first).dig(0, -1)
|
15
15
|
end
|
16
16
|
|
17
17
|
def select_one(sql)
|
18
18
|
response = get(body: sql, query: { default_format: 'JSON' })
|
19
|
-
Response::Factory
|
19
|
+
Response::Factory.response(response, config).first
|
20
20
|
end
|
21
21
|
end
|
22
22
|
end
|
@@ -10,7 +10,12 @@ module ClickHouse
|
|
10
10
|
|
11
11
|
# @return [ResultSet]
|
12
12
|
def describe_table(name)
|
13
|
-
Response::Factory
|
13
|
+
Response::Factory.response(execute("DESCRIBE TABLE #{name} FORMAT JSON"), config)
|
14
|
+
end
|
15
|
+
|
16
|
+
# @return [ResultSet]
|
17
|
+
def table_schema(name)
|
18
|
+
Response::Factory.response(execute("SELECT * FROM #{name} WHERE 1=0 FORMAT JSON"), config)
|
14
19
|
end
|
15
20
|
|
16
21
|
# @return [Boolean]
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ClickHouse
|
4
|
+
module Response
|
5
|
+
class Execution
|
6
|
+
SUMMARY_HEADER = 'x-clickhouse-summary'
|
7
|
+
|
8
|
+
attr_reader :headers, :summary
|
9
|
+
|
10
|
+
# @param headers [Faraday::Utils::Headers]
|
11
|
+
def initialize(headers: Faraday::Utils::Headers.new)
|
12
|
+
@headers = headers
|
13
|
+
@summary = parse_summary(headers[SUMMARY_HEADER])
|
14
|
+
end
|
15
|
+
|
16
|
+
# @return [Integer]
|
17
|
+
def read_rows
|
18
|
+
summary['read_rows'].to_i
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [Integer]
|
22
|
+
def read_bytes
|
23
|
+
summary['read_bytes'].to_i
|
24
|
+
end
|
25
|
+
|
26
|
+
# @return [Integer]
|
27
|
+
def written_rows
|
28
|
+
summary['written_rows'].to_i
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [Integer]
|
32
|
+
def written_bytes
|
33
|
+
summary['written_bytes'].to_i
|
34
|
+
end
|
35
|
+
|
36
|
+
# @return [Integer]
|
37
|
+
def total_rows_to_read
|
38
|
+
summary['total_rows_to_read'].to_i
|
39
|
+
end
|
40
|
+
|
41
|
+
# @return [Integer]
|
42
|
+
def result_rows
|
43
|
+
summary['result_rows'].to_i
|
44
|
+
end
|
45
|
+
|
46
|
+
# @return [Integer]
|
47
|
+
def result_bytes
|
48
|
+
summary['result_bytes'].to_i
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
# @return [Hash]
|
54
|
+
# {
|
55
|
+
# "read_rows" => "1",
|
56
|
+
# "read_bytes" => "23",
|
57
|
+
# "written_rows" => "1",
|
58
|
+
# "written_bytes" => "23",
|
59
|
+
# "total_rows_to_read" => "0",
|
60
|
+
# "result_rows" => "1",
|
61
|
+
# "result_bytes" => "23",
|
62
|
+
# }
|
63
|
+
def parse_summary(value)
|
64
|
+
return {} if value.nil? || value.empty?
|
65
|
+
|
66
|
+
JSON.parse(value)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -3,21 +3,41 @@
|
|
3
3
|
module ClickHouse
|
4
4
|
module Response
|
5
5
|
class Factory
|
6
|
-
|
7
|
-
|
8
|
-
|
6
|
+
KEY_META = 'meta'
|
7
|
+
KEY_DATA = 'data'
|
8
|
+
KEY_TOTALS = 'totals'
|
9
|
+
KEY_STATISTICS = 'statistics'
|
10
|
+
KEY_ROWS_BEFORE_LIMIT_AT_LEAST = 'rows_before_limit_at_least'
|
11
|
+
|
12
|
+
# @return [Nil], ResultSet]
|
13
|
+
# @params faraday [Faraday::Response]
|
14
|
+
# @params config [Config]
|
15
|
+
def self.response(faraday, config)
|
9
16
|
body = faraday.body
|
10
17
|
|
11
|
-
return body
|
18
|
+
return body unless body.is_a?(Hash)
|
19
|
+
return body unless body.key?(config.key(KEY_META)) && body.key?(config.key(KEY_DATA))
|
12
20
|
|
13
21
|
ResultSet.new(
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
22
|
+
config: config,
|
23
|
+
meta: body.fetch(config.key(KEY_META)),
|
24
|
+
data: body.fetch(config.key(KEY_DATA)),
|
25
|
+
totals: body[config.key(KEY_TOTALS)],
|
26
|
+
statistics: body[config.key(KEY_STATISTICS)],
|
27
|
+
rows_before_limit_at_least: body[config.key(KEY_ROWS_BEFORE_LIMIT_AT_LEAST)]
|
19
28
|
)
|
20
29
|
end
|
30
|
+
|
31
|
+
# @return [Response::Execution]
|
32
|
+
# @params faraday [Faraday::Response]
|
33
|
+
def self.exec(faraday)
|
34
|
+
Execution.new(headers: faraday.headers)
|
35
|
+
end
|
36
|
+
|
37
|
+
# @return [Response::Execution]
|
38
|
+
def self.empty_exec
|
39
|
+
Execution.new
|
40
|
+
end
|
21
41
|
end
|
22
42
|
end
|
23
43
|
end
|
@@ -6,27 +6,63 @@ module ClickHouse
|
|
6
6
|
extend Forwardable
|
7
7
|
include Enumerable
|
8
8
|
|
9
|
-
|
10
|
-
|
9
|
+
KEY_META_NAME = 'name'
|
10
|
+
KEY_META_TYPE = 'type'
|
11
11
|
|
12
12
|
def_delegators :to_a,
|
13
13
|
:inspect, :each, :fetch, :length, :count, :size,
|
14
14
|
:first, :last, :[], :to_h
|
15
15
|
|
16
|
-
attr_reader :meta, :data, :totals, :statistics, :rows_before_limit_at_least
|
16
|
+
attr_reader :config, :meta, :data, :totals, :statistics, :rows_before_limit_at_least
|
17
17
|
|
18
|
+
# @param config [Config]
|
18
19
|
# @param meta [Array]
|
19
20
|
# @param data [Array]
|
20
21
|
# @param totals [Array|Hash|NilClass] Support for 'GROUP BY WITH TOTALS' modifier
|
21
22
|
# https://clickhouse.tech/docs/en/sql-reference/statements/select/group-by/#with-totals-modifier
|
22
23
|
# Hash in JSON format and Array in JSONCompact
|
23
|
-
|
24
|
+
# rubocop:disable Metrics/ParameterLists
|
25
|
+
def initialize(config:, meta:, data:, totals: nil, statistics: nil, rows_before_limit_at_least: nil)
|
26
|
+
@config = config
|
24
27
|
@meta = meta
|
25
28
|
@data = data
|
26
29
|
@totals = totals
|
27
30
|
@rows_before_limit_at_least = rows_before_limit_at_least
|
28
31
|
@statistics = Hash(statistics)
|
29
32
|
end
|
33
|
+
# rubocop:enable Metrics/ParameterLists
|
34
|
+
|
35
|
+
# @return [Array, Hash]
|
36
|
+
# @param data [Array, Hash]
|
37
|
+
def serialize(data)
|
38
|
+
case data
|
39
|
+
when Hash
|
40
|
+
serialize_one(data)
|
41
|
+
when Array
|
42
|
+
data.map(&method(:serialize_one))
|
43
|
+
else
|
44
|
+
raise ArgumentError, "expect Hash or Array, got: #{data.class}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# @return [Hash]
|
49
|
+
# @param row [Hash]
|
50
|
+
def serialize_one(row)
|
51
|
+
row.each_with_object({}) do |(key, value), object|
|
52
|
+
object[key] = serialize_column(key, value)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# @param name [String] column name
|
57
|
+
# @param value [Any]
|
58
|
+
def serialize_column(name, value)
|
59
|
+
stmt = types.fetch(name)
|
60
|
+
serialize_type(stmt, value)
|
61
|
+
rescue KeyError => e
|
62
|
+
raise SerializeError, "field <#{name}> does not exists in table schema: #{types}", e.backtrace
|
63
|
+
rescue StandardError => e
|
64
|
+
raise SerializeError, "failed to serialize <#{name}> with #{stmt}, #{e.class}, #{e.message}", e.backtrace
|
65
|
+
end
|
30
66
|
|
31
67
|
def to_a
|
32
68
|
@to_a ||= data.each do |row|
|
@@ -39,8 +75,11 @@ module ClickHouse
|
|
39
75
|
# @return [Hash<String, Ast::Statement>]
|
40
76
|
def types
|
41
77
|
@types ||= meta.each_with_object({}) do |row, object|
|
42
|
-
|
43
|
-
|
78
|
+
column = row.fetch(config.key(KEY_META_NAME))
|
79
|
+
# make symbol keys, if config.symbolize_keys is true,
|
80
|
+
# to be able to cast and serialize properly
|
81
|
+
object[config.key(column)] = begin
|
82
|
+
current = Ast::Parser.new(row.fetch(config.key(KEY_META_TYPE))).parse
|
44
83
|
assign_type(current)
|
45
84
|
current
|
46
85
|
end
|
@@ -96,6 +135,42 @@ module ClickHouse
|
|
96
135
|
cast_type(stmt.arguments.fetch(ix), item)
|
97
136
|
end
|
98
137
|
end
|
138
|
+
|
139
|
+
# @param stmt [Ast::Statement]
|
140
|
+
def serialize_type(stmt, value)
|
141
|
+
return serialize_container(stmt, value) if stmt.caster.container?
|
142
|
+
return serialize_map(stmt, value) if stmt.caster.map?
|
143
|
+
return serialize_tuple(stmt, Array(value)) if stmt.caster.tuple?
|
144
|
+
|
145
|
+
stmt.caster.serialize(value, *stmt.arguments.map(&:value))
|
146
|
+
end
|
147
|
+
|
148
|
+
# @param stmt [Ast::Statement]
|
149
|
+
def serialize_container(stmt, value)
|
150
|
+
stmt.caster.cast_each(value) do |item|
|
151
|
+
# TODO: raise an error if multiple arguments
|
152
|
+
serialize_type(stmt.arguments.first, item)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
# @return [Hash]
|
157
|
+
# @param stmt [Ast::Statement]
|
158
|
+
# @param hash [Hash]
|
159
|
+
def serialize_map(stmt, hash)
|
160
|
+
raise ArgumentError, "expect hash got #{hash.class}" unless hash.is_a?(Hash)
|
161
|
+
|
162
|
+
key_type, value_type = stmt.arguments
|
163
|
+
hash.each_with_object({}) do |(key, value), object|
|
164
|
+
object[serialize_type(key_type, key)] = serialize_type(value_type, value)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
# @param stmt [Ast::Statement]
|
169
|
+
def serialize_tuple(stmt, value)
|
170
|
+
value.map.with_index do |item, ix|
|
171
|
+
serialize_type(stmt.arguments.fetch(ix), item)
|
172
|
+
end
|
173
|
+
end
|
99
174
|
end
|
100
175
|
end
|
101
176
|
end
|
data/lib/click_house/response.rb
CHANGED
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ClickHouse
|
4
|
+
module Serializer
|
5
|
+
class Base
|
6
|
+
attr_reader :config
|
7
|
+
|
8
|
+
# @param config [Config]
|
9
|
+
def initialize(config)
|
10
|
+
@config = config
|
11
|
+
on_setup
|
12
|
+
end
|
13
|
+
|
14
|
+
def dump(data)
|
15
|
+
raise NotImplementedError, __method__
|
16
|
+
end
|
17
|
+
|
18
|
+
# @return [String]
|
19
|
+
# @param data [Array]
|
20
|
+
def dump_each_row(data, sep = "\n")
|
21
|
+
data.map(&method(:dump)).join(sep)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# require external dependencies here
|
27
|
+
def on_setup
|
28
|
+
nil
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ClickHouse
|
4
|
+
module Serializer
|
5
|
+
class JsonOjSerializer < Base
|
6
|
+
def dump(data)
|
7
|
+
Oj.dump(data, config.oj_dump_options)
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def on_setup
|
13
|
+
require 'oj' unless defined?(Oj)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ClickHouse
|
4
|
+
module Serializer
|
5
|
+
autoload :Base, 'click_house/serializer/base'
|
6
|
+
autoload :JsonSerializer, 'click_house/serializer/json_serializer'
|
7
|
+
autoload :JsonOjSerializer, 'click_house/serializer/json_oj_serializer'
|
8
|
+
end
|
9
|
+
end
|
@@ -11,6 +11,10 @@ module ClickHouse
|
|
11
11
|
raise NotImplementedError, __method__
|
12
12
|
end
|
13
13
|
|
14
|
+
def serialize_each(_value, *)
|
15
|
+
raise NotImplementedError, __method__
|
16
|
+
end
|
17
|
+
|
14
18
|
# @return [Boolean]
|
15
19
|
# true if type contains another type like Nullable(T) or Array(T)
|
16
20
|
def container?
|
@@ -3,14 +3,14 @@
|
|
3
3
|
module ClickHouse
|
4
4
|
module Type
|
5
5
|
class DecimalType < BaseType
|
6
|
-
MAXIMUM = Float::DIG
|
6
|
+
MAXIMUM = Float::DIG.next
|
7
7
|
|
8
8
|
# clickhouse:
|
9
9
|
# P - precision. Valid range: [ 1 : 76 ]. Determines how many decimal digits number can have (including fraction).
|
10
10
|
# S - scale. Valid range: [ 0 : P ]. Determines how many decimal digits fraction can have.
|
11
11
|
#
|
12
12
|
# when Oj parser @refs https://stackoverflow.com/questions/47885304/deserialise-json-numbers-as-bigdecimal
|
13
|
-
def cast(value, precision =
|
13
|
+
def cast(value, precision = MAXIMUM, _scale = nil)
|
14
14
|
case value
|
15
15
|
when BigDecimal
|
16
16
|
value
|
@@ -21,8 +21,9 @@ module ClickHouse
|
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
|
-
|
25
|
-
|
24
|
+
# @return [BigDecimal]
|
25
|
+
def serialize(value, precision = MAXIMUM, _scale = nil)
|
26
|
+
cast(value, precision)
|
26
27
|
end
|
27
28
|
end
|
28
29
|
end
|
data/lib/click_house/version.rb
CHANGED
data/lib/click_house.rb
CHANGED
@@ -12,6 +12,7 @@ require 'active_support/core_ext/time/calculations'
|
|
12
12
|
require 'click_house/version'
|
13
13
|
require 'click_house/errors'
|
14
14
|
require 'click_house/response'
|
15
|
+
require 'click_house/serializer'
|
15
16
|
require 'click_house/type'
|
16
17
|
require 'click_house/middleware'
|
17
18
|
require 'click_house/extend'
|
@@ -33,6 +34,10 @@ module ClickHouse
|
|
33
34
|
add_type 'LowCardinality', Type::LowCardinalityType.new
|
34
35
|
add_type 'Tuple', Type::TupleType.new
|
35
36
|
|
37
|
+
%w[Bool].each do |column|
|
38
|
+
add_type column, Type::BooleanType.new
|
39
|
+
end
|
40
|
+
|
36
41
|
%w[Date].each do |column|
|
37
42
|
add_type column, Type::DateType.new
|
38
43
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: click_house
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aliaksandr Shylau
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-11-
|
11
|
+
date: 2022-11-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -175,6 +175,7 @@ files:
|
|
175
175
|
- lib/click_house/ast/parser.rb
|
176
176
|
- lib/click_house/ast/statement.rb
|
177
177
|
- lib/click_house/ast/ticker.rb
|
178
|
+
- lib/click_house/benchmark/map_join.rb
|
178
179
|
- lib/click_house/config.rb
|
179
180
|
- lib/click_house/connection.rb
|
180
181
|
- lib/click_house/definition.rb
|
@@ -200,8 +201,13 @@ files:
|
|
200
201
|
- lib/click_house/middleware/raise_error.rb
|
201
202
|
- lib/click_house/middleware/response_base.rb
|
202
203
|
- lib/click_house/response.rb
|
204
|
+
- lib/click_house/response/execution.rb
|
203
205
|
- lib/click_house/response/factory.rb
|
204
206
|
- lib/click_house/response/result_set.rb
|
207
|
+
- lib/click_house/serializer.rb
|
208
|
+
- lib/click_house/serializer/base.rb
|
209
|
+
- lib/click_house/serializer/json_oj_serializer.rb
|
210
|
+
- lib/click_house/serializer/json_serializer.rb
|
205
211
|
- lib/click_house/type.rb
|
206
212
|
- lib/click_house/type/array_type.rb
|
207
213
|
- lib/click_house/type/base_type.rb
|