clickhouse-sql 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +9 -0
- data/LICENSE +25 -0
- data/README.md +426 -0
- data/lib/clickhouse/http_client/database.rb +42 -0
- data/lib/clickhouse/http_client/error.rb +18 -0
- data/lib/clickhouse/http_client/parameter_serializer.rb +127 -0
- data/lib/clickhouse/http_client/query.rb +54 -0
- data/lib/clickhouse/http_client/response_formatter.rb +78 -0
- data/lib/clickhouse/http_client/value_normalization.rb +29 -0
- data/lib/clickhouse/http_client.rb +288 -0
- data/lib/clickhouse/sql/bind_index_manager.rb +26 -0
- data/lib/clickhouse/sql/error.rb +13 -0
- data/lib/clickhouse/sql/fragment.rb +171 -0
- data/lib/clickhouse/sql/join.rb +70 -0
- data/lib/clickhouse/sql/part.rb +37 -0
- data/lib/clickhouse/sql/placeholder.rb +41 -0
- data/lib/clickhouse/sql/placeholder_type.rb +219 -0
- data/lib/clickhouse/sql/query.rb +139 -0
- data/lib/clickhouse/sql/raw.rb +53 -0
- data/lib/clickhouse/sql/select_item.rb +48 -0
- data/lib/clickhouse/sql/select_query.rb +393 -0
- data/lib/clickhouse/sql/setting.rb +89 -0
- data/lib/clickhouse/sql/table.rb +97 -0
- data/lib/clickhouse/sql/version.rb +7 -0
- data/lib/clickhouse/sql.rb +562 -0
- data/lib/clickhouse-sql.rb +5 -0
- metadata +133 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 75be76b3e8d20fe8c8d7471d37ccf6e4b3385491a354eab01e4147448cbcd77f
|
|
4
|
+
data.tar.gz: 4c2271359a308db966129ead49e7134112ffef8cca9ad95a69e49079dde733fd
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 282220375980be2f0dc6e7011ba10fea2bb0d7926ad4a87f6268d24a285948a3aa49389aa6657b26348c6ac5753236acf62810ffe54980292865b37938a6cce3
|
|
7
|
+
data.tar.gz: 9939de13d3641ebaf3631e8928e300037aea959ac1dd95706ec620fb59e0e3c5fac8dc92c7d3d152651f46313c22200a1748a41fc5a69f6b768a94106ad5550f
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 - Unreleased
|
|
4
|
+
|
|
5
|
+
- Extract `ClickHouse::SQL` and `ClickHouse::HTTPClient` from
|
|
6
|
+
`github.com/buildkite/buildkite` at
|
|
7
|
+
`feb7aaff975b2f90d0e33a69331b105112ff8fbf`.
|
|
8
|
+
- Support standalone use on Ruby 3.4 and later with no runtime gem
|
|
9
|
+
dependencies.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Buildkite Pty Ltd
|
|
4
|
+
|
|
5
|
+
Portions derived from GitLab's MIT-licensed click_house-client gem
|
|
6
|
+
(https://gitlab.com/gitlab-org/gitlab/-/tree/e9d4efda854653515bf3aab0e8c2441976f83c9f/gems/click_house-client):
|
|
7
|
+
Copyright (c) 2011-present GitLab B.V.
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
# clickhouse-sql
|
|
2
|
+
|
|
3
|
+
`clickhouse-sql` provides a typed `ClickHouse::SQL` query builder and the
|
|
4
|
+
`ClickHouse::HTTPClient` transport for ClickHouse. It keeps SQL visible while
|
|
5
|
+
binding runtime values through ClickHouse named typed placeholders.
|
|
6
|
+
|
|
7
|
+
The gem supports Ruby 3.4 and later and has no runtime gem dependencies.
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
Add the gem to your bundle:
|
|
12
|
+
|
|
13
|
+
```ruby
|
|
14
|
+
gem "clickhouse-sql"
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Then run `bundle install` and load both public components:
|
|
18
|
+
|
|
19
|
+
```ruby
|
|
20
|
+
require "clickhouse-sql"
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Applications that need only one component can load it independently:
|
|
24
|
+
|
|
25
|
+
```ruby
|
|
26
|
+
require "clickhouse/sql"
|
|
27
|
+
# or
|
|
28
|
+
require "clickhouse/http_client"
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The SQL builder and HTTP client interoperate through query objects responding
|
|
32
|
+
to `#to_sql` and `#placeholders`; neither component requires a framework.
|
|
33
|
+
|
|
34
|
+
## Security and trust model
|
|
35
|
+
|
|
36
|
+
**All runtime or untrusted values belong in named typed placeholders.** Plain
|
|
37
|
+
SQL strings, fragment slots, table names, `raw_static`, and `unsafe_raw` are
|
|
38
|
+
trusted SQL structure. Never interpolate request parameters, user input, or
|
|
39
|
+
other untrusted values into them.
|
|
40
|
+
|
|
41
|
+
```ruby
|
|
42
|
+
# Safe: the value is serialized separately as param_account_id.
|
|
43
|
+
query.where("{} = {account_id:UUID}", events[:account_id], account_id: account_id)
|
|
44
|
+
|
|
45
|
+
# Unsafe: account_id becomes executable SQL structure.
|
|
46
|
+
query.where("account_id = '#{account_id}'")
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
`raw_static` is for compile-time or allowlisted structure. `unsafe_raw` requires
|
|
50
|
+
a reason to make exceptional use reviewable, but the reason is advisory: the
|
|
51
|
+
gem does not inspect or sanitize the supplied SQL. Query-level settings are
|
|
52
|
+
rendered into SQL rather than sent as placeholders, so setting names and values
|
|
53
|
+
may appear in logs or instrumentation.
|
|
54
|
+
|
|
55
|
+
An opted-in instrumenter receives the full query object, including its SQL text
|
|
56
|
+
and raw placeholder values. Treat instrumentation backends and subscribers as
|
|
57
|
+
sensitive-data sinks, and do not attach an instrumenter that cannot safely
|
|
58
|
+
handle the values your application queries.
|
|
59
|
+
|
|
60
|
+
Outside local development, use a TLS (`https://`) ClickHouse endpoint and a
|
|
61
|
+
least-privilege ClickHouse user restricted to the databases, tables, and query
|
|
62
|
+
operations the application needs.
|
|
63
|
+
|
|
64
|
+
## HTTPClient
|
|
65
|
+
|
|
66
|
+
`ClickHouse::HTTPClient` registers ClickHouse databases and executes queries
|
|
67
|
+
over HTTP. It exposes `select`, `execute`, and `insert_csv`; each accepts an
|
|
68
|
+
optional per-query `timeout:`.
|
|
69
|
+
|
|
70
|
+
### Database registration
|
|
71
|
+
|
|
72
|
+
Register connections during application initialization. Registry names are
|
|
73
|
+
process-global, and duplicate names raise
|
|
74
|
+
`ClickHouse::HTTPClient::ConfigurationError`.
|
|
75
|
+
|
|
76
|
+
```ruby
|
|
77
|
+
require "clickhouse-sql"
|
|
78
|
+
require "logger"
|
|
79
|
+
|
|
80
|
+
ClickHouse::HTTPClient.logger = Logger.new($stdout)
|
|
81
|
+
ClickHouse::HTTPClient.register_database(
|
|
82
|
+
:analytics,
|
|
83
|
+
database: "default",
|
|
84
|
+
url: ENV.fetch("CLICKHOUSE_URL", "http://localhost:8123"),
|
|
85
|
+
username: ENV.fetch("CLICKHOUSE_USERNAME", "default"),
|
|
86
|
+
password: ENV.fetch("CLICKHOUSE_PASSWORD", ""),
|
|
87
|
+
variables: { max_execution_time: 30 },
|
|
88
|
+
)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
The client uses a two-second connection-open timeout and a 30-second request
|
|
92
|
+
timeout by default. A query-specific `timeout:` replaces the request timeout:
|
|
93
|
+
|
|
94
|
+
```ruby
|
|
95
|
+
rows = ClickHouse::HTTPClient.select("SELECT 1 AS value", :analytics, timeout: 5)
|
|
96
|
+
# => [{ "value" => 1 }]
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Plain query strings are accepted as trusted SQL. To bind values without the
|
|
100
|
+
SQL builder, use the transport query object:
|
|
101
|
+
|
|
102
|
+
```ruby
|
|
103
|
+
query = ClickHouse::HTTPClient::Query.new(
|
|
104
|
+
sql: "SELECT {id:UInt64} AS id",
|
|
105
|
+
placeholders: { id: 42 },
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
rows = ClickHouse::HTTPClient.select(query, :analytics)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Passing a `ClickHouse::SQL` builder directly compiles it through `#to_query`,
|
|
112
|
+
so builder validation still runs:
|
|
113
|
+
|
|
114
|
+
```ruby
|
|
115
|
+
query = ClickHouse::SQL
|
|
116
|
+
.select("id", "name")
|
|
117
|
+
.from("events")
|
|
118
|
+
.where("id = {id:UInt64}", id: 42)
|
|
119
|
+
|
|
120
|
+
rows = ClickHouse::HTTPClient.select(query, :analytics)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`select` returns an array of string-keyed hashes. It typecasts integer and
|
|
124
|
+
float widths, `Date`/`Date32`, UTC `DateTime`/`DateTime64`,
|
|
125
|
+
`IntervalSecond`/`IntervalMillisecond`, and
|
|
126
|
+
arrays of supported types. UTC datetimes become UTC Ruby `Time` values;
|
|
127
|
+
`IntervalSecond` is integer seconds and `IntervalMillisecond` is floating-point
|
|
128
|
+
seconds. Non-UTC datetimes and unrecognized types, including `Map` and `Tuple`
|
|
129
|
+
internals, pass through as parsed from JSON. Zone-less `DateTime` values are
|
|
130
|
+
treated as UTC, so the ClickHouse server should use UTC as its default timezone.
|
|
131
|
+
|
|
132
|
+
### Commands and CSV insertion
|
|
133
|
+
|
|
134
|
+
`execute` and `insert_csv` return an immutable `ExecutionResult`. Failures
|
|
135
|
+
raise rather than returning a failed result.
|
|
136
|
+
|
|
137
|
+
```ruby
|
|
138
|
+
result = ClickHouse::HTTPClient.execute(
|
|
139
|
+
"CREATE TABLE IF NOT EXISTS events (id UInt64) ENGINE = MergeTree ORDER BY id",
|
|
140
|
+
:analytics,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
result.query_id # server query ID, or nil if the server omitted it
|
|
144
|
+
result.statistics # symbol-keyed final summary, or {}
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
`insert_csv` streams a gzip-compressed IO as a chunked request. The SQL must not
|
|
148
|
+
contain placeholders because the request body is occupied by CSV data.
|
|
149
|
+
|
|
150
|
+
```ruby
|
|
151
|
+
require "stringio"
|
|
152
|
+
require "zlib"
|
|
153
|
+
|
|
154
|
+
compressed = StringIO.new
|
|
155
|
+
gzip = Zlib::GzipWriter.new(compressed)
|
|
156
|
+
gzip.write("1,created\n2,updated\n")
|
|
157
|
+
gzip.finish
|
|
158
|
+
compressed.rewind
|
|
159
|
+
|
|
160
|
+
result = ClickHouse::HTTPClient.insert_csv(
|
|
161
|
+
"INSERT INTO events (id, name) FORMAT CSV",
|
|
162
|
+
compressed,
|
|
163
|
+
:analytics,
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
result.statistics.fetch(:written_rows)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Errors
|
|
170
|
+
|
|
171
|
+
Failures raise subclasses of `ClickHouse::HTTPClient::Error`:
|
|
172
|
+
|
|
173
|
+
- `DatabaseError` for ClickHouse error responses;
|
|
174
|
+
- `ConnectionError` for connection and protocol failures;
|
|
175
|
+
- `TimeoutError` for client-side request timeouts;
|
|
176
|
+
- `QueryError` for query-protocol misuse; and
|
|
177
|
+
- `ConfigurationError` for database registry problems.
|
|
178
|
+
|
|
179
|
+
Transport errors retain the underlying standard-library exception as their
|
|
180
|
+
cause. A server-side `max_execution_time` kill is a `DatabaseError` mentioning
|
|
181
|
+
`TIMEOUT_EXCEEDED`, not a client-side `TimeoutError`.
|
|
182
|
+
|
|
183
|
+
`nil` placeholder values are sent as `NULL`. ClickHouse rejects `NULL` for most
|
|
184
|
+
non-Nullable types but can parse it as an empty string for a plain `String`
|
|
185
|
+
placeholder. `ClickHouse::SQL` validates nil values against placeholder types
|
|
186
|
+
before execution to close that gap.
|
|
187
|
+
|
|
188
|
+
### Logging and instrumentation
|
|
189
|
+
|
|
190
|
+
Set any logger responding to `#debug`:
|
|
191
|
+
|
|
192
|
+
```ruby
|
|
193
|
+
ClickHouse::HTTPClient.logger = Logger.new($stdout)
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Instrumentation is opt-in. Supply an object implementing
|
|
197
|
+
`instrument(name, payload) { |mutable_payload| ... }`. It must yield exactly
|
|
198
|
+
once and propagate exceptions. The payload initially contains `:query` and
|
|
199
|
+
`:database`; the client adds `:query_id` and `:statistics` as they become
|
|
200
|
+
available.
|
|
201
|
+
|
|
202
|
+
```ruby
|
|
203
|
+
instrumenter = Object.new
|
|
204
|
+
def instrumenter.instrument(name, payload)
|
|
205
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
206
|
+
yield payload
|
|
207
|
+
ensure
|
|
208
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
|
|
209
|
+
warn({ event: name, elapsed: elapsed, payload: payload }.inspect)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
ClickHouse::HTTPClient.instrumenter = instrumenter
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The default instrumenter is a no-op. Framework integrations belong in the host
|
|
216
|
+
application; for example, an application that already uses Active Support may
|
|
217
|
+
assign `ActiveSupport::Notifications` because it implements this protocol.
|
|
218
|
+
Remember that the payload exposes raw placeholder values as described in the
|
|
219
|
+
security section.
|
|
220
|
+
|
|
221
|
+
## SQL
|
|
222
|
+
|
|
223
|
+
`ClickHouse::SQL` is a small, clause-oriented builder for ClickHouse `SELECT`
|
|
224
|
+
queries. It builds mutable SQL objects that compose safely, then compiles them
|
|
225
|
+
with `#to_query` into immutable HTTP-client-compatible query objects. It is not
|
|
226
|
+
an ORM: callers still choose exact select lists and map returned rows into
|
|
227
|
+
their own objects.
|
|
228
|
+
|
|
229
|
+
### Quick example
|
|
230
|
+
|
|
231
|
+
```ruby
|
|
232
|
+
CH = ClickHouse::SQL
|
|
233
|
+
|
|
234
|
+
events = CH.table("events")
|
|
235
|
+
accounts = CH.table("accounts", as: "a")
|
|
236
|
+
|
|
237
|
+
query = CH
|
|
238
|
+
.select(events.columns(:id, :timestamp, :duration), accounts[:status])
|
|
239
|
+
.from(events)
|
|
240
|
+
.left_join(accounts, final: true, on: [
|
|
241
|
+
CH.fragment("{} = {}", accounts[:id], events[:account_id]),
|
|
242
|
+
])
|
|
243
|
+
.where("{} = {account_id:UUID}", events[:account_id], account_id: account_id)
|
|
244
|
+
.where("{} IN {categories:Array(String)}", events[:category], categories: categories)
|
|
245
|
+
.where("{}.{tag_key:Identifier} = {tag_value:String}", events[:tags], tag_key: "geo.country", tag_value: "AU")
|
|
246
|
+
.order_by("timestamp DESC")
|
|
247
|
+
.limit("{limit:UInt32}", limit: 100)
|
|
248
|
+
|
|
249
|
+
rows = ClickHouse::HTTPClient.select(query, :analytics)
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
Table handles build qualified column references. Passing the aliased
|
|
253
|
+
`accounts` handle to `left_join` renders `LEFT JOIN accounts FINAL AS a`, and
|
|
254
|
+
every `accounts[...]` reference renders with the `a` qualifier. Plain SQL such
|
|
255
|
+
as `order_by("timestamp DESC")` is accepted as trusted structure where no
|
|
256
|
+
qualification is needed.
|
|
257
|
+
|
|
258
|
+
### Typed placeholders
|
|
259
|
+
|
|
260
|
+
Use ClickHouse named typed placeholders for values known only at runtime:
|
|
261
|
+
|
|
262
|
+
```ruby
|
|
263
|
+
CH.fragment("account_id = {account_id:UUID}", account_id: account_id)
|
|
264
|
+
CH.fragment("account_id IN {account_ids:Array(UUID)}", account_ids: account_ids)
|
|
265
|
+
CH.fragment("timestamp >= {from:DateTime64(6)}", from: from_time)
|
|
266
|
+
CH.fragment("tags.{tag_key:Identifier} = {value:String}", tag_key: "geo.country", value: "AU")
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
The declared ClickHouse type is the source of truth. The builder validates
|
|
270
|
+
supported types such as `UUID`, `Array(T)`, `Nullable(T)`, `Identifier`, `Date`,
|
|
271
|
+
`DateTime`, `DateTime64(N)`, signed and unsigned integers, and floats before
|
|
272
|
+
the client sends the query.
|
|
273
|
+
|
|
274
|
+
### Fragment slots
|
|
275
|
+
|
|
276
|
+
Inside a fragment, `{name:Type}` is a ClickHouse value placeholder. Untyped
|
|
277
|
+
braces are local slots for trusted SQL objects. Anonymous `{}` slots are filled
|
|
278
|
+
in source order by positional arguments:
|
|
279
|
+
|
|
280
|
+
```ruby
|
|
281
|
+
CH.fragment("{} = {}", accounts[:id], events[:account_id])
|
|
282
|
+
CH.fragment("{} = {account_id:UUID}", events[:account_id], account_id: account_id)
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
Named `{name}` slots are filled by keyword argument:
|
|
286
|
+
|
|
287
|
+
```ruby
|
|
288
|
+
tag_column = CH.raw_static("events.tags")
|
|
289
|
+
|
|
290
|
+
CH.fragment(
|
|
291
|
+
"{tag_column}.{tag_key:Identifier}::Nullable(String)",
|
|
292
|
+
tag_column: tag_column,
|
|
293
|
+
tag_key: "triage.reason",
|
|
294
|
+
)
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
Fragment slots accept SQL fragments, raw/static fragments, nested builders,
|
|
298
|
+
table/column handles, and compiled SQL queries. They do not accept arbitrary
|
|
299
|
+
runtime strings or unrelated query objects.
|
|
300
|
+
|
|
301
|
+
### Query composition
|
|
302
|
+
|
|
303
|
+
`ClickHouse::SQL.select` returns a mutable, chainable `SelectQuery`. Its clause
|
|
304
|
+
methods accept trusted SQL strings or fragments, positional fills for anonymous
|
|
305
|
+
slots, keyword values for typed placeholders, or arrays of fragments:
|
|
306
|
+
|
|
307
|
+
- `with` and `with_cte`;
|
|
308
|
+
- `from` and `from_subquery`;
|
|
309
|
+
- `join`, `left_join`, `inner_join`, and `joins`;
|
|
310
|
+
- `where`, `having`, `group_by`, and `order_by`;
|
|
311
|
+
- `limit` and `offset`; and
|
|
312
|
+
- `settings` and `setting`.
|
|
313
|
+
|
|
314
|
+
Repeated `where`/`having` conditions and multiple join `on:` conditions are
|
|
315
|
+
`AND`-joined with every condition parenthesized. A condition containing a
|
|
316
|
+
top-level `OR` therefore cannot change the meaning of adjacent conditions.
|
|
317
|
+
`CH.and` and `CH.or` provide the same parenthesized composition, while `CH.csv`
|
|
318
|
+
and `CH.parens` handle small expression lists.
|
|
319
|
+
|
|
320
|
+
`CH.bind_list` generates one typed placeholder per item when a single
|
|
321
|
+
`Array(T)` parameter could exceed ClickHouse's per-field size limit
|
|
322
|
+
(`http_max_field_value_size`, 128 KiB by default):
|
|
323
|
+
|
|
324
|
+
```ruby
|
|
325
|
+
ids = CH.bind_list(:account_id, account_ids, type: "UUID")
|
|
326
|
+
query.where(CH.fragment("{} IN ({})", events[:account_id], ids))
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
### Table handles, CTEs, and derived tables
|
|
330
|
+
|
|
331
|
+
Table handles validate table names and qualify columns, preventing ambiguity
|
|
332
|
+
when joined tables share column names:
|
|
333
|
+
|
|
334
|
+
```ruby
|
|
335
|
+
events = CH.table("events")
|
|
336
|
+
accounts = CH.table("accounts", as: "a")
|
|
337
|
+
|
|
338
|
+
CH.select(events[:id], accounts[:name])
|
|
339
|
+
.from(events)
|
|
340
|
+
.left_join(accounts, on: [
|
|
341
|
+
CH.fragment("{} = {}", accounts[:id], events[:account_id]),
|
|
342
|
+
])
|
|
343
|
+
.where("{} >= {from:DateTime64(6)}", events[:timestamp], from: from_time)
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
Column references use the alias when present (`a.name`) and otherwise the table
|
|
347
|
+
name (`events.id`). Names interpolate into raw SQL, so each dot-separated
|
|
348
|
+
segment must be a simple identifier. Hyphenated names such as tag keys belong
|
|
349
|
+
in `{name:Identifier}` placeholders instead. Nested paths such as
|
|
350
|
+
`events["tags.client.version"]` are supported.
|
|
351
|
+
|
|
352
|
+
A handle owns its alias. Passing `as:` alongside a handle raises, even if the
|
|
353
|
+
aliases match. `as:` on `from`/`join` is only for string table names.
|
|
354
|
+
|
|
355
|
+
Handles also keep CTE and derived-table references consistent:
|
|
356
|
+
|
|
357
|
+
```ruby
|
|
358
|
+
filtered = CH.table("filtered")
|
|
359
|
+
filtered_query = CH.select("id", "timestamp").from("events")
|
|
360
|
+
|
|
361
|
+
with_cte = CH.select(filtered[:id])
|
|
362
|
+
.with_cte(filtered, filtered_query)
|
|
363
|
+
.from(filtered)
|
|
364
|
+
|
|
365
|
+
from_subquery = CH.select(filtered[:id])
|
|
366
|
+
.from_subquery(filtered_query, as: filtered)
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
In fragment slots, a table handle renders as its table name, not its qualifier.
|
|
370
|
+
Use `table[:column]` or `table.qualifier` inside expressions rather than
|
|
371
|
+
`{table}.column`.
|
|
372
|
+
|
|
373
|
+
### Settings
|
|
374
|
+
|
|
375
|
+
ClickHouse does not parameterize query-level settings. Use `setting` for a
|
|
376
|
+
validated name and value, or `settings` only for fixed, trusted fragments:
|
|
377
|
+
|
|
378
|
+
```ruby
|
|
379
|
+
query
|
|
380
|
+
.setting(:max_threads, 4)
|
|
381
|
+
.setting(:use_query_cache, false)
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
String setting values are escaped as ClickHouse string literals, but they are
|
|
385
|
+
still rendered into the query. Do not put secrets or untrusted structure in
|
|
386
|
+
settings; their values may be visible in SQL logs and instrumenter payloads.
|
|
387
|
+
|
|
388
|
+
### Trusted and unsafe raw SQL
|
|
389
|
+
|
|
390
|
+
`CH.raw_static(sql)` is for compile-time or allowlisted SQL structure such as
|
|
391
|
+
`*`, fixed table names, fixed expressions, or fixed sort directions. It does
|
|
392
|
+
not collect placeholder values.
|
|
393
|
+
|
|
394
|
+
`CH.unsafe_raw(sql, reason:)` is an explicit escape hatch for trusted ClickHouse
|
|
395
|
+
syntax that cannot be represented with placeholders, such as a fixed
|
|
396
|
+
`INTERVAL` literal in `WITH FILL`. Always provide a durable reason, and never
|
|
397
|
+
use this method to bypass value binding.
|
|
398
|
+
|
|
399
|
+
### Known parser limitations
|
|
400
|
+
|
|
401
|
+
Fragment parsing treats every `{...}` in fragment SQL as a placeholder; it is
|
|
402
|
+
not aware of SQL string literals or comments. SQL containing literal braces —
|
|
403
|
+
for example `match(col, 'a{2,3}')` or `{'a': 1}` — raises
|
|
404
|
+
`ClickHouse::SQL::Error` (or treats a brace-wrapped bare identifier as a
|
|
405
|
+
fragment slot). Wrap a fixed, trusted expression containing literal braces in
|
|
406
|
+
`raw_static` or `unsafe_raw` so it is not parsed for placeholders.
|
|
407
|
+
|
|
408
|
+
Compact `CH.bind_list` names use `"#{prefix}#{index.to_s(36)}"` without a
|
|
409
|
+
separator. Two lists whose prefixes overlap can generate the same name (`:t`
|
|
410
|
+
at index 36 and `:t1` at index 0 both produce `t10`). A collision raises when
|
|
411
|
+
values differ but coalesces when they are equal. Within one query, use prefixes
|
|
412
|
+
that do not end in digits and are not prefixes of one another.
|
|
413
|
+
|
|
414
|
+
## Package contents
|
|
415
|
+
|
|
416
|
+
The published gem is intended to contain only Ruby files under `lib/` plus
|
|
417
|
+
`README.md`, `LICENSE`, and `CHANGELOG.md`. Development tools and tests are not
|
|
418
|
+
runtime dependencies or package contents.
|
|
419
|
+
|
|
420
|
+
## Attribution
|
|
421
|
+
|
|
422
|
+
Parts of `ClickHouse::HTTPClient` are derived from GitLab's MIT-licensed
|
|
423
|
+
[click_house-client](https://gitlab.com/gitlab-org/gitlab/-/tree/e9d4efda854653515bf3aab0e8c2441976f83c9f/gems/click_house-client)
|
|
424
|
+
gem — thanks to GitLab for publishing it under a permissive license. See
|
|
425
|
+
[LICENSE](LICENSE) for the full notice. The `ClickHouse::SQL` query builder is
|
|
426
|
+
original to this gem.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module HTTPClient
|
|
5
|
+
# Connection settings and headers for one registered ClickHouse database.
|
|
6
|
+
Database = Data.define(:url, :variables, :headers) do
|
|
7
|
+
def self.build(database:, url:, username:, password:, variables: {})
|
|
8
|
+
new(
|
|
9
|
+
url: url,
|
|
10
|
+
variables: {
|
|
11
|
+
database: database,
|
|
12
|
+
enable_http_compression: 1,
|
|
13
|
+
# Buffer each response server-side before sending: without this,
|
|
14
|
+
# ClickHouse commits HTTP 200 once it starts streaming, and an
|
|
15
|
+
# error later in query execution (memory limit, server-side
|
|
16
|
+
# timeout) is appended to the already-started body rather than
|
|
17
|
+
# producing an error status.
|
|
18
|
+
wait_end_of_query: 1,
|
|
19
|
+
}.merge(variables).freeze,
|
|
20
|
+
headers: {
|
|
21
|
+
"X-ClickHouse-User" => username,
|
|
22
|
+
"X-ClickHouse-Key" => password,
|
|
23
|
+
"X-ClickHouse-Format" => "JSON", # always return JSON data
|
|
24
|
+
}.freeze,
|
|
25
|
+
)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# @return [String] Physical ClickHouse database name.
|
|
29
|
+
def database_name
|
|
30
|
+
variables.fetch(:database)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @param extra_variables [Hash] Additional URL query parameters.
|
|
34
|
+
# @return [URI::HTTP] Endpoint URI for this database.
|
|
35
|
+
def uri(extra_variables = {})
|
|
36
|
+
URI.parse(url).tap do |uri|
|
|
37
|
+
uri.query = URI.encode_www_form(variables.merge(extra_variables))
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module HTTPClient
|
|
5
|
+
Error = Class.new(StandardError)
|
|
6
|
+
ConfigurationError = Class.new(Error)
|
|
7
|
+
DatabaseError = Class.new(Error)
|
|
8
|
+
QueryError = Class.new(Error)
|
|
9
|
+
|
|
10
|
+
# Transport failures are re-raised as these wrappers (with the original
|
|
11
|
+
# Net::HTTP error as the cause) so callers rescue one hierarchy without
|
|
12
|
+
# knowing the underlying HTTP library. TimeoutError is the client giving
|
|
13
|
+
# up waiting; a server-side kill (max_execution_time) surfaces as a
|
|
14
|
+
# DatabaseError mentioning TIMEOUT_EXCEEDED instead.
|
|
15
|
+
ConnectionError = Class.new(Error)
|
|
16
|
+
TimeoutError = Class.new(Error)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module HTTPClient
|
|
5
|
+
# Serializes Ruby placeholder values into the text format ClickHouse
|
|
6
|
+
# expects for query parameters, for binding values into SQL.
|
|
7
|
+
#
|
|
8
|
+
# ClickHouse parses a top-level parameter value in its escaped text
|
|
9
|
+
# format, where an unquoted `\N` denotes NULL, while values nested inside
|
|
10
|
+
# Array/Map literals are parsed as SQL-like literals, where NULL is
|
|
11
|
+
# spelled `NULL` and strings must be quoted.
|
|
12
|
+
#
|
|
13
|
+
# - https://clickhouse.com/docs/sql-reference/syntax#literals
|
|
14
|
+
# - https://clickhouse.com/docs/sql-reference/syntax#defining-and-using-query-parameters
|
|
15
|
+
module ParameterSerializer
|
|
16
|
+
# Serializes a value for the top/outer/root layer, where e.g. strings
|
|
17
|
+
# don't need to be surrounded by quotes (whereas e.g. strings in an
|
|
18
|
+
# Array do need quotes).
|
|
19
|
+
#
|
|
20
|
+
# @param input [Object] Placeholder value to serialize.
|
|
21
|
+
# @return [String] ClickHouse parameter text.
|
|
22
|
+
def self.serialize(input)
|
|
23
|
+
case input
|
|
24
|
+
|
|
25
|
+
when NilClass
|
|
26
|
+
"\\N"
|
|
27
|
+
|
|
28
|
+
when String
|
|
29
|
+
escape_top_level_string(input)
|
|
30
|
+
|
|
31
|
+
when Symbol
|
|
32
|
+
escape_top_level_string(input.to_s)
|
|
33
|
+
|
|
34
|
+
when Array
|
|
35
|
+
elements = input.map { serialize_nested(_1) }
|
|
36
|
+
"[#{elements.join(",")}]"
|
|
37
|
+
|
|
38
|
+
when Hash
|
|
39
|
+
elements = input.map { |k, v| serialize_nested(k) + ":" + serialize_nested(v) }
|
|
40
|
+
"{#{elements.join(",")}}"
|
|
41
|
+
|
|
42
|
+
when DateTime
|
|
43
|
+
# DateTime is a Date subclass, so it must be matched first and
|
|
44
|
+
# converted to a Time (preserving its offset) to render as UTC
|
|
45
|
+
# wall-clock text like Time below, not as a date-only string.
|
|
46
|
+
serialize(input.to_time)
|
|
47
|
+
|
|
48
|
+
when Date
|
|
49
|
+
input.to_s # Ruby and ClickHouse agree on YYYY-MM-DD
|
|
50
|
+
|
|
51
|
+
when Time
|
|
52
|
+
# ClickHouse interprets the rendered text in the placeholder's (or
|
|
53
|
+
# server's) timezone, so only UTC wall-clock text is safe to send.
|
|
54
|
+
# Sub-second precision is dropped; when precision matters, callers
|
|
55
|
+
# pre-format the value as a String (as ClickHouse::SQL does for
|
|
56
|
+
# DateTime64 placeholders).
|
|
57
|
+
input.getutc.strftime("%Y-%m-%d %H:%M:%S")
|
|
58
|
+
|
|
59
|
+
else
|
|
60
|
+
input.to_s
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Serializes a value nested within a container e.g. an Array or
|
|
65
|
+
# Hash/Map, where strings are surrounded by quotes and NULL is spelled
|
|
66
|
+
# out.
|
|
67
|
+
#
|
|
68
|
+
# @param input [Object] Nested placeholder value to serialize.
|
|
69
|
+
# @return [String] ClickHouse literal text.
|
|
70
|
+
def self.serialize_nested(input)
|
|
71
|
+
case input
|
|
72
|
+
|
|
73
|
+
when NilClass
|
|
74
|
+
"NULL"
|
|
75
|
+
|
|
76
|
+
when Symbol
|
|
77
|
+
serialize_nested(input.to_s)
|
|
78
|
+
|
|
79
|
+
when String
|
|
80
|
+
quote_and_escape_string(input)
|
|
81
|
+
|
|
82
|
+
when Date, Time
|
|
83
|
+
quote_and_escape_string(serialize(input))
|
|
84
|
+
|
|
85
|
+
else
|
|
86
|
+
serialize(input)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# ClickHouse parses each top-level parameter value with the type's
|
|
91
|
+
# escaped-text deserialization: backslash sequences (\t, \n, \N, \xNN,
|
|
92
|
+
# ...) are decoded, and a literal TAB or LF byte terminates the value,
|
|
93
|
+
# failing the query. Escaping the backslash and terminator bytes makes
|
|
94
|
+
# String values round-trip byte-for-byte. Values serialized from other
|
|
95
|
+
# Ruby types cannot contain these bytes.
|
|
96
|
+
ESCAPED_TEXT_REPLACEMENTS = {
|
|
97
|
+
"\\" => "\\\\",
|
|
98
|
+
"\t" => "\\t",
|
|
99
|
+
"\n" => "\\n",
|
|
100
|
+
"\r" => "\\r",
|
|
101
|
+
}.freeze
|
|
102
|
+
|
|
103
|
+
# Escapes a top-level String value for ClickHouse's escaped text format.
|
|
104
|
+
#
|
|
105
|
+
# @param str [String] String value to escape.
|
|
106
|
+
# @return [String] Escaped parameter text.
|
|
107
|
+
def self.escape_top_level_string(str)
|
|
108
|
+
str.gsub(/[\\\t\n\r]/, ESCAPED_TEXT_REPLACEMENTS)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Escapes each single-quote and backslash by prefixing it with a
|
|
112
|
+
# backslash, and wraps the result in single quotes.
|
|
113
|
+
#
|
|
114
|
+
# @param str [String] String value to quote.
|
|
115
|
+
# @return [String] Quoted and escaped string literal.
|
|
116
|
+
def self.quote_and_escape_string(str)
|
|
117
|
+
# Note the double-escaping in the replacement string:
|
|
118
|
+
# - Ruby strings consume one backslash
|
|
119
|
+
# - ruby syntax "\\" => ruby string "\"
|
|
120
|
+
# - Regexp replacement consumes one backslash
|
|
121
|
+
# - ruby literal syntax \\\\ => ruby string \\ => regexp replacement \
|
|
122
|
+
# - ruby literal syntax \\0 => ruby string \0 => regexp replacement match #0
|
|
123
|
+
"'#{str.gsub(/['\\]/, "\\\\\\0")}'"
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|