clickhouse-sql 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +9 -0
- data/LICENSE +25 -0
- data/README.md +426 -0
- data/lib/clickhouse/http_client/database.rb +42 -0
- data/lib/clickhouse/http_client/error.rb +18 -0
- data/lib/clickhouse/http_client/parameter_serializer.rb +127 -0
- data/lib/clickhouse/http_client/query.rb +54 -0
- data/lib/clickhouse/http_client/response_formatter.rb +78 -0
- data/lib/clickhouse/http_client/value_normalization.rb +29 -0
- data/lib/clickhouse/http_client.rb +288 -0
- data/lib/clickhouse/sql/bind_index_manager.rb +26 -0
- data/lib/clickhouse/sql/error.rb +13 -0
- data/lib/clickhouse/sql/fragment.rb +171 -0
- data/lib/clickhouse/sql/join.rb +70 -0
- data/lib/clickhouse/sql/part.rb +37 -0
- data/lib/clickhouse/sql/placeholder.rb +41 -0
- data/lib/clickhouse/sql/placeholder_type.rb +219 -0
- data/lib/clickhouse/sql/query.rb +139 -0
- data/lib/clickhouse/sql/raw.rb +53 -0
- data/lib/clickhouse/sql/select_item.rb +48 -0
- data/lib/clickhouse/sql/select_query.rb +393 -0
- data/lib/clickhouse/sql/setting.rb +89 -0
- data/lib/clickhouse/sql/table.rb +97 -0
- data/lib/clickhouse/sql/version.rb +7 -0
- data/lib/clickhouse/sql.rb +562 -0
- data/lib/clickhouse-sql.rb +5 -0
- metadata +133 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module HTTPClient
|
|
5
|
+
# Immutable query value object: SQL text with ClickHouse `{name:Type}`
|
|
6
|
+
# placeholders, plus the placeholder values to bind. Also the normal form
|
|
7
|
+
# every accepted query is converted into at the client boundary.
|
|
8
|
+
Query = Data.define(:sql, :placeholders) do
|
|
9
|
+
# Accepts a String of trusted SQL or any object honoring the query
|
|
10
|
+
# protocol (#to_sql and #placeholders).
|
|
11
|
+
#
|
|
12
|
+
# @param query [String, #to_sql & #placeholders]
|
|
13
|
+
# @return [ClickHouse::HTTPClient::Query]
|
|
14
|
+
def self.wrap(query)
|
|
15
|
+
case query
|
|
16
|
+
when String
|
|
17
|
+
new(sql: query)
|
|
18
|
+
else
|
|
19
|
+
# A builder that can compile itself (e.g. ClickHouse::SQL's
|
|
20
|
+
# SelectQuery#to_query) does so here, so its compile-time
|
|
21
|
+
# validations — like the nil-placeholder net — apply even when a
|
|
22
|
+
# caller passes the builder directly instead of calling #to_query.
|
|
23
|
+
# The arity check excludes ActiveSupport's unrelated
|
|
24
|
+
# Object#to_query(namespace), which every object responds to.
|
|
25
|
+
if query.respond_to?(:to_query) && query.method(:to_query).arity.zero?
|
|
26
|
+
query = query.to_query
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
unless query.respond_to?(:to_sql) && query.respond_to?(:placeholders)
|
|
30
|
+
raise QueryError, "Expected a SQL string or an object responding to #to_sql and #placeholders, got #{query.class}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
new(sql: query.to_sql, placeholders: query.placeholders)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def initialize(sql:, placeholders: {})
|
|
38
|
+
raise QueryError, "Empty query string given" if sql.to_s.strip.empty?
|
|
39
|
+
|
|
40
|
+
super(
|
|
41
|
+
sql: -sql.to_s,
|
|
42
|
+
placeholders: ValueNormalization.immutable_copy(
|
|
43
|
+
ValueNormalization.symbolize_keys(placeholders || {}),
|
|
44
|
+
),
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @return [String] SQL query text.
|
|
49
|
+
def to_sql
|
|
50
|
+
sql
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module HTTPClient
|
|
5
|
+
# Typecasts values from ClickHouse's JSON response according to the
|
|
6
|
+
# declared column metadata.
|
|
7
|
+
module ResponseFormatter
|
|
8
|
+
TYPE_CASTERS = [
|
|
9
|
+
[/\AU?Int\d+\z/, ->(value) { Integer(value) }],
|
|
10
|
+
[/\AFloat(?:32|64)\z/, ->(value) { Float(value) }],
|
|
11
|
+
[/\ADate(?:32)?\z/, ->(value) { Date.parse(value) }],
|
|
12
|
+
# ClickHouse renders zone-less DateTime/DateTime64 values in the
|
|
13
|
+
# *server's* default timezone, so parsing them as UTC assumes every
|
|
14
|
+
# ClickHouse server in the fleet runs UTC. Explicit non-UTC zones
|
|
15
|
+
# deliberately don't match and pass through as strings.
|
|
16
|
+
[/\ADateTime(?:64)?(?:\(\d+\)|\(\d+,\s*'UTC'\)|\('UTC'\))?\z/, ->(value) { Time.parse("#{value} UTC") }],
|
|
17
|
+
[/\AIntervalSecond\z/, ->(value) { Integer(value) }],
|
|
18
|
+
[/\AIntervalMillisecond\z/, ->(value) { Integer(value) / 1000.0 }],
|
|
19
|
+
].freeze
|
|
20
|
+
private_constant :TYPE_CASTERS
|
|
21
|
+
|
|
22
|
+
IDENTITY = ->(value) { value }
|
|
23
|
+
private_constant :IDENTITY
|
|
24
|
+
|
|
25
|
+
# Type wrappers that don't change the text encoding of the inner value.
|
|
26
|
+
WRAPPER_TYPE = /\A(?:Nullable|LowCardinality)\((.*)\)\z/
|
|
27
|
+
private_constant :WRAPPER_TYPE
|
|
28
|
+
|
|
29
|
+
AGGREGATE_WRAPPER_TYPE = /\ASimpleAggregateFunction\(\s*\w+\s*,\s*(.*)\)\z/
|
|
30
|
+
private_constant :AGGREGATE_WRAPPER_TYPE
|
|
31
|
+
|
|
32
|
+
ARRAY_TYPE = /\AArray\((.*)\)\z/
|
|
33
|
+
private_constant :ARRAY_TYPE
|
|
34
|
+
|
|
35
|
+
def self.format_rows(parsed)
|
|
36
|
+
casters = parsed["meta"].to_h do |column|
|
|
37
|
+
[column["name"], caster_for(column["type"])]
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
parsed["data"].map do |row|
|
|
41
|
+
row.to_h { |column, value| [column, casters.fetch(column, IDENTITY).call(value)] }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.caster_for(type)
|
|
46
|
+
unwrapped = unwrap_type(type)
|
|
47
|
+
|
|
48
|
+
# Array elements need the same casting as scalars: JSON output quotes
|
|
49
|
+
# 64-bit integers and datetimes inside arrays too, so Array(UInt64)
|
|
50
|
+
# arrives as an array of strings.
|
|
51
|
+
if (array = ARRAY_TYPE.match(unwrapped))
|
|
52
|
+
element_caster = caster_for(array[1].strip)
|
|
53
|
+
return IDENTITY if element_caster.equal?(IDENTITY)
|
|
54
|
+
|
|
55
|
+
return ->(value) { value.nil? ? nil : value.map { |element| element_caster.call(element) } }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
_, cast = TYPE_CASTERS.find { |regex, _| regex.match?(unwrapped) }
|
|
59
|
+
return IDENTITY unless cast
|
|
60
|
+
|
|
61
|
+
->(value) { value.nil? ? nil : cast.call(value) }
|
|
62
|
+
end
|
|
63
|
+
private_class_method :caster_for
|
|
64
|
+
|
|
65
|
+
def self.unwrap_type(type)
|
|
66
|
+
type = type.to_s
|
|
67
|
+
|
|
68
|
+
while (match = WRAPPER_TYPE.match(type) || AGGREGATE_WRAPPER_TYPE.match(type))
|
|
69
|
+
type = match[1]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
type
|
|
73
|
+
end
|
|
74
|
+
private_class_method :unwrap_type
|
|
75
|
+
end
|
|
76
|
+
private_constant :ResponseFormatter
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module HTTPClient
|
|
5
|
+
module ValueNormalization
|
|
6
|
+
def self.symbolize_keys(hash)
|
|
7
|
+
hash.transform_keys do |key|
|
|
8
|
+
key.respond_to?(:to_sym) ? key.to_sym : key
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# Recursively copies and freezes Arrays, Hashes, and Strings so a
|
|
13
|
+
# value can be held without observing callers' later mutations.
|
|
14
|
+
def self.immutable_copy(value)
|
|
15
|
+
case value
|
|
16
|
+
when Array
|
|
17
|
+
value.map { |item| immutable_copy(item) }.freeze
|
|
18
|
+
when Hash
|
|
19
|
+
value.to_h { |key, item| [immutable_copy(key), immutable_copy(item)] }.freeze
|
|
20
|
+
when String
|
|
21
|
+
value.dup.freeze
|
|
22
|
+
else
|
|
23
|
+
value
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
private_constant :ValueNormalization
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
require "json"
|
|
5
|
+
require "logger"
|
|
6
|
+
require "net/http"
|
|
7
|
+
require "securerandom"
|
|
8
|
+
require "time"
|
|
9
|
+
require "uri"
|
|
10
|
+
require "zlib"
|
|
11
|
+
|
|
12
|
+
require_relative "http_client/error"
|
|
13
|
+
require_relative "http_client/value_normalization"
|
|
14
|
+
require_relative "http_client/database"
|
|
15
|
+
require_relative "http_client/query"
|
|
16
|
+
require_relative "http_client/parameter_serializer"
|
|
17
|
+
require_relative "http_client/response_formatter"
|
|
18
|
+
|
|
19
|
+
module ClickHouse
|
|
20
|
+
# HTTP transport for ClickHouse queries: owns database configuration,
|
|
21
|
+
# HTTP POST mechanics, parameter serialization, response typecasting,
|
|
22
|
+
# logging, and instrumentation.
|
|
23
|
+
#
|
|
24
|
+
# Queries are duck-typed: any object responding to #to_sql (SQL text with
|
|
25
|
+
# ClickHouse `{name:Type}` placeholders intact) and #placeholders (a Hash of
|
|
26
|
+
# placeholder names to Ruby values) is accepted, as is a plain String of
|
|
27
|
+
# trusted SQL. ClickHouse::SQL::Query satisfies this protocol; so does the
|
|
28
|
+
# HTTPClient::Query value object. An object that also responds to
|
|
29
|
+
# #to_query (e.g. ClickHouse::SQL::SelectQuery) is compiled through it
|
|
30
|
+
# first, so builder-level validations always apply.
|
|
31
|
+
#
|
|
32
|
+
# nil placeholder values are always sent as NULL. Binding nil to a
|
|
33
|
+
# non-Nullable placeholder type fails loudly in ClickHouse for most types,
|
|
34
|
+
# but parses as an empty string for a plain String placeholder — callers
|
|
35
|
+
# (e.g. ClickHouse::SQL) are expected to validate nil against placeholder
|
|
36
|
+
# types before execution.
|
|
37
|
+
#
|
|
38
|
+
# Derived in part from GitLab's MIT-licensed click_house-client gem;
|
|
39
|
+
# see LICENSE.
|
|
40
|
+
module HTTPClient
|
|
41
|
+
# Instrumentation event emitted around every query.
|
|
42
|
+
NOTIFICATION_NAME = "sql.click_house"
|
|
43
|
+
QUERY_ID_HEADER = "x-clickhouse-query-id"
|
|
44
|
+
private_constant :QUERY_ID_HEADER
|
|
45
|
+
|
|
46
|
+
# Successful outcome of a command-style query. Failures raise instead of
|
|
47
|
+
# returning a result. Statistics are parsed from x-clickhouse-summary and
|
|
48
|
+
# are final because every request sends wait_end_of_query=1.
|
|
49
|
+
ExecutionResult = Data.define(:query_id, :statistics)
|
|
50
|
+
|
|
51
|
+
NOOP_INSTRUMENTER = Object.new
|
|
52
|
+
def NOOP_INSTRUMENTER.instrument(_name, payload)
|
|
53
|
+
yield payload
|
|
54
|
+
end
|
|
55
|
+
private_constant :NOOP_INSTRUMENTER
|
|
56
|
+
|
|
57
|
+
# Applications typically configure the logger, instrumenter, and database
|
|
58
|
+
# registry once during initialization.
|
|
59
|
+
class << self
|
|
60
|
+
# @return [Logger] Structured query logger.
|
|
61
|
+
attr_writer :logger # rubocop:disable ThreadSafety/ClassAndModuleAttributes
|
|
62
|
+
|
|
63
|
+
def logger
|
|
64
|
+
@logger ||= Logger.new(IO::NULL) # rubocop:disable ThreadSafety/ClassInstanceVariable
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# @return [#instrument] Query instrumenter.
|
|
68
|
+
attr_writer :instrumenter # rubocop:disable ThreadSafety/ClassAndModuleAttributes
|
|
69
|
+
|
|
70
|
+
def instrumenter
|
|
71
|
+
@instrumenter || NOOP_INSTRUMENTER # rubocop:disable ThreadSafety/ClassInstanceVariable
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Registers a database connection under a name.
|
|
75
|
+
def register_database(name, **args)
|
|
76
|
+
raise ConfigurationError, "The database '#{name}' is already registered" if databases.key?(name)
|
|
77
|
+
|
|
78
|
+
databases[name] = Database.build(**args)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @param name [Symbol] Registered database name.
|
|
82
|
+
# @return [ClickHouse::HTTPClient::Database]
|
|
83
|
+
def database(name)
|
|
84
|
+
databases.fetch(name) do
|
|
85
|
+
raise ConfigurationError, "The database '#{name}' is not registered"
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Executes a SELECT query and returns rows as an Array of Hashes, with
|
|
90
|
+
# values typecast according to the column types in the response. Callers
|
|
91
|
+
# migrating an existing result contract can supply a formatter block,
|
|
92
|
+
# which receives the parsed response including its column metadata.
|
|
93
|
+
def select(query, database, timeout: nil, &formatter)
|
|
94
|
+
query = Query.wrap(query)
|
|
95
|
+
body, content_type = encode_multipart(form_data(query))
|
|
96
|
+
|
|
97
|
+
run(query, database, timeout: timeout, body: body, extra_headers: { "Content-Type" => content_type }) do |response|
|
|
98
|
+
parsed = parse_result_body(response)
|
|
99
|
+
|
|
100
|
+
statistics = ValueNormalization.symbolize_keys(parsed["statistics"]) if parsed["statistics"]
|
|
101
|
+
rows = formatter ? formatter.call(parsed) : ResponseFormatter.format_rows(parsed)
|
|
102
|
+
[rows, statistics]
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Executes any query without returning row data (INSERT, DELETE, DDL).
|
|
107
|
+
#
|
|
108
|
+
# @return [ClickHouse::HTTPClient::ExecutionResult]
|
|
109
|
+
def execute(query, database, timeout: nil)
|
|
110
|
+
query = Query.wrap(query)
|
|
111
|
+
body, content_type = encode_multipart(form_data(query))
|
|
112
|
+
|
|
113
|
+
run(query, database, timeout: timeout, body: body, extra_headers: { "Content-Type" => content_type }) do |response|
|
|
114
|
+
statistics = summary_statistics(response)
|
|
115
|
+
result = ExecutionResult.new(
|
|
116
|
+
query_id: response[QUERY_ID_HEADER],
|
|
117
|
+
statistics: statistics,
|
|
118
|
+
)
|
|
119
|
+
[result, statistics]
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Inserts a gzip-compressed CSV IO to ClickHouse.
|
|
124
|
+
#
|
|
125
|
+
# Usage:
|
|
126
|
+
#
|
|
127
|
+
# io = StringIO.new(gzip_compressed_csv)
|
|
128
|
+
# ClickHouse::HTTPClient.insert_csv("INSERT INTO events (id) FORMAT CSV", io, :main)
|
|
129
|
+
#
|
|
130
|
+
# @return [ClickHouse::HTTPClient::ExecutionResult]
|
|
131
|
+
def insert_csv(query, io, database, timeout: nil)
|
|
132
|
+
query = Query.wrap(query)
|
|
133
|
+
|
|
134
|
+
# The request body carries the CSV data, so there's nowhere to send
|
|
135
|
+
# param_* placeholder values; ClickHouse would receive unresolved
|
|
136
|
+
# {name:Type} markers in the SQL.
|
|
137
|
+
unless query.placeholders.empty?
|
|
138
|
+
raise QueryError, "insert_csv does not support query placeholders"
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
run(query, database, timeout: timeout, body: io, extra_headers: CSV_HEADERS, uri_variables: { query: query.sql }) do |response|
|
|
142
|
+
statistics = summary_statistics(response)
|
|
143
|
+
result = ExecutionResult.new(
|
|
144
|
+
query_id: response[QUERY_ID_HEADER],
|
|
145
|
+
statistics: statistics,
|
|
146
|
+
)
|
|
147
|
+
[result, statistics]
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
private
|
|
152
|
+
|
|
153
|
+
CSV_HEADERS = { "Transfer-Encoding" => "chunked", "Content-Encoding" => "gzip" }.freeze
|
|
154
|
+
|
|
155
|
+
def databases
|
|
156
|
+
@databases ||= {} # rubocop:disable ThreadSafety/ClassInstanceVariable
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def run(query, database_name, timeout:, body:, extra_headers:, uri_variables: {})
|
|
160
|
+
db = database(database_name)
|
|
161
|
+
uri = db.uri(uri_variables)
|
|
162
|
+
|
|
163
|
+
logger.debug { { clickhouse_query: query.sql } }
|
|
164
|
+
|
|
165
|
+
result = nil
|
|
166
|
+
instrumenter.instrument(NOTIFICATION_NAME, { query: query, database: database_name }) do |instrument|
|
|
167
|
+
response = begin
|
|
168
|
+
perform_request(uri, db.headers.merge(extra_headers), body, timeout || 30)
|
|
169
|
+
rescue Net::OpenTimeout => e
|
|
170
|
+
raise ConnectionError, e.message
|
|
171
|
+
rescue Net::ReadTimeout, Net::WriteTimeout, Timeout::Error, Errno::ETIMEDOUT => e
|
|
172
|
+
raise TimeoutError, e.message
|
|
173
|
+
rescue SocketError, EOFError, IOError, SystemCallError, OpenSSL::SSL::SSLError,
|
|
174
|
+
Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, Zlib::Error => e
|
|
175
|
+
raise ConnectionError, e.message
|
|
176
|
+
end
|
|
177
|
+
normalize_response_body(response)
|
|
178
|
+
|
|
179
|
+
# The query ID correlates with ClickHouse's own system.query_log and
|
|
180
|
+
# is useful to instrumentation even when ClickHouse rejects a query.
|
|
181
|
+
instrument[:query_id] = response[QUERY_ID_HEADER]
|
|
182
|
+
|
|
183
|
+
raise DatabaseError, response.body unless response.is_a?(Net::HTTPSuccess)
|
|
184
|
+
|
|
185
|
+
result, statistics = yield(response)
|
|
186
|
+
|
|
187
|
+
instrument[:statistics] = statistics
|
|
188
|
+
|
|
189
|
+
result
|
|
190
|
+
end
|
|
191
|
+
result
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def perform_request(uri, headers, body, timeout)
|
|
195
|
+
request = Net::HTTP::Post.new(uri.request_uri, headers)
|
|
196
|
+
if body.respond_to?(:read)
|
|
197
|
+
request.body_stream = body
|
|
198
|
+
else
|
|
199
|
+
request.body = body
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
Net::HTTP.start(
|
|
203
|
+
uri.hostname,
|
|
204
|
+
uri.port,
|
|
205
|
+
use_ssl: uri.scheme == "https",
|
|
206
|
+
open_timeout: 2,
|
|
207
|
+
read_timeout: timeout,
|
|
208
|
+
write_timeout: timeout,
|
|
209
|
+
max_retries: 0,
|
|
210
|
+
) { |http| http.request(request) }
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def form_data(query)
|
|
214
|
+
query.placeholders.each_with_object({ "query" => query.sql }) do |(name, value), form|
|
|
215
|
+
form["param_#{name}"] = ParameterSerializer.serialize(value)
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Placeholders are sent as param_-prefixed multipart form fields
|
|
220
|
+
# alongside the query, dodging URL length limits. Net::HTTP#set_form
|
|
221
|
+
# writes non-chunked multipart requests through a Tempfile, so encode the
|
|
222
|
+
# all-String subset used here directly instead.
|
|
223
|
+
# See: https://github.com/ClickHouse/ClickHouse/issues/8842
|
|
224
|
+
def encode_multipart(fields)
|
|
225
|
+
boundary = "----RubyMultipart#{SecureRandom.hex(16)}"
|
|
226
|
+
body = String.new(encoding: Encoding::BINARY)
|
|
227
|
+
|
|
228
|
+
fields.each do |name, value|
|
|
229
|
+
escaped_name = name.to_s
|
|
230
|
+
.gsub(/["\\]/) { |character| "\\#{character}" }
|
|
231
|
+
.gsub("\r", "%0D")
|
|
232
|
+
.gsub("\n", "%0A")
|
|
233
|
+
body << "--#{boundary}\r\n"
|
|
234
|
+
body << "Content-Disposition: form-data; name=\"#{escaped_name}\"\r\n\r\n"
|
|
235
|
+
body << value.b
|
|
236
|
+
body << "\r\n"
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
body << "--#{boundary}--\r\n"
|
|
240
|
+
[body, "multipart/form-data; boundary=#{boundary}"]
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def normalize_response_body(response)
|
|
244
|
+
return unless response.body
|
|
245
|
+
|
|
246
|
+
/\bcharset=([^;]+)/.match(response["content-type"]) do |match|
|
|
247
|
+
response.body.force_encoding(Encoding.find(match[1].strip))
|
|
248
|
+
end
|
|
249
|
+
rescue ArgumentError
|
|
250
|
+
# Match Faraday's NetHTTP adapter: an unsupported charset leaves the
|
|
251
|
+
# response body in its original binary encoding.
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# A malformed body on a success status is still a server-side failure:
|
|
255
|
+
# even with wait_end_of_query buffering, an error can slip in after the
|
|
256
|
+
# response has been committed (e.g. the buffer overflowing to the
|
|
257
|
+
# client). ClickHouse appends its error text to the body, so surface
|
|
258
|
+
# the tail in the DatabaseError rather than a bare JSON::ParserError.
|
|
259
|
+
def parse_result_body(response)
|
|
260
|
+
JSON.parse(response.body)
|
|
261
|
+
rescue JSON::ParserError
|
|
262
|
+
body = response.body.to_s
|
|
263
|
+
body_tail = body.length > 500 ? body[-500, 500] : body
|
|
264
|
+
raise DatabaseError, "Malformed ClickHouse response body (HTTP #{response.code}), body tail: #{body_tail}"
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def summary_statistics(headers)
|
|
268
|
+
summary = headers["x-clickhouse-summary"]
|
|
269
|
+
return {}.freeze unless summary
|
|
270
|
+
|
|
271
|
+
# The `x-clickhouse-summary` header encodes its numeric values as
|
|
272
|
+
# strings, unlike the `statistics` block in the JSON body which uses
|
|
273
|
+
# real numbers. Coerce them so instrumentation receives numeric metrics
|
|
274
|
+
# consistently across query paths.
|
|
275
|
+
ValueNormalization
|
|
276
|
+
.symbolize_keys(JSON.parse(summary))
|
|
277
|
+
.transform_values { |value| coerce_numeric(value) }
|
|
278
|
+
.freeze
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def coerce_numeric(value)
|
|
282
|
+
return value unless value.is_a?(String)
|
|
283
|
+
|
|
284
|
+
Integer(value, exception: false) || Float(value, exception: false) || value
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module SQL
|
|
5
|
+
# Emits sequential `$1`-style bind markers when rendering redacted SQL,
|
|
6
|
+
# so every typed placeholder in a query gets a stable positional marker
|
|
7
|
+
# suitable for external log aggregation.
|
|
8
|
+
#
|
|
9
|
+
# Derived from GitLab's MIT-licensed click_house-client gem; see LICENSE.
|
|
10
|
+
class BindIndexManager
|
|
11
|
+
# @param start_index [Integer] Index of the first bind marker.
|
|
12
|
+
def initialize(start_index = 1)
|
|
13
|
+
@current_index = start_index
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Returns the next positional bind marker.
|
|
17
|
+
#
|
|
18
|
+
# @return [String] Bind marker such as `$1`.
|
|
19
|
+
def next_bind_str
|
|
20
|
+
bind_str = "$#{@current_index}"
|
|
21
|
+
@current_index += 1
|
|
22
|
+
bind_str
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module SQL
|
|
5
|
+
# Raised when ClickHouse::SQL cannot safely build or validate a query.
|
|
6
|
+
#
|
|
7
|
+
# Deliberately not a ClickHouse::HTTPClient error: a query-building error
|
|
8
|
+
# is a caller bug in this layer, raised before anything reaches the
|
|
9
|
+
# client. Rescue it alongside ClickHouse::HTTPClient errors where callers
|
|
10
|
+
# degrade gracefully on query failure.
|
|
11
|
+
class Error < StandardError; end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ClickHouse
|
|
4
|
+
module SQL
|
|
5
|
+
class Fragment
|
|
6
|
+
include Part
|
|
7
|
+
|
|
8
|
+
# Builds a composable SQL fragment with typed placeholders and slots.
|
|
9
|
+
#
|
|
10
|
+
# @param sql [String, #to_s] SQL containing `{name:Type}` values, `{name}`
|
|
11
|
+
# slots, and/or anonymous `{}` slots.
|
|
12
|
+
# @param bindings [Hash{Symbol,String=>Object}] Named placeholder and slot bindings.
|
|
13
|
+
# @param fills [Array<ClickHouse::SQL::Part>] Positional fills for anonymous
|
|
14
|
+
# `{}` slots in source order.
|
|
15
|
+
# @return [ClickHouse::SQL::Fragment] A validated fragment.
|
|
16
|
+
def initialize(sql, bindings = {}, fills: [])
|
|
17
|
+
raise Error, "Empty SQL fragment given" if ValueNormalization.blank_string?(sql)
|
|
18
|
+
|
|
19
|
+
@sql = sql.to_s
|
|
20
|
+
@bindings = ValueNormalization.symbolize_keys(bindings || {})
|
|
21
|
+
@fills = Array(fills)
|
|
22
|
+
@placeholders = ClickHouse::SQL.parse_placeholders(@sql)
|
|
23
|
+
|
|
24
|
+
validate_fills!
|
|
25
|
+
validate_bindings!
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Renders the fragment with ClickHouse typed placeholders intact.
|
|
29
|
+
#
|
|
30
|
+
# @return [String] SQL fragment text.
|
|
31
|
+
def to_sql
|
|
32
|
+
render(redacted: false)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Renders the fragment with typed placeholders replaced by bind markers.
|
|
36
|
+
#
|
|
37
|
+
# @param bind_index_manager [ClickHouse::SQL::BindIndexManager] Redacted bind index state.
|
|
38
|
+
# @return [String] Redacted SQL fragment text.
|
|
39
|
+
def to_redacted_sql(bind_index_manager = ClickHouse::SQL::BindIndexManager.new)
|
|
40
|
+
render(redacted: true, bind_index_manager:)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Returns typed placeholder values from this fragment and nested slots.
|
|
44
|
+
#
|
|
45
|
+
# @return [Hash{Symbol=>Object}] Placeholder values keyed by name.
|
|
46
|
+
def placeholders
|
|
47
|
+
ClickHouse::SQL.merge_placeholders!(@typed_placeholder_values.dup, nested_placeholders)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Returns ClickHouse placeholder types from this fragment and nested slots.
|
|
51
|
+
#
|
|
52
|
+
# @return [Hash{Symbol=>String}] Placeholder types keyed by name.
|
|
53
|
+
def placeholder_types
|
|
54
|
+
ClickHouse::SQL.merge_placeholder_types!(@placeholder_types.dup, nested_placeholder_types)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
attr_reader :bindings
|
|
60
|
+
|
|
61
|
+
def validate_fills!
|
|
62
|
+
anonymous = @placeholders.select(&:anonymous?)
|
|
63
|
+
unless anonymous.length == @fills.length
|
|
64
|
+
raise Error, "Fragment has #{anonymous.length} anonymous {} slots but #{@fills.length} positional fills were given"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
@fills.each do |fill|
|
|
68
|
+
next if ClickHouse::SQL.part?(fill)
|
|
69
|
+
|
|
70
|
+
raise Error, "Anonymous fragment slot {} requires a SQL fragment, got #{fill.class}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
@fill_by_position = anonymous.map(&:begin_pos).zip(@fills).to_h
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def validate_bindings!
|
|
77
|
+
named = @placeholders.reject(&:anonymous?)
|
|
78
|
+
used_names = named.map(&:name_sym)
|
|
79
|
+
unused_names = bindings.keys - used_names
|
|
80
|
+
raise Error, "Unused ClickHouse placeholder values: #{unused_names.join(', ')}" if unused_names.any?
|
|
81
|
+
|
|
82
|
+
typed_names = named.select(&:typed?).map(&:name_sym)
|
|
83
|
+
slot_names = named.reject(&:typed?).map(&:name_sym)
|
|
84
|
+
collisions = typed_names & slot_names
|
|
85
|
+
raise Error, "Placeholder names cannot be both typed values and fragment slots: #{collisions.join(', ')}" if collisions.any?
|
|
86
|
+
|
|
87
|
+
missing_names = used_names.uniq - bindings.keys
|
|
88
|
+
raise Error, "Missing ClickHouse placeholder values: #{missing_names.join(', ')}" if missing_names.any?
|
|
89
|
+
|
|
90
|
+
@typed_placeholder_values = {}
|
|
91
|
+
@placeholder_types = {}
|
|
92
|
+
|
|
93
|
+
named.select(&:typed?).each do |placeholder|
|
|
94
|
+
type = PlaceholderType.new(placeholder.type)
|
|
95
|
+
value = type.validate_and_normalize(bindings.fetch(placeholder.name_sym))
|
|
96
|
+
|
|
97
|
+
register_typed_placeholder!(placeholder.name_sym, type.normalized, value)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
named.reject(&:typed?).each do |placeholder|
|
|
101
|
+
value = bindings.fetch(placeholder.name_sym)
|
|
102
|
+
next if ClickHouse::SQL.part?(value)
|
|
103
|
+
|
|
104
|
+
raise Error, "Fragment slot {#{placeholder.name}} requires a SQL fragment, got #{value.class}"
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def register_typed_placeholder!(name, type, value)
|
|
109
|
+
if @placeholder_types.key?(name)
|
|
110
|
+
unless @placeholder_types.fetch(name) == type
|
|
111
|
+
raise Error, "mismatching types for the '#{name}' placeholder: #{@placeholder_types.fetch(name)} vs #{type}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
unless @typed_placeholder_values.fetch(name) == value
|
|
115
|
+
raise Error, "mismatching values for the '#{name}' placeholder: " \
|
|
116
|
+
"#{ClickHouse::SQL.truncated_inspect(@typed_placeholder_values.fetch(name))} vs #{ClickHouse::SQL.truncated_inspect(value)}"
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
@placeholder_types[name] = type
|
|
121
|
+
@typed_placeholder_values[name] = value
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Returns nested slot parts (anonymous fills and named slot values) in source order.
|
|
125
|
+
def slot_parts
|
|
126
|
+
@placeholders.reject(&:typed?).map { |placeholder| slot_part(placeholder) }
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def slot_part(placeholder)
|
|
130
|
+
if placeholder.anonymous?
|
|
131
|
+
@fill_by_position.fetch(placeholder.begin_pos)
|
|
132
|
+
else
|
|
133
|
+
bindings.fetch(placeholder.name_sym)
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def nested_placeholders
|
|
138
|
+
slot_parts.each_with_object({}) do |part, values|
|
|
139
|
+
ClickHouse::SQL.merge_placeholders!(values, part.placeholders)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def nested_placeholder_types
|
|
144
|
+
slot_parts.each_with_object({}) do |part, types|
|
|
145
|
+
ClickHouse::SQL.merge_placeholder_types!(types, part.placeholder_types)
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def render(redacted:, bind_index_manager: ClickHouse::SQL::BindIndexManager.new)
|
|
150
|
+
rendered = +""
|
|
151
|
+
last_pos = 0
|
|
152
|
+
|
|
153
|
+
@placeholders.each do |placeholder|
|
|
154
|
+
rendered << @sql[last_pos...placeholder.begin_pos]
|
|
155
|
+
rendered << render_placeholder(placeholder, redacted:, bind_index_manager:)
|
|
156
|
+
last_pos = placeholder.end_pos
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
rendered << @sql[last_pos..].to_s
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def render_placeholder(placeholder, redacted:, bind_index_manager:)
|
|
163
|
+
if placeholder.typed?
|
|
164
|
+
redacted ? bind_index_manager.next_bind_str : placeholder.raw
|
|
165
|
+
else
|
|
166
|
+
ClickHouse::SQL.render_part(slot_part(placeholder), redacted:, bind_index_manager:)
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|