clickhouse-activerecord 1.6.6 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/testing.yml +12 -14
- data/CHANGELOG.md +26 -0
- data/README.md +122 -1
- data/bin/console +1 -1
- data/bin/test-single +7 -0
- data/clickhouse-activerecord.gemspec +0 -1
- data/lib/active_record/connection_adapters/clickhouse/column.rb +23 -3
- data/lib/active_record/connection_adapters/clickhouse/oid/json.rb +23 -0
- data/lib/active_record/connection_adapters/clickhouse/oid/map.rb +16 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +113 -38
- data/lib/active_record/connection_adapters/clickhouse/statement/response_processor.rb +8 -3
- data/lib/active_record/connection_adapters/clickhouse/table_definition.rb +7 -1
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +105 -11
- data/lib/arel/nodes/final_table.rb +10 -0
- data/lib/arel/visitors/clickhouse.rb +13 -4
- data/lib/clickhouse-activerecord/rspec.rb +1 -1
- data/lib/clickhouse-activerecord/schema_dumper.rb +3 -8
- data/lib/clickhouse-activerecord/tasks.rb +19 -15
- data/lib/clickhouse-activerecord/version.rb +1 -1
- data/lib/clickhouse-activerecord.rb +10 -0
- data/lib/core_extensions/active_record/relation.rb +121 -2
- metadata +6 -20
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'tempfile'
|
|
4
|
+
|
|
3
5
|
module ActiveRecord
|
|
4
6
|
module ConnectionAdapters
|
|
5
7
|
module Clickhouse
|
|
@@ -52,7 +54,7 @@ module ActiveRecord
|
|
|
52
54
|
|
|
53
55
|
# @return [String, Hash, Array]
|
|
54
56
|
def process_successful_response
|
|
55
|
-
|
|
57
|
+
raise_database_error!(@sql) if @body.include?('DB::Exception') && @body.match?(DB_EXCEPTION_REGEXP)
|
|
56
58
|
|
|
57
59
|
format_body_response
|
|
58
60
|
end
|
|
@@ -101,14 +103,17 @@ module ActiveRecord
|
|
|
101
103
|
JSON.parse(payload, decimal_class: BigDecimal)
|
|
102
104
|
end
|
|
103
105
|
|
|
104
|
-
def raise_database_error!
|
|
106
|
+
def raise_database_error!(sql = nil)
|
|
105
107
|
case @body
|
|
106
108
|
when /DB::Exception:.*\(UNKNOWN_DATABASE\)/
|
|
107
109
|
raise ActiveRecord::NoDatabaseError
|
|
108
110
|
when /DB::Exception:.*\(DATABASE_ALREADY_EXISTS\)/
|
|
109
111
|
raise ActiveRecord::DatabaseAlreadyExists
|
|
112
|
+
when /DB::Exception:.*\(TIMEOUT_EXCEEDED\)/
|
|
113
|
+
# server-side cancellation, vs. ActiveRecord::AdapterTimeout (client gave up)
|
|
114
|
+
raise ActiveRecord::StatementTimeout, "Response code: #{@raw_response.code}:\n#{@body}#{"\nQuery: #{sql}" if sql}"
|
|
110
115
|
else
|
|
111
|
-
raise_generic!
|
|
116
|
+
raise_generic!(sql)
|
|
112
117
|
end
|
|
113
118
|
end
|
|
114
119
|
|
|
@@ -41,7 +41,7 @@ module ActiveRecord
|
|
|
41
41
|
unsigned = options[:unsigned]
|
|
42
42
|
unsigned = true if unsigned.nil?
|
|
43
43
|
|
|
44
|
-
kind = :uint32 # default
|
|
44
|
+
kind = unsigned ? :uint32 : :int32 # default
|
|
45
45
|
|
|
46
46
|
if options[:limit]
|
|
47
47
|
if unsigned
|
|
@@ -63,6 +63,12 @@ module ActiveRecord
|
|
|
63
63
|
args.each { |name| column(name, kind, **options.except(:limit, :unsigned)) }
|
|
64
64
|
end
|
|
65
65
|
|
|
66
|
+
def float(*args, **options)
|
|
67
|
+
kind = options[:limit] == 8 ? :float64 : :float
|
|
68
|
+
|
|
69
|
+
args.each { |name| column(name, kind, **options.except(:limit)) }
|
|
70
|
+
end
|
|
71
|
+
|
|
66
72
|
def datetime(*args, **options)
|
|
67
73
|
kind = :datetime
|
|
68
74
|
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'arel/visitors/clickhouse'
|
|
4
4
|
require 'arel/nodes/final'
|
|
5
|
+
require 'arel/nodes/final_table'
|
|
5
6
|
require 'arel/nodes/grouping_sets'
|
|
6
7
|
require 'arel/nodes/settings'
|
|
7
8
|
require 'arel/nodes/using'
|
|
@@ -11,6 +12,7 @@ require 'active_record/connection_adapters/clickhouse/oid/date'
|
|
|
11
12
|
require 'active_record/connection_adapters/clickhouse/oid/date_time'
|
|
12
13
|
require 'active_record/connection_adapters/clickhouse/oid/big_integer'
|
|
13
14
|
require 'active_record/connection_adapters/clickhouse/oid/map'
|
|
15
|
+
require 'active_record/connection_adapters/clickhouse/oid/json'
|
|
14
16
|
require 'active_record/connection_adapters/clickhouse/oid/uuid'
|
|
15
17
|
require 'active_record/connection_adapters/clickhouse/column'
|
|
16
18
|
require 'active_record/connection_adapters/clickhouse/quoting'
|
|
@@ -20,6 +22,7 @@ require 'active_record/connection_adapters/clickhouse/statement'
|
|
|
20
22
|
require 'active_record/connection_adapters/clickhouse/table_definition'
|
|
21
23
|
require 'net/http'
|
|
22
24
|
require 'openssl'
|
|
25
|
+
require 'uri'
|
|
23
26
|
|
|
24
27
|
module ActiveRecord
|
|
25
28
|
class Base
|
|
@@ -28,10 +31,22 @@ module ActiveRecord
|
|
|
28
31
|
def clickhouse_connection(config)
|
|
29
32
|
config = config.symbolize_keys
|
|
30
33
|
|
|
34
|
+
if config[:url]
|
|
35
|
+
url_config = ConnectionAdapters::ClickhouseAdapter.parse_clickhouse_url(config.delete(:url))
|
|
36
|
+
config = url_config.merge(config)
|
|
37
|
+
end
|
|
38
|
+
|
|
31
39
|
unless config.key?(:database)
|
|
32
40
|
raise ArgumentError, 'No database specified. Missing argument: database.'
|
|
33
41
|
end
|
|
34
42
|
|
|
43
|
+
if config[:http_auth]
|
|
44
|
+
unless ConnectionAdapters::Clickhouse::SchemaStatements::HTTP_AUTH_TYPES.include?(config[:http_auth]&.to_sym)
|
|
45
|
+
raise ArgumentError, "Unknown :http_auth mode #{config[:http_auth].inspect}. " \
|
|
46
|
+
+ "Use one of #{ConnectionAdapters::Clickhouse::SchemaStatements::HTTP_AUTH_TYPES}."
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
35
50
|
ConnectionAdapters::ClickhouseAdapter.new(config)
|
|
36
51
|
end
|
|
37
52
|
end
|
|
@@ -52,6 +67,8 @@ module ActiveRecord
|
|
|
52
67
|
module ModelSchema
|
|
53
68
|
module ClassMethods
|
|
54
69
|
delegate :final, :final!,
|
|
70
|
+
:joins_final, :joins_final!,
|
|
71
|
+
:left_joins_final, :left_joins_final!,
|
|
55
72
|
:group_by_grouping_sets, :group_by_grouping_sets!,
|
|
56
73
|
:settings, :settings!,
|
|
57
74
|
:window, :window!,
|
|
@@ -90,6 +107,7 @@ module ActiveRecord
|
|
|
90
107
|
integer: { name: 'UInt32' },
|
|
91
108
|
big_integer: { name: 'UInt64' },
|
|
92
109
|
float: { name: 'Float32' },
|
|
110
|
+
float64: { name: 'Float64' },
|
|
93
111
|
decimal: { name: 'Decimal' },
|
|
94
112
|
datetime: { name: 'DateTime' },
|
|
95
113
|
datetime64: { name: 'DateTime64' },
|
|
@@ -119,8 +137,16 @@ module ActiveRecord
|
|
|
119
137
|
|
|
120
138
|
include Clickhouse::SchemaStatements
|
|
121
139
|
|
|
140
|
+
# our `execute` shadows the wrapper `QueryCache` installs on `AbstractAdapter`
|
|
141
|
+
ActiveRecord::ConnectionAdapters::QueryCache.dirties_query_cache(self, :execute)
|
|
142
|
+
|
|
122
143
|
# Initializes and connects a Clickhouse adapter.
|
|
123
144
|
def initialize(config_or_deprecated_connection, deprecated_logger = nil, deprecated_connection_options = nil, deprecated_config = nil)
|
|
145
|
+
if config_or_deprecated_connection.is_a?(Hash) && config_or_deprecated_connection[:url]
|
|
146
|
+
config_or_deprecated_connection = config_or_deprecated_connection.dup
|
|
147
|
+
url_config = self.class.parse_clickhouse_url(config_or_deprecated_connection.delete(:url))
|
|
148
|
+
config_or_deprecated_connection = url_config.merge(config_or_deprecated_connection)
|
|
149
|
+
end
|
|
124
150
|
super
|
|
125
151
|
if @config[:connection]
|
|
126
152
|
connection = {
|
|
@@ -132,7 +158,9 @@ module ActiveRecord
|
|
|
132
158
|
host: @config[:host] || 'localhost',
|
|
133
159
|
port: port,
|
|
134
160
|
ssl: @config[:ssl].present? ? @config[:ssl] : port == 443,
|
|
161
|
+
insecure: @config.fetch(:insecure, true),
|
|
135
162
|
sslca: @config[:sslca],
|
|
163
|
+
open_timeout: @config[:open_timeout],
|
|
136
164
|
read_timeout: @config[:read_timeout],
|
|
137
165
|
write_timeout: @config[:write_timeout],
|
|
138
166
|
keep_alive_timeout: @config[:keep_alive_timeout]
|
|
@@ -140,9 +168,16 @@ module ActiveRecord
|
|
|
140
168
|
end
|
|
141
169
|
@connection_parameters = connection
|
|
142
170
|
|
|
143
|
-
@connection_config = {
|
|
171
|
+
@connection_config = {
|
|
172
|
+
user: @config[:username],
|
|
173
|
+
password: @config[:password],
|
|
174
|
+
database: @config[:database],
|
|
175
|
+
max_execution_time: @config[:max_execution_time],
|
|
176
|
+
cancel_http_readonly_queries_on_client_close: @config[:cancel_http_readonly_queries_on_client_close],
|
|
177
|
+
}.compact
|
|
144
178
|
@debug = @config[:debug] || false
|
|
145
|
-
@
|
|
179
|
+
@default_response_format = @config[:format] || DEFAULT_RESPONSE_FORMAT
|
|
180
|
+
@http_auth = @config[:http_auth]&.to_sym
|
|
146
181
|
|
|
147
182
|
@prepared_statements = false
|
|
148
183
|
|
|
@@ -185,11 +220,55 @@ module ActiveRecord
|
|
|
185
220
|
!native_database_types[type].nil?
|
|
186
221
|
end
|
|
187
222
|
|
|
223
|
+
# Rails appends the limit to the native type name (e.g. "Float32(8)"), which is not valid in ClickHouse.
|
|
224
|
+
# Resolve the float width by limit here so that add_column / change_column behave like TableDefinition#float.
|
|
225
|
+
def type_to_sql(type, limit: nil, precision: nil, scale: nil, **)
|
|
226
|
+
return native_database_types[limit == 8 ? :float64 : :float][:name] if type.to_s == 'float'
|
|
227
|
+
|
|
228
|
+
super
|
|
229
|
+
end
|
|
230
|
+
|
|
188
231
|
def supports_indexes_in_create?
|
|
189
232
|
true
|
|
190
233
|
end
|
|
191
234
|
|
|
192
235
|
class << self
|
|
236
|
+
def parse_clickhouse_url(url)
|
|
237
|
+
uri = URI.parse(url)
|
|
238
|
+
config = {}
|
|
239
|
+
|
|
240
|
+
config[:host] = uri.host if uri.host
|
|
241
|
+
config[:port] = uri.port if uri.port
|
|
242
|
+
config[:username] = URI.decode_www_form_component(uri.user) if uri.user
|
|
243
|
+
config[:password] = URI.decode_www_form_component(uri.password) if uri.password
|
|
244
|
+
|
|
245
|
+
if uri.path && uri.path.length > 1
|
|
246
|
+
config[:database] = uri.path.delete_prefix('/')
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
if uri.query
|
|
250
|
+
URI.decode_www_form(uri.query).each do |key, value|
|
|
251
|
+
case key
|
|
252
|
+
when 'ssl' then config[:ssl] = (value == 'true')
|
|
253
|
+
when 'debug' then config[:debug] = (value == 'true')
|
|
254
|
+
when 'insecure' then config[:insecure] = (value == 'true')
|
|
255
|
+
when 'open_timeout' then config[:open_timeout] = value.to_i
|
|
256
|
+
when 'read_timeout' then config[:read_timeout] = value.to_i
|
|
257
|
+
when 'write_timeout' then config[:write_timeout] = value.to_i
|
|
258
|
+
when 'keep_alive_timeout' then config[:keep_alive_timeout] = value.to_i
|
|
259
|
+
when 'http_auth' then config[:http_auth] = value
|
|
260
|
+
when 'cluster_name' then config[:cluster_name] = value
|
|
261
|
+
when 'sslca' then config[:sslca] = value
|
|
262
|
+
when 'max_execution_time' then config[:max_execution_time] = value.to_i
|
|
263
|
+
when 'cancel_http_readonly_queries_on_client_close'
|
|
264
|
+
config[:cancel_http_readonly_queries_on_client_close] = (value == 'true')
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
config
|
|
270
|
+
end
|
|
271
|
+
|
|
193
272
|
def extract_limit(sql_type) # :nodoc:
|
|
194
273
|
case sql_type
|
|
195
274
|
when /(Nullable)?\(?String\)?/
|
|
@@ -204,6 +283,10 @@ module ActiveRecord
|
|
|
204
283
|
8
|
|
205
284
|
when /(Nullable)?\(?U?Int128\)?/
|
|
206
285
|
16
|
|
286
|
+
when /(Nullable)?\(?Float32\)?/
|
|
287
|
+
nil
|
|
288
|
+
when /(Nullable)?\(?Float64\)?/
|
|
289
|
+
8
|
|
207
290
|
else
|
|
208
291
|
super
|
|
209
292
|
end
|
|
@@ -254,7 +337,7 @@ module ActiveRecord
|
|
|
254
337
|
Clickhouse::OID::Map.new(sql_type)
|
|
255
338
|
end
|
|
256
339
|
|
|
257
|
-
m.register_type
|
|
340
|
+
m.register_type(%r(JSON)) { Clickhouse::OID::Json.new }
|
|
258
341
|
end
|
|
259
342
|
end
|
|
260
343
|
|
|
@@ -304,7 +387,7 @@ module ActiveRecord
|
|
|
304
387
|
end
|
|
305
388
|
|
|
306
389
|
pk = table_structure(table_name).first
|
|
307
|
-
return ['id'] if pk.present? && pk
|
|
390
|
+
return ['id'] if pk.present? && pk.name == 'id'
|
|
308
391
|
[]
|
|
309
392
|
end
|
|
310
393
|
|
|
@@ -426,9 +509,9 @@ module ActiveRecord
|
|
|
426
509
|
end
|
|
427
510
|
|
|
428
511
|
def change_column_null(table_name, column_name, null, default = nil)
|
|
429
|
-
structure = table_structure(table_name).
|
|
512
|
+
structure = table_structure(table_name).find { |col| col.name == column_name.to_s }
|
|
430
513
|
raise "Column #{column_name} not found in table #{table_name}" if structure.nil?
|
|
431
|
-
change_column table_name, column_name, structure
|
|
514
|
+
change_column table_name, column_name, structure.sql_type.gsub(/(Nullable\()?(.*?)\)?/, '\2'), {null: null, default: default}.compact
|
|
432
515
|
end
|
|
433
516
|
|
|
434
517
|
def change_column_default(table_name, column_name, default)
|
|
@@ -539,8 +622,13 @@ module ActiveRecord
|
|
|
539
622
|
|
|
540
623
|
protected
|
|
541
624
|
|
|
542
|
-
def last_inserted_id(
|
|
543
|
-
|
|
625
|
+
def last_inserted_id(_result)
|
|
626
|
+
# Rails 7.1 persistence.rb:1258:
|
|
627
|
+
# _write_attribute(column, value) # wrote true directly — ugly but no crash
|
|
628
|
+
|
|
629
|
+
# Rails 8.0 persistence.rb:933:
|
|
630
|
+
# _write_attribute(column, type_for_attribute(column).deserialize(value)) # now deserializes first
|
|
631
|
+
nil
|
|
544
632
|
end
|
|
545
633
|
|
|
546
634
|
def change_column_for_alter(table_name, column_name, type, **options)
|
|
@@ -552,9 +640,15 @@ module ActiveRecord
|
|
|
552
640
|
private
|
|
553
641
|
|
|
554
642
|
def connect
|
|
555
|
-
@connection = @connection_parameters[:connection] || Net::HTTP.start(
|
|
556
|
-
|
|
557
|
-
|
|
643
|
+
@connection = @connection_parameters[:connection] || Net::HTTP.start(
|
|
644
|
+
@connection_parameters[:host],
|
|
645
|
+
@connection_parameters[:port],
|
|
646
|
+
use_ssl: @connection_parameters[:ssl],
|
|
647
|
+
verify_mode: @connection_parameters[:insecure] ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER,
|
|
648
|
+
**{ open_timeout: @connection_parameters[:open_timeout] }.compact
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
@connection.ca_file = @connection_parameters[:sslca] if @connection_parameters[:sslca]
|
|
558
652
|
@connection.read_timeout = @connection_parameters[:read_timeout] if @connection_parameters[:read_timeout]
|
|
559
653
|
@connection.write_timeout = @connection_parameters[:write_timeout] if @connection_parameters[:write_timeout]
|
|
560
654
|
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
module Arel # :nodoc: all
|
|
2
|
+
module Nodes
|
|
3
|
+
# Wraps the table source of a JOIN so the ClickHouse visitor renders
|
|
4
|
+
# `<table> FINAL ON ...` for that join. Built by
|
|
5
|
+
# ActiveRecord::Relation#joins_final, which marks the matching join
|
|
6
|
+
# sources after Arel has been constructed.
|
|
7
|
+
class FinalTable < Arel::Nodes::Unary
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
end
|
|
@@ -10,8 +10,7 @@ module Arel
|
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
def aggregate(name, o, collector)
|
|
13
|
-
|
|
14
|
-
if o.expressions.first && o.expressions.first != '*' && !o.expressions.first.is_a?(String) && o.expressions.first.relation&.is_view
|
|
13
|
+
if o.expressions.first && o.expressions.first != '*' && !o.expressions.first.is_a?(String) && o.expressions.first.respond_to?(:relation) && o.expressions.first.relation&.is_view
|
|
15
14
|
super("#{name.downcase}Merge", o, collector)
|
|
16
15
|
else
|
|
17
16
|
super
|
|
@@ -89,6 +88,16 @@ module Arel
|
|
|
89
88
|
collector
|
|
90
89
|
end
|
|
91
90
|
|
|
91
|
+
# Renders the FINAL modifier for a single JOIN source, e.g.
|
|
92
|
+
# `INNER JOIN espm_binaries FINAL ON ...`. The join visitors inherited
|
|
93
|
+
# from ToSql all emit their table via `visit o.left`, so wrapping that
|
|
94
|
+
# table in a FinalTable node is enough to add FINAL to any join type.
|
|
95
|
+
def visit_Arel_Nodes_FinalTable(o, collector)
|
|
96
|
+
collector = visit o.expr, collector
|
|
97
|
+
collector << ' FINAL'
|
|
98
|
+
collector
|
|
99
|
+
end
|
|
100
|
+
|
|
92
101
|
def visit_Arel_Nodes_GroupingSets(o, collector)
|
|
93
102
|
collector << 'GROUPING SETS '
|
|
94
103
|
grouping_array_or_grouping_element(o.expr, collector)
|
|
@@ -100,7 +109,7 @@ module Arel
|
|
|
100
109
|
collector << "SETTINGS "
|
|
101
110
|
o.expr.each_with_index do |(key, value), i|
|
|
102
111
|
collector << ", " if i > 0
|
|
103
|
-
collector << key.to_s
|
|
112
|
+
collector << sanitize_as_setting_name(key).to_s
|
|
104
113
|
collector << " = "
|
|
105
114
|
collector << sanitize_as_setting_value(value)
|
|
106
115
|
end
|
|
@@ -146,7 +155,7 @@ module Arel
|
|
|
146
155
|
|
|
147
156
|
def sanitize_as_setting_name(value)
|
|
148
157
|
return value if Arel::Nodes::SqlLiteral === value
|
|
149
|
-
|
|
158
|
+
value.to_s.gsub(/\W+/, "")
|
|
150
159
|
end
|
|
151
160
|
|
|
152
161
|
private
|
|
@@ -5,7 +5,7 @@ RSpec.configure do |config|
|
|
|
5
5
|
original_connection_config = ActiveRecord::Base.connection_db_config
|
|
6
6
|
ActiveRecord::Base.configurations.configurations.select { |x| x.env_name == Rails.env && x.adapter == 'clickhouse' }.each do |db_config|
|
|
7
7
|
ActiveRecord::Base.establish_connection(db_config)
|
|
8
|
-
ActiveRecord::Base.connection.
|
|
8
|
+
ActiveRecord::Base.connection.execute("TRUNCATE ALL TABLES FROM #{db_config.database}")
|
|
9
9
|
end
|
|
10
10
|
ActiveRecord::Base.establish_connection(original_connection_config)
|
|
11
11
|
end
|
|
@@ -99,7 +99,7 @@ module ClickhouseActiverecord
|
|
|
99
99
|
end
|
|
100
100
|
end
|
|
101
101
|
|
|
102
|
-
indexes = sql.scan(/INDEX \S+
|
|
102
|
+
indexes = sql.scan(/INDEX \S+ .+? TYPE .*? GRANULARITY \d+/)
|
|
103
103
|
if indexes.any?
|
|
104
104
|
tbl.puts ''
|
|
105
105
|
indexes.flatten.map!(&:strip).each do |index|
|
|
@@ -154,22 +154,17 @@ module ClickhouseActiverecord
|
|
|
154
154
|
end
|
|
155
155
|
end
|
|
156
156
|
|
|
157
|
-
def schema_limit(column)
|
|
158
|
-
return nil if column.type == :float
|
|
159
|
-
super
|
|
160
|
-
end
|
|
161
|
-
|
|
162
157
|
def schema_unsigned(column)
|
|
163
158
|
return nil unless column.type == :integer && !simple
|
|
164
159
|
(column.sql_type =~ /(Nullable)?\(?UInt\d+\)?/).nil? ? false : nil
|
|
165
160
|
end
|
|
166
161
|
|
|
167
162
|
def schema_array(column)
|
|
168
|
-
|
|
163
|
+
column.array ? true : nil
|
|
169
164
|
end
|
|
170
165
|
|
|
171
166
|
def schema_map(column)
|
|
172
|
-
if column.sql_type =~ /Map\(
|
|
167
|
+
if column.sql_type =~ /Map\([^,]+,\s*Array\(/
|
|
173
168
|
return :array
|
|
174
169
|
end
|
|
175
170
|
|
|
@@ -34,28 +34,32 @@ module ClickhouseActiverecord
|
|
|
34
34
|
create
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
def structure_dump(*
|
|
37
|
+
def structure_dump(path, *)
|
|
38
38
|
establish_master_connection
|
|
39
39
|
|
|
40
|
+
# get all functions
|
|
41
|
+
functions = connection.execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined' ORDER BY name")['data']
|
|
42
|
+
.flatten
|
|
43
|
+
.map { |function| function.gsub('\\n', "\n") }
|
|
44
|
+
|
|
40
45
|
# get all tables
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
46
|
+
table_defs = connection.execute("SHOW TABLES FROM #{@configuration.database} WHERE name NOT LIKE '.inner_id.%'")['data']
|
|
47
|
+
.flatten
|
|
48
|
+
.map { |name| connection.show_create_table(name, single_line: false).gsub("#{@configuration.database}.", '') }
|
|
44
49
|
|
|
45
|
-
#
|
|
46
|
-
tables.
|
|
50
|
+
# separate views from tables
|
|
51
|
+
views, tables = table_defs.partition { |sql| sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) }
|
|
47
52
|
|
|
48
|
-
#
|
|
49
|
-
|
|
53
|
+
# separate materialized from regular views
|
|
54
|
+
mat_views, views = views.partition { |sql| sql.match(/^CREATE\s+MATERIALIZED\s+VIEW/) }
|
|
50
55
|
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
functions.each do |function|
|
|
54
|
-
file.puts function.gsub('\\n', "\n") + ";\n\n"
|
|
55
|
-
end
|
|
56
|
+
# sort: UDFs -> tables -> materialized views -> views
|
|
57
|
+
ordered_definitions = functions.sort + tables.sort + mat_views.sort + views.sort
|
|
56
58
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
+
# puts to file
|
|
60
|
+
File.open(path, 'w:utf-8') do |file|
|
|
61
|
+
ordered_definitions.each do |sql|
|
|
62
|
+
file.puts "#{sql};\n\n"
|
|
59
63
|
end
|
|
60
64
|
end
|
|
61
65
|
end
|
|
@@ -30,5 +30,15 @@ module ClickhouseActiverecord
|
|
|
30
30
|
Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)
|
|
31
31
|
Arel::SelectManager.prepend(CoreExtensions::Arel::SelectManager)
|
|
32
32
|
Arel::Table.prepend(CoreExtensions::Arel::Table)
|
|
33
|
+
|
|
34
|
+
# Prevent TimeZoneConverter wrapping for Map OID with DateTime subtype.
|
|
35
|
+
# Map(String, DateTime) would crash with "Hash values are not supported"
|
|
36
|
+
# when TimeZoneConverter tries to convert Hash values.
|
|
37
|
+
ActiveRecord::AttributeMethods::TimeZoneConversion::ClassMethods.prepend(Module.new do
|
|
38
|
+
def create_time_zone_conversion_attribute?(name, cast_type)
|
|
39
|
+
return false if cast_type.is_a?(ActiveRecord::ConnectionAdapters::Clickhouse::OID::Map)
|
|
40
|
+
super
|
|
41
|
+
end
|
|
42
|
+
end)
|
|
33
43
|
end
|
|
34
44
|
end
|
|
@@ -3,11 +3,12 @@ module CoreExtensions
|
|
|
3
3
|
module Relation
|
|
4
4
|
|
|
5
5
|
def self.prepended(base)
|
|
6
|
-
base::VALID_UNSCOPING_VALUES << :final << :settings
|
|
6
|
+
base::VALID_UNSCOPING_VALUES << :final << :settings << :joins_final
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
def reverse_order!
|
|
10
|
-
|
|
10
|
+
conn = respond_to?(:lease_connection) ? lease_connection : connection
|
|
11
|
+
return super unless conn.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
11
12
|
|
|
12
13
|
orders = order_values.uniq.reject(&:blank?)
|
|
13
14
|
return super unless orders.empty? && !primary_key
|
|
@@ -90,6 +91,79 @@ module CoreExtensions
|
|
|
90
91
|
@values.fetch(:final, nil)
|
|
91
92
|
end
|
|
92
93
|
|
|
94
|
+
# Apply the FINAL modifier to one or more joined tables. Unlike #final,
|
|
95
|
+
# which only adds FINAL to the query's primary FROM table, #joins_final
|
|
96
|
+
# adds the join(s) and renders FINAL on the joined table so ClickHouse
|
|
97
|
+
# merges those rows before joining.
|
|
98
|
+
# For example:
|
|
99
|
+
#
|
|
100
|
+
# AppControlEvent.final.joins_final(:espm_binary)
|
|
101
|
+
# # SELECT ... FROM espm_app_control_events FINAL
|
|
102
|
+
# # INNER JOIN espm_binaries FINAL ON ...
|
|
103
|
+
#
|
|
104
|
+
# Each argument is an association name and is added as a join (like
|
|
105
|
+
# #joins); FINAL is then rendered on the joined table. Joins are matched
|
|
106
|
+
# by table name, so if the same table is joined more than once every join
|
|
107
|
+
# of that table receives FINAL. An
|
|
108
|
+
# <tt>ActiveRecord::ActiveRecordError</tt> will be raised if the database
|
|
109
|
+
# is not ClickHouse.
|
|
110
|
+
#
|
|
111
|
+
# @param [Array] args
|
|
112
|
+
def joins_final(*args)
|
|
113
|
+
spawn.joins_final!(*args)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# @param [Array] args
|
|
117
|
+
def joins_final!(*args)
|
|
118
|
+
check_command!('FINAL')
|
|
119
|
+
joins!(*args)
|
|
120
|
+
self.joins_final_values |= args
|
|
121
|
+
self
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Like #joins_final, but adds the join(s) as LEFT OUTER JOINs (via
|
|
125
|
+
# #left_outer_joins) and renders FINAL on the joined table. Use this when
|
|
126
|
+
# the joined rows must be preserved even with no match (e.g. an optional
|
|
127
|
+
# catalog lookup) while still merging the joined table with FINAL.
|
|
128
|
+
# For example:
|
|
129
|
+
#
|
|
130
|
+
# AppControlEvent.left_joins_final(:espm_binary)
|
|
131
|
+
# # SELECT ... FROM espm_app_control_events
|
|
132
|
+
# # LEFT OUTER JOIN espm_binaries FINAL ON ...
|
|
133
|
+
#
|
|
134
|
+
# Each argument is an association name and is added as a left outer join
|
|
135
|
+
# (like #left_outer_joins); FINAL is then rendered on the joined table.
|
|
136
|
+
# Joins are matched by table name, so if the same table is joined more
|
|
137
|
+
# than once every join of that table receives FINAL. An
|
|
138
|
+
# <tt>ActiveRecord::ActiveRecordError</tt> will be raised if the database
|
|
139
|
+
# is not ClickHouse.
|
|
140
|
+
#
|
|
141
|
+
# @param [Array] args
|
|
142
|
+
def left_joins_final(*args)
|
|
143
|
+
spawn.left_joins_final!(*args)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# @param [Array] args
|
|
147
|
+
def left_joins_final!(*args)
|
|
148
|
+
check_command!('FINAL')
|
|
149
|
+
left_outer_joins!(*args)
|
|
150
|
+
self.joins_final_values |= args
|
|
151
|
+
self
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def joins_final_values
|
|
155
|
+
@values.fetch(:joins_final, ::ActiveRecord::QueryMethods::FROZEN_EMPTY_ARRAY)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def joins_final_values=(value)
|
|
159
|
+
if ::ActiveRecord::version >= Gem::Version.new('7.2')
|
|
160
|
+
assert_modifiable!
|
|
161
|
+
else
|
|
162
|
+
assert_mutability!
|
|
163
|
+
end
|
|
164
|
+
@values[:joins_final] = value
|
|
165
|
+
end
|
|
166
|
+
|
|
93
167
|
# GROUPING SETS allows you to specify multiple groupings in the GROUP BY clause.
|
|
94
168
|
# Whereas GROUP BY CUBE generates all possible groupings, GROUP BY GROUPING SETS generates only the specified groupings.
|
|
95
169
|
# For example:
|
|
@@ -175,6 +249,50 @@ module CoreExtensions
|
|
|
175
249
|
raise ::ActiveRecord::ActiveRecordError, cmd + ' is a ClickHouse specific query clause' unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
176
250
|
end
|
|
177
251
|
|
|
252
|
+
# Wrap the left (table) side of every join source that matches a
|
|
253
|
+
# requested #joins_final argument in an Arel::Nodes::FinalTable so the
|
|
254
|
+
# ClickHouse visitor renders FINAL on that join. Called before
|
|
255
|
+
# +arel.final!+ so the join source is still the plain JoinSource (and not
|
|
256
|
+
# yet wrapped in the FROM-level Final node).
|
|
257
|
+
def mark_final_joins!(arel)
|
|
258
|
+
names = final_join_table_names
|
|
259
|
+
return if names.empty?
|
|
260
|
+
|
|
261
|
+
arel.join_sources.each do |join|
|
|
262
|
+
next unless join.respond_to?(:left) && join.respond_to?(:left=)
|
|
263
|
+
|
|
264
|
+
left = join.left
|
|
265
|
+
next if left.is_a?(::Arel::Nodes::FinalTable)
|
|
266
|
+
next unless final_join_match?(left, names)
|
|
267
|
+
|
|
268
|
+
join.left = ::Arel::Nodes::FinalTable.new(left)
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Resolve the #joins_final arguments to the set of table names that
|
|
273
|
+
# should carry FINAL. Association names are mapped to their table;
|
|
274
|
+
# anything else is treated as a literal table name.
|
|
275
|
+
def final_join_table_names
|
|
276
|
+
joins_final_values.flatten.filter_map do |arg|
|
|
277
|
+
case arg
|
|
278
|
+
when Symbol, String
|
|
279
|
+
reflection = klass.reflect_on_association(arg.to_sym)
|
|
280
|
+
reflection ? reflection.table_name : arg.to_s
|
|
281
|
+
end
|
|
282
|
+
end.to_set
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# Does the join's table source match one of the requested FINAL names?
|
|
286
|
+
# Handles both Arel::Table (name) and Arel::Nodes::TableAlias (alias name
|
|
287
|
+
# plus the underlying relation's name).
|
|
288
|
+
def final_join_match?(left, names)
|
|
289
|
+
candidates = []
|
|
290
|
+
candidates << left.name if left.respond_to?(:name)
|
|
291
|
+
candidates << left.table_alias if left.respond_to?(:table_alias) && left.table_alias
|
|
292
|
+
candidates << left.relation.name if left.respond_to?(:relation) && left.relation.respond_to?(:name)
|
|
293
|
+
candidates.any? { |c| names.include?(c.to_s) }
|
|
294
|
+
end
|
|
295
|
+
|
|
178
296
|
def build_arel(connection_or_aliases = nil, aliases = nil)
|
|
179
297
|
requirement = Gem::Requirement.new('>= 7.2', '< 8.1')
|
|
180
298
|
|
|
@@ -184,6 +302,7 @@ module CoreExtensions
|
|
|
184
302
|
arel = super(connection_or_aliases)
|
|
185
303
|
end
|
|
186
304
|
|
|
305
|
+
mark_final_joins!(arel) if joins_final_values.present?
|
|
187
306
|
arel.final! if final_value
|
|
188
307
|
arel.limit_by(*@values[:limit_by]) if @values[:limit_by].present?
|
|
189
308
|
arel.settings(settings_values) unless settings_values.empty?
|