clickhouse-activerecord 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.docker/clickhouse/cluster/server1_config.xml +117 -0
- data/.docker/clickhouse/cluster/server2_config.xml +117 -0
- data/.docker/clickhouse/single/config.xml +54 -0
- data/.docker/clickhouse/users.xml +34 -0
- data/.docker/docker-compose.cluster.yml +52 -0
- data/.docker/docker-compose.yml +17 -0
- data/.docker/nginx/local.conf +12 -0
- data/.github/workflows/testing.yml +77 -0
- data/CHANGELOG.md +28 -0
- data/README.md +31 -2
- data/lib/active_record/connection_adapters/clickhouse/oid/uuid.rb +16 -6
- data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb +26 -2
- data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb +17 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +89 -18
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +69 -10
- data/lib/arel/visitors/clickhouse.rb +7 -0
- data/lib/clickhouse-activerecord/schema_dumper.rb +30 -2
- data/lib/clickhouse-activerecord/tasks.rb +22 -6
- data/lib/clickhouse-activerecord/version.rb +1 -1
- data/lib/clickhouse-activerecord.rb +4 -6
- data/lib/core_extensions/active_record/internal_metadata.rb +42 -34
- data/lib/core_extensions/active_record/schema_migration.rb +28 -30
- data/lib/tasks/clickhouse.rake +14 -11
- metadata +12 -4
- /data/{core_extensions → lib/core_extensions}/active_record/migration/command_recorder.rb +0 -0
@@ -94,6 +94,23 @@ module ActiveRecord
|
|
94
94
|
args.each { |name| column(name, kind, **options.except(:limit)) }
|
95
95
|
end
|
96
96
|
end
|
97
|
+
|
98
|
+
class IndexDefinition
|
99
|
+
attr_reader :table, :name, :expression, :type, :granularity, :first, :after, :if_exists, :if_not_exists
|
100
|
+
|
101
|
+
def initialize(table, name, expression, type, granularity, first:, after:, if_exists:, if_not_exists:)
|
102
|
+
@table = table
|
103
|
+
@name = name
|
104
|
+
@expression = expression
|
105
|
+
@type = type
|
106
|
+
@granularity = granularity
|
107
|
+
@first = first
|
108
|
+
@after = after
|
109
|
+
@if_exists = if_exists
|
110
|
+
@if_not_exists = if_not_exists
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
97
114
|
end
|
98
115
|
end
|
99
116
|
end
|
@@ -6,6 +6,8 @@ module ActiveRecord
|
|
6
6
|
module ConnectionAdapters
|
7
7
|
module Clickhouse
|
8
8
|
module SchemaStatements
|
9
|
+
DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
|
10
|
+
|
9
11
|
def execute(sql, name = nil, settings: {})
|
10
12
|
do_execute(sql, name, settings: settings)
|
11
13
|
end
|
@@ -33,13 +35,20 @@ module ActiveRecord
|
|
33
35
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update
|
34
36
|
def exec_update(_sql, _name = nil, _binds = [])
|
35
37
|
do_execute(_sql, _name, format: nil)
|
36
|
-
|
38
|
+
0
|
37
39
|
end
|
38
40
|
|
39
41
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/delete
|
40
42
|
def exec_delete(_sql, _name = nil, _binds = [])
|
41
|
-
|
42
|
-
|
43
|
+
log(_sql, "#{adapter_name} #{_name}") do
|
44
|
+
res = request(_sql)
|
45
|
+
begin
|
46
|
+
data = JSON.parse(res.header['x-clickhouse-summary'])
|
47
|
+
data['result_rows'].to_i
|
48
|
+
rescue JSONError
|
49
|
+
0
|
50
|
+
end
|
51
|
+
end
|
43
52
|
end
|
44
53
|
|
45
54
|
def tables(name = nil)
|
@@ -48,6 +57,16 @@ module ActiveRecord
|
|
48
57
|
result['data'].flatten
|
49
58
|
end
|
50
59
|
|
60
|
+
def functions
|
61
|
+
result = do_system_execute("SELECT name FROM system.functions WHERE origin = 'SQLUserDefined'")
|
62
|
+
return [] if result.nil?
|
63
|
+
result['data'].flatten
|
64
|
+
end
|
65
|
+
|
66
|
+
def show_create_function(function)
|
67
|
+
do_execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined' AND name = '#{function}'", format: nil)
|
68
|
+
end
|
69
|
+
|
51
70
|
def table_options(table)
|
52
71
|
sql = show_create_table(table)
|
53
72
|
{ options: sql.gsub(/^(?:.*?)(?:ENGINE = (.*?))?( AS SELECT .*?)?$/, '\\1').presence, as: sql.match(/^CREATE (?:.*?) AS (SELECT .*?)$/).try(:[], 1) }.compact
|
@@ -58,25 +77,29 @@ module ActiveRecord
|
|
58
77
|
[]
|
59
78
|
end
|
60
79
|
|
80
|
+
def add_index_options(table_name, expression, **options)
|
81
|
+
options.assert_valid_keys(:name, :type, :granularity, :first, :after, :if_not_exists, :if_exists)
|
82
|
+
|
83
|
+
validate_index_length!(table_name, options[:name])
|
84
|
+
|
85
|
+
IndexDefinition.new(table_name, options[:name], expression, options[:type], options[:granularity], first: options[:first], after: options[:after], if_not_exists: options[:if_not_exists], if_exists: options[:if_exists])
|
86
|
+
end
|
87
|
+
|
61
88
|
def data_sources
|
62
89
|
tables
|
63
90
|
end
|
64
91
|
|
65
92
|
def do_system_execute(sql, name = nil)
|
66
93
|
log_with_debug(sql, "#{adapter_name} #{name}") do
|
67
|
-
res =
|
68
|
-
|
69
|
-
process_response(res)
|
94
|
+
res = request(sql, DEFAULT_RESPONSE_FORMAT)
|
95
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT, sql)
|
70
96
|
end
|
71
97
|
end
|
72
98
|
|
73
|
-
def do_execute(sql, name = nil, format:
|
99
|
+
def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {})
|
74
100
|
log(sql, "#{adapter_name} #{name}") do
|
75
|
-
|
76
|
-
|
77
|
-
res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
78
|
-
|
79
|
-
process_response(res)
|
101
|
+
res = request(sql, format, settings)
|
102
|
+
process_response(res, format, sql)
|
80
103
|
end
|
81
104
|
end
|
82
105
|
|
@@ -96,7 +119,7 @@ module ActiveRecord
|
|
96
119
|
if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
|
97
120
|
raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
|
98
121
|
end
|
99
|
-
do_execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
122
|
+
do_execute(insert_versions_sql(inserting), nil, format: nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
100
123
|
end
|
101
124
|
end
|
102
125
|
|
@@ -112,17 +135,28 @@ module ActiveRecord
|
|
112
135
|
|
113
136
|
private
|
114
137
|
|
138
|
+
# Make HTTP request to ClickHouse server
|
139
|
+
# @param [String] sql
|
140
|
+
# @param [String, nil] format
|
141
|
+
# @param [Hash] settings
|
142
|
+
# @return [Net::HTTPResponse]
|
143
|
+
def request(sql, format = nil, settings = {})
|
144
|
+
formatted_sql = apply_format(sql, format)
|
145
|
+
request_params = @connection_config || {}
|
146
|
+
@connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
147
|
+
end
|
148
|
+
|
115
149
|
def apply_format(sql, format)
|
116
150
|
format ? "#{sql} FORMAT #{format}" : sql
|
117
151
|
end
|
118
152
|
|
119
|
-
def process_response(res)
|
153
|
+
def process_response(res, format, sql = nil)
|
120
154
|
case res.code.to_i
|
121
155
|
when 200
|
122
156
|
if res.body.to_s.include?("DB::Exception")
|
123
|
-
raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}"
|
157
|
+
raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}#{sql ? "\nQuery: #{sql}" : ''}"
|
124
158
|
else
|
125
|
-
res.body
|
159
|
+
format_body_response(res.body, format)
|
126
160
|
end
|
127
161
|
else
|
128
162
|
case res.body
|
@@ -168,8 +202,7 @@ module ActiveRecord
|
|
168
202
|
|
169
203
|
return data unless data.empty?
|
170
204
|
|
171
|
-
raise ActiveRecord::StatementInvalid,
|
172
|
-
"Could not find table '#{table_name}'"
|
205
|
+
raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
|
173
206
|
end
|
174
207
|
alias column_definitions table_structure
|
175
208
|
|
@@ -203,6 +236,44 @@ module ActiveRecord
|
|
203
236
|
def has_default_function?(default_value, default) # :nodoc:
|
204
237
|
!default_value && (%r{\w+\(.*\)} === default)
|
205
238
|
end
|
239
|
+
|
240
|
+
def format_body_response(body, format)
|
241
|
+
return body if body.blank?
|
242
|
+
|
243
|
+
case format
|
244
|
+
when 'JSONCompact'
|
245
|
+
format_from_json_compact(body)
|
246
|
+
when 'JSONCompactEachRowWithNamesAndTypes'
|
247
|
+
format_from_json_compact_each_row_with_names_and_types(body)
|
248
|
+
else
|
249
|
+
body
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
def format_from_json_compact(body)
|
254
|
+
parse_json_payload(body)
|
255
|
+
end
|
256
|
+
|
257
|
+
def format_from_json_compact_each_row_with_names_and_types(body)
|
258
|
+
rows = body.split("\n").map { |row| parse_json_payload(row) }
|
259
|
+
names, types, *data = rows
|
260
|
+
|
261
|
+
meta = names.zip(types).map do |name, type|
|
262
|
+
{
|
263
|
+
'name' => name,
|
264
|
+
'type' => type
|
265
|
+
}
|
266
|
+
end
|
267
|
+
|
268
|
+
{
|
269
|
+
'meta' => meta,
|
270
|
+
'data' => data
|
271
|
+
}
|
272
|
+
end
|
273
|
+
|
274
|
+
def parse_json_payload(payload)
|
275
|
+
JSON.parse(payload, decimal_class: BigDecimal)
|
276
|
+
end
|
206
277
|
end
|
207
278
|
end
|
208
279
|
end
|
@@ -73,10 +73,11 @@ module ActiveRecord
|
|
73
73
|
def is_view=(value)
|
74
74
|
@is_view = value
|
75
75
|
end
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
76
|
+
|
77
|
+
def _delete_record(constraints)
|
78
|
+
raise ActiveRecord::ActiveRecordError.new('Deleting a row is not possible without a primary key') unless self.primary_key
|
79
|
+
super
|
80
|
+
end
|
80
81
|
end
|
81
82
|
end
|
82
83
|
|
@@ -148,6 +149,10 @@ module ActiveRecord
|
|
148
149
|
!native_database_types[type].nil?
|
149
150
|
end
|
150
151
|
|
152
|
+
def supports_indexes_in_create?
|
153
|
+
true
|
154
|
+
end
|
155
|
+
|
151
156
|
class << self
|
152
157
|
def extract_limit(sql_type) # :nodoc:
|
153
158
|
case sql_type
|
@@ -267,7 +272,7 @@ module ActiveRecord
|
|
267
272
|
sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
|
268
273
|
log_with_debug(sql, adapter_name) do
|
269
274
|
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
270
|
-
process_response(res)
|
275
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
271
276
|
end
|
272
277
|
end
|
273
278
|
|
@@ -307,12 +312,23 @@ module ActiveRecord
|
|
307
312
|
end
|
308
313
|
end
|
309
314
|
|
315
|
+
def create_function(name, body)
|
316
|
+
fd = "CREATE FUNCTION #{apply_cluster(quote_table_name(name))} AS #{body}"
|
317
|
+
do_execute(fd, format: nil)
|
318
|
+
end
|
319
|
+
|
310
320
|
# Drops a ClickHouse database.
|
311
321
|
def drop_database(name) #:nodoc:
|
312
322
|
sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
|
313
323
|
log_with_debug(sql, adapter_name) do
|
314
324
|
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
315
|
-
process_response(res)
|
325
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
329
|
+
def drop_functions
|
330
|
+
functions.each do |function|
|
331
|
+
drop_function(function)
|
316
332
|
end
|
317
333
|
end
|
318
334
|
|
@@ -335,6 +351,16 @@ module ActiveRecord
|
|
335
351
|
end
|
336
352
|
end
|
337
353
|
|
354
|
+
def drop_function(name, options = {})
|
355
|
+
query = "DROP FUNCTION"
|
356
|
+
query = "#{query} IF EXISTS " if options[:if_exists]
|
357
|
+
query = "#{query} #{quote_table_name(name)}"
|
358
|
+
query = apply_cluster(query)
|
359
|
+
query = "#{query} SYNC" if options[:sync]
|
360
|
+
|
361
|
+
do_execute(query, format: nil)
|
362
|
+
end
|
363
|
+
|
338
364
|
def add_column(table_name, column_name, type, **options)
|
339
365
|
return if options[:if_not_exists] == true && column_exists?(table_name, column_name, type)
|
340
366
|
|
@@ -349,8 +375,8 @@ module ActiveRecord
|
|
349
375
|
execute("ALTER TABLE #{quote_table_name(table_name)} #{remove_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
|
350
376
|
end
|
351
377
|
|
352
|
-
def change_column(table_name, column_name, type, options
|
353
|
-
result = do_execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
|
378
|
+
def change_column(table_name, column_name, type, **options)
|
379
|
+
result = do_execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, **options)}", nil, settings: {wait_end_of_query: 1, send_progress_in_http_headers: 1})
|
354
380
|
raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
|
355
381
|
end
|
356
382
|
|
@@ -364,6 +390,39 @@ module ActiveRecord
|
|
364
390
|
change_column table_name, column_name, nil, {default: default}.compact
|
365
391
|
end
|
366
392
|
|
393
|
+
# Adds index description to tables metadata
|
394
|
+
# @link https://clickhouse.com/docs/en/sql-reference/statements/alter/skipping-index
|
395
|
+
def add_index(table_name, expression, **options)
|
396
|
+
index = add_index_options(apply_cluster(table_name), expression, **options)
|
397
|
+
execute schema_creation.accept(CreateIndexDefinition.new(index))
|
398
|
+
end
|
399
|
+
|
400
|
+
# Removes index description from tables metadata and deletes index files from disk
|
401
|
+
def remove_index(table_name, name)
|
402
|
+
query = apply_cluster("ALTER TABLE #{quote_table_name(table_name)}")
|
403
|
+
execute "#{query} DROP INDEX #{quote_column_name(name)}"
|
404
|
+
end
|
405
|
+
|
406
|
+
# Rebuilds the secondary index name for the specified partition_name
|
407
|
+
def rebuild_index(table_name, name, if_exists: false, partition: nil)
|
408
|
+
query = [apply_cluster("ALTER TABLE #{quote_table_name(table_name)}")]
|
409
|
+
query << 'MATERIALIZE INDEX'
|
410
|
+
query << 'IF EXISTS' if if_exists
|
411
|
+
query << quote_column_name(name)
|
412
|
+
query << "IN PARTITION #{quote_column_name(partition)}" if partition
|
413
|
+
execute query.join(' ')
|
414
|
+
end
|
415
|
+
|
416
|
+
# Deletes the secondary index files from disk without removing description
|
417
|
+
def clear_index(table_name, name, if_exists: false, partition: nil)
|
418
|
+
query = [apply_cluster("ALTER TABLE #{quote_table_name(table_name)}")]
|
419
|
+
query << 'CLEAR INDEX'
|
420
|
+
query << 'IF EXISTS' if if_exists
|
421
|
+
query << quote_column_name(name)
|
422
|
+
query << "IN PARTITION #{quote_column_name(partition)}" if partition
|
423
|
+
execute query.join(' ')
|
424
|
+
end
|
425
|
+
|
367
426
|
def cluster
|
368
427
|
@config[:cluster_name]
|
369
428
|
end
|
@@ -419,9 +478,9 @@ module ActiveRecord
|
|
419
478
|
result
|
420
479
|
end
|
421
480
|
|
422
|
-
def change_column_for_alter(table_name, column_name, type, options
|
481
|
+
def change_column_for_alter(table_name, column_name, type, **options)
|
423
482
|
td = create_table_definition(table_name)
|
424
|
-
cd = td.new_column_definition(column_name, type, options)
|
483
|
+
cd = td.new_column_definition(column_name, type, **options)
|
425
484
|
schema_creation.accept(ChangeColumnDefinition.new(cd, column_name))
|
426
485
|
end
|
427
486
|
|
@@ -13,6 +13,13 @@ module Arel
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
+
# https://clickhouse.com/docs/en/sql-reference/statements/delete
|
17
|
+
# DELETE and UPDATE in ClickHouse working only without table name
|
18
|
+
def visit_Arel_Attributes_Attribute(o, collector)
|
19
|
+
collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ')
|
20
|
+
collector << quote_column_name(o.name)
|
21
|
+
end
|
22
|
+
|
16
23
|
def visit_Arel_Nodes_SelectOptions(o, collector)
|
17
24
|
maybe_visit o.settings, super
|
18
25
|
end
|
@@ -34,8 +34,12 @@ HEADER
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def tables(stream)
|
37
|
-
|
37
|
+
functions = @connection.functions
|
38
|
+
functions.each do |function|
|
39
|
+
function(function, stream)
|
40
|
+
end
|
38
41
|
|
42
|
+
sorted_tables = @connection.tables.sort {|a,b| @connection.show_create_table(a).match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : a <=> b }
|
39
43
|
sorted_tables.each do |table_name|
|
40
44
|
table(table_name, stream) unless ignored?(table_name)
|
41
45
|
end
|
@@ -104,7 +108,13 @@ HEADER
|
|
104
108
|
end
|
105
109
|
end
|
106
110
|
|
107
|
-
|
111
|
+
indexes = sql.scan(/INDEX \S+ \S+ TYPE .*? GRANULARITY \d+/)
|
112
|
+
if indexes.any?
|
113
|
+
tbl.puts ''
|
114
|
+
indexes.flatten.map!(&:strip).each do |index|
|
115
|
+
tbl.puts " t.index #{index_parts(index).join(', ')}"
|
116
|
+
end
|
117
|
+
end
|
108
118
|
|
109
119
|
tbl.puts " end"
|
110
120
|
tbl.puts
|
@@ -119,6 +129,13 @@ HEADER
|
|
119
129
|
end
|
120
130
|
end
|
121
131
|
|
132
|
+
def function(function, stream)
|
133
|
+
stream.puts " # FUNCTION: #{function}"
|
134
|
+
sql = @connection.show_create_function(function)
|
135
|
+
stream.puts " # SQL: #{sql}" if sql
|
136
|
+
stream.puts " create_function \"#{function}\", \"#{sql.gsub(/^CREATE FUNCTION (.*?) AS/, '').strip}\"" if sql
|
137
|
+
end
|
138
|
+
|
122
139
|
def format_options(options)
|
123
140
|
if options && options[:options]
|
124
141
|
options[:options] = options[:options].gsub(/^Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "\\1(\\2)")
|
@@ -154,5 +171,16 @@ HEADER
|
|
154
171
|
spec[:array] = schema_array(column)
|
155
172
|
spec.merge(super).compact
|
156
173
|
end
|
174
|
+
|
175
|
+
def index_parts(index)
|
176
|
+
idx = index.match(/^INDEX (?<name>\S+) (?<expr>.*?) TYPE (?<type>.*?) GRANULARITY (?<granularity>\d+)$/)
|
177
|
+
index_parts = [
|
178
|
+
format_index_parts(idx['expr']),
|
179
|
+
"name: #{format_index_parts(idx['name'])}",
|
180
|
+
"type: #{format_index_parts(idx['type'])}",
|
181
|
+
]
|
182
|
+
index_parts << "granularity: #{idx['granularity']}" if idx['granularity']
|
183
|
+
index_parts
|
184
|
+
end
|
157
185
|
end
|
158
186
|
end
|
@@ -5,12 +5,12 @@ module ClickhouseActiverecord
|
|
5
5
|
delegate :connection, :establish_connection, to: ActiveRecord::Base
|
6
6
|
|
7
7
|
def initialize(configuration)
|
8
|
-
@configuration = configuration
|
8
|
+
@configuration = configuration
|
9
9
|
end
|
10
10
|
|
11
11
|
def create
|
12
12
|
establish_master_connection
|
13
|
-
connection.create_database @configuration
|
13
|
+
connection.create_database @configuration.database
|
14
14
|
rescue ActiveRecord::StatementInvalid => e
|
15
15
|
if e.cause.to_s.include?('already exists')
|
16
16
|
raise ActiveRecord::DatabaseAlreadyExists
|
@@ -21,7 +21,7 @@ module ClickhouseActiverecord
|
|
21
21
|
|
22
22
|
def drop
|
23
23
|
establish_master_connection
|
24
|
-
connection.drop_database @configuration
|
24
|
+
connection.drop_database @configuration.database
|
25
25
|
end
|
26
26
|
|
27
27
|
def purge
|
@@ -31,12 +31,28 @@ module ClickhouseActiverecord
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def structure_dump(*args)
|
34
|
-
|
34
|
+
establish_master_connection
|
35
|
+
|
36
|
+
# get all tables
|
37
|
+
tables = connection.execute("SHOW TABLES FROM #{@configuration.database} WHERE name NOT LIKE '.inner_id.%'")['data'].flatten.map do |table|
|
38
|
+
next if %w[schema_migrations ar_internal_metadata].include?(table)
|
39
|
+
connection.show_create_table(table).gsub("#{@configuration.database}.", '')
|
40
|
+
end.compact
|
41
|
+
|
42
|
+
# sort view to last
|
43
|
+
tables.sort_by! {|table| table.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : 0}
|
35
44
|
|
45
|
+
# get all functions
|
46
|
+
functions = connection.execute("SELECT create_query FROM system.functions WHERE origin = 'SQLUserDefined'")['data'].flatten
|
47
|
+
|
48
|
+
# put to file
|
36
49
|
File.open(args.first, 'w:utf-8') do |file|
|
50
|
+
functions.each do |function|
|
51
|
+
file.puts function + ";\n\n"
|
52
|
+
end
|
53
|
+
|
37
54
|
tables.each do |table|
|
38
|
-
|
39
|
-
file.puts connection.execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first).gsub("#{@configuration['database']}.", '') + ";\n\n"
|
55
|
+
file.puts table + ";\n\n"
|
40
56
|
end
|
41
57
|
end
|
42
58
|
end
|
@@ -5,15 +5,12 @@ require 'active_record/connection_adapters/clickhouse_adapter'
|
|
5
5
|
require 'core_extensions/active_record/internal_metadata'
|
6
6
|
require 'core_extensions/active_record/relation'
|
7
7
|
require 'core_extensions/active_record/schema_migration'
|
8
|
-
|
8
|
+
require 'core_extensions/active_record/migration/command_recorder'
|
9
9
|
require 'core_extensions/arel/nodes/select_core'
|
10
10
|
require 'core_extensions/arel/nodes/select_statement'
|
11
11
|
require 'core_extensions/arel/select_manager'
|
12
12
|
require 'core_extensions/arel/table'
|
13
13
|
|
14
|
-
require_relative '../core_extensions/active_record/migration/command_recorder'
|
15
|
-
ActiveRecord::Migration::CommandRecorder.include CoreExtensions::ActiveRecord::Migration::CommandRecorder
|
16
|
-
|
17
14
|
if defined?(Rails::Railtie)
|
18
15
|
require 'clickhouse-activerecord/railtie'
|
19
16
|
require 'clickhouse-activerecord/schema'
|
@@ -24,9 +21,10 @@ end
|
|
24
21
|
|
25
22
|
module ClickhouseActiverecord
|
26
23
|
def self.load
|
27
|
-
ActiveRecord::InternalMetadata.
|
24
|
+
ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata)
|
25
|
+
ActiveRecord::Migration::CommandRecorder.include(CoreExtensions::ActiveRecord::Migration::CommandRecorder)
|
28
26
|
ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
|
29
|
-
ActiveRecord::SchemaMigration.
|
27
|
+
ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration)
|
30
28
|
|
31
29
|
Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore)
|
32
30
|
Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)
|
@@ -1,46 +1,54 @@
|
|
1
1
|
module CoreExtensions
|
2
2
|
module ActiveRecord
|
3
3
|
module InternalMetadata
|
4
|
-
module ClassMethods
|
5
4
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
5
|
+
def create_table
|
6
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
7
|
+
return if table_exists? || !enabled?
|
8
|
+
|
9
|
+
key_options = connection.internal_string_options_for_primary_key
|
10
|
+
table_options = {
|
11
|
+
id: false,
|
12
|
+
options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
|
13
|
+
if_not_exists: true
|
14
|
+
}
|
15
|
+
full_config = connection.instance_variable_get(:@config) || {}
|
16
|
+
|
17
|
+
if full_config[:distributed_service_tables]
|
18
|
+
table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
|
12
19
|
|
13
|
-
|
14
|
-
|
20
|
+
distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
|
21
|
+
else
|
22
|
+
distributed_suffix = ''
|
15
23
|
end
|
16
24
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
key_options = connection.internal_string_options_for_primary_key
|
22
|
-
table_options = {
|
23
|
-
id: false,
|
24
|
-
options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '',
|
25
|
-
if_not_exists: true
|
26
|
-
}
|
27
|
-
full_config = connection.instance_variable_get(:@config) || {}
|
28
|
-
|
29
|
-
if full_config[:distributed_service_tables]
|
30
|
-
table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
|
31
|
-
|
32
|
-
distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
|
33
|
-
else
|
34
|
-
distributed_suffix = ''
|
35
|
-
end
|
36
|
-
|
37
|
-
connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
|
38
|
-
t.string :key, **key_options
|
39
|
-
t.string :value
|
40
|
-
t.timestamps
|
41
|
-
end
|
25
|
+
connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
|
26
|
+
t.string :key, **key_options
|
27
|
+
t.string :value
|
28
|
+
t.timestamps
|
42
29
|
end
|
43
30
|
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def update_entry(key, new_value)
|
35
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
36
|
+
|
37
|
+
create_entry(key, new_value)
|
38
|
+
end
|
39
|
+
|
40
|
+
def select_entry(key)
|
41
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
42
|
+
|
43
|
+
sm = ::Arel::SelectManager.new(arel_table)
|
44
|
+
sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
|
45
|
+
sm.project(::Arel.star)
|
46
|
+
sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
|
47
|
+
sm.order(arel_table[primary_key].asc)
|
48
|
+
sm.limit = 1
|
49
|
+
|
50
|
+
connection.select_one(sm, "#{self.class} Load")
|
51
|
+
end
|
44
52
|
end
|
45
53
|
end
|
46
54
|
end
|