activerecord-clickhouse-adapter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +19 -0
- data/LICENSE +21 -0
- data/README.md +313 -0
- data/lib/active_record/connection_adapters/clickhouse/database_statements.rb +464 -0
- data/lib/active_record/connection_adapters/clickhouse/error_translation.rb +52 -0
- data/lib/active_record/connection_adapters/clickhouse/gem_version.rb +9 -0
- data/lib/active_record/connection_adapters/clickhouse/http_connection.rb +233 -0
- data/lib/active_record/connection_adapters/clickhouse/querying.rb +417 -0
- data/lib/active_record/connection_adapters/clickhouse/quoting.rb +92 -0
- data/lib/active_record/connection_adapters/clickhouse/row_binary.rb +190 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb +146 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_dumper.rb +201 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +722 -0
- data/lib/active_record/connection_adapters/clickhouse/type_parser.rb +207 -0
- data/lib/active_record/connection_adapters/clickhouse/types.rb +238 -0
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +126 -0
- data/lib/active_record/tasks/clickhouse_database_tasks.rb +91 -0
- data/lib/activerecord-clickhouse-adapter.rb +21 -0
- data/lib/arel/visitors/clickhouse.rb +172 -0
- metadata +82 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ipaddr"
|
|
4
|
+
|
|
5
|
+
module ActiveRecord
|
|
6
|
+
module ConnectionAdapters
|
|
7
|
+
module ClickHouse
|
|
8
|
+
module Quoting
|
|
9
|
+
extend ActiveSupport::Concern
|
|
10
|
+
|
|
11
|
+
# disallow_raw_sql! vets order/pluck arguments against these; the abstract
|
|
12
|
+
# matchers reject backtick-quoted names — this adapter's own quoting style —
|
|
13
|
+
# so they admit `table`.`column` exactly as MySQL's do.
|
|
14
|
+
COLUMN_NAME_MATCHER = /
|
|
15
|
+
\A
|
|
16
|
+
(
|
|
17
|
+
(?:
|
|
18
|
+
# `table_name`.`column_name` | function(one or no argument)
|
|
19
|
+
((?:\w+\.|`\w+`\.)?(?:\w+|`\w+`) | \w+\((?:|\g<2>)\))
|
|
20
|
+
)
|
|
21
|
+
(?:(?:\s+AS)?\s+(?:\w+|`\w+`))?
|
|
22
|
+
)
|
|
23
|
+
(?:\s*,\s*\g<1>)*
|
|
24
|
+
\z
|
|
25
|
+
/ix
|
|
26
|
+
|
|
27
|
+
COLUMN_NAME_WITH_ORDER_MATCHER = /
|
|
28
|
+
\A
|
|
29
|
+
(
|
|
30
|
+
(?:
|
|
31
|
+
# `table_name`.`column_name` | function(one or no argument)
|
|
32
|
+
((?:\w+\.|`\w+`\.)?(?:\w+|`\w+`) | \w+\((?:|\g<2>)\))
|
|
33
|
+
)
|
|
34
|
+
(?:\s+ASC|\s+DESC)?
|
|
35
|
+
(?:\s+NULLS\s+(?:FIRST|LAST))?
|
|
36
|
+
)
|
|
37
|
+
(?:\s*,\s*\g<1>)*
|
|
38
|
+
\z
|
|
39
|
+
/ix
|
|
40
|
+
|
|
41
|
+
class_methods do
|
|
42
|
+
def quote_column_name(name)
|
|
43
|
+
"`#{name.to_s.gsub("`", "``")}`"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def quote_table_name(name)
|
|
47
|
+
name.to_s.split(".").map { |part| quote_column_name(part) }.join(".")
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def column_name_matcher = COLUMN_NAME_MATCHER
|
|
51
|
+
|
|
52
|
+
def column_name_with_order_matcher = COLUMN_NAME_WITH_ORDER_MATCHER
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Control characters travel as escape sequences, not raw bytes: DDL passes
|
|
56
|
+
# through String#squish (which would collapse a raw newline inside a quoted
|
|
57
|
+
# literal) and the server echoes them back escaped anyway.
|
|
58
|
+
QUOTED_STRING_ESCAPES = {
|
|
59
|
+
"\\" => "\\\\", "'" => "\\'",
|
|
60
|
+
"\n" => "\\n", "\r" => "\\r", "\t" => "\\t", "\0" => "\\0"
|
|
61
|
+
}.freeze
|
|
62
|
+
|
|
63
|
+
def quote_string(string)
|
|
64
|
+
string.gsub(/[\\'\n\r\t\0]/, QUOTED_STRING_ESCAPES)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# DateTime64 stores an epoch and the server parses naive strings in its own
|
|
68
|
+
# timezone (UTC here); params reject offsets outright (code 457, PLAN.md §2).
|
|
69
|
+
# UTC is therefore the only faithful wire encoding, whatever default_timezone says.
|
|
70
|
+
def quoted_date(value)
|
|
71
|
+
value = value.getutc if value.acts_like?(:time) && !value.utc?
|
|
72
|
+
result = value.to_fs(:db)
|
|
73
|
+
value.respond_to?(:usec) && value.usec.positive? ? "#{result}.#{format("%06d", value.usec)}" : result
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def quoted_true = "true"
|
|
77
|
+
def quoted_false = "false"
|
|
78
|
+
def unquoted_true = true
|
|
79
|
+
def unquoted_false = false
|
|
80
|
+
|
|
81
|
+
def quote(value)
|
|
82
|
+
case value
|
|
83
|
+
when Array then "[#{value.map { |item| quote(item) }.join(", ")}]"
|
|
84
|
+
when Hash then "{#{value.map { |key, item| "#{quote(key)}: #{quote(item)}" }.join(", ")}}"
|
|
85
|
+
when IPAddr then "'#{quote_string(value.to_s)}'"
|
|
86
|
+
else super
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
require "date"
|
|
5
|
+
require "ipaddr"
|
|
6
|
+
|
|
7
|
+
require "active_record/connection_adapters/clickhouse/type_parser"
|
|
8
|
+
|
|
9
|
+
module ActiveRecord
|
|
10
|
+
module ConnectionAdapters
|
|
11
|
+
module ClickHouse
|
|
12
|
+
# Decodes RowBinaryWithNamesAndTypes bodies: a varint column count, the names,
|
|
13
|
+
# the type strings, then rows of packed values (probed 2026-07-13, PLAN.md §2).
|
|
14
|
+
# Yields the same Ruby values the JSON casters produce, so the cast layer treats
|
|
15
|
+
# both wires identically. Types without a binary decoder raise Undecodable and
|
|
16
|
+
# the connection retries the query on the JSON wire.
|
|
17
|
+
class RowBinary
|
|
18
|
+
Undecodable = Class.new(StandardError) # rubocop:disable Style/EmptyClassDefinition
|
|
19
|
+
|
|
20
|
+
INTEGER_FORMATS = {
|
|
21
|
+
"Int8" => ["c", 1], "Int16" => ["s<", 2], "Int32" => ["l<", 4], "Int64" => ["q<", 8],
|
|
22
|
+
"UInt8" => ["C", 1], "UInt16" => ["S<", 2], "UInt32" => ["L<", 4], "UInt64" => ["Q<", 8]
|
|
23
|
+
}.freeze
|
|
24
|
+
|
|
25
|
+
# Storage width is decided by precision; the server normalizes every Decimal
|
|
26
|
+
# alias to Decimal(precision, scale) in the type header.
|
|
27
|
+
DECIMAL_BYTES = [[9, 4], [18, 8], [38, 16], [76, 32]].freeze
|
|
28
|
+
|
|
29
|
+
UNIX_EPOCH_JULIAN_DAY = Date.new(1970, 1, 1).jd
|
|
30
|
+
|
|
31
|
+
def self.decode(body) = new(body).decode
|
|
32
|
+
|
|
33
|
+
def initialize(body)
|
|
34
|
+
@body = body
|
|
35
|
+
@position = 0
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def decode
|
|
39
|
+
return [[], [], []] if @body.empty?
|
|
40
|
+
|
|
41
|
+
names = Array.new(read_varint) { read_string }
|
|
42
|
+
types = Array.new(names.length) { read_string }
|
|
43
|
+
decoders = types.map { |type| value_decoder(TypeParser.parse(type)) }
|
|
44
|
+
rows = []
|
|
45
|
+
rows << decoders.map(&:call) until end_of_body?
|
|
46
|
+
[names, types, rows]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def end_of_body? = @position >= @body.bytesize
|
|
52
|
+
|
|
53
|
+
def value_decoder(node) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/PerceivedComplexity
|
|
54
|
+
case node.name
|
|
55
|
+
when "Nullable" then nullable_decoder(node)
|
|
56
|
+
when "LowCardinality" then value_decoder(node.args.fetch(0))
|
|
57
|
+
when "SimpleAggregateFunction" then value_decoder(node.args.fetch(1))
|
|
58
|
+
when "Array" then array_decoder(node)
|
|
59
|
+
when "Map" then map_decoder(node)
|
|
60
|
+
when "Tuple" then tuple_decoder(node)
|
|
61
|
+
# output_format_binary_write_json_as_string=1 delivers JSON columns as String.
|
|
62
|
+
when "String", "JSON" then -> { read_string }
|
|
63
|
+
when "FixedString" then -> { read_bytes(node.args.fetch(0)) }
|
|
64
|
+
when "Bool" then -> { read_unsigned(1) != 0 }
|
|
65
|
+
when "Enum8" then enum_decoder(node, 1)
|
|
66
|
+
when "Enum16" then enum_decoder(node, 2)
|
|
67
|
+
when "UUID" then -> { read_uuid }
|
|
68
|
+
when "IPv4" then -> { IPAddr.new_ntoh([read_unsigned(4)].pack("N")) }
|
|
69
|
+
when "IPv6" then -> { IPAddr.new_ntoh(@body.byteslice(advance(16), 16)) }
|
|
70
|
+
when "Date" then -> { Date.jd(UNIX_EPOCH_JULIAN_DAY + read_unsigned(2)) }
|
|
71
|
+
when "Date32" then -> { Date.jd(UNIX_EPOCH_JULIAN_DAY + read_signed(4)) }
|
|
72
|
+
when "DateTime" then -> { represent_instant(Time.at(read_unsigned(4))) }
|
|
73
|
+
when "DateTime64" then datetime64_decoder(node)
|
|
74
|
+
when "Float32" then -> { @body.unpack1("e", offset: advance(4)) }
|
|
75
|
+
when "Float64" then -> { @body.unpack1("E", offset: advance(8)) }
|
|
76
|
+
when "Decimal" then decimal_decoder(node)
|
|
77
|
+
when "Int128", "Int256" then -> { read_signed(node.name == "Int128" ? 16 : 32) }
|
|
78
|
+
when "UInt128", "UInt256" then -> { read_unsigned(node.name == "UInt128" ? 16 : 32) }
|
|
79
|
+
when "Nothing" then -> { advance(1) && nil }
|
|
80
|
+
else
|
|
81
|
+
INTEGER_FORMATS.key?(node.name) ? integer_decoder(node) : raise_undecodable(node)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def raise_undecodable(node)
|
|
86
|
+
raise Undecodable, "no RowBinary decoder for ClickHouse type #{node.name}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def integer_decoder(node)
|
|
90
|
+
format, size = INTEGER_FORMATS.fetch(node.name)
|
|
91
|
+
-> { @body.unpack1(format, offset: advance(size)) }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def nullable_decoder(node)
|
|
95
|
+
inner = value_decoder(node.args.fetch(0))
|
|
96
|
+
-> { read_unsigned(1) == 1 ? nil : inner.call }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def array_decoder(node)
|
|
100
|
+
inner = value_decoder(node.args.fetch(0))
|
|
101
|
+
-> { Array.new(read_varint) { inner.call } }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def map_decoder(node)
|
|
105
|
+
key = value_decoder(node.args.fetch(0))
|
|
106
|
+
value = value_decoder(node.args.fetch(1))
|
|
107
|
+
-> { Array.new(read_varint) { [key.call, value.call] }.to_h }
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Named tuples come back as hashes to match the JSON wire's object shape.
|
|
111
|
+
def tuple_decoder(node)
|
|
112
|
+
if node.args.first.is_a?(Array)
|
|
113
|
+
members = node.args.map { |name, type| [name, value_decoder(type)] }
|
|
114
|
+
-> { members.to_h { |name, decoder| [name, decoder.call] } }
|
|
115
|
+
else
|
|
116
|
+
members = node.args.map { |type| value_decoder(type) }
|
|
117
|
+
-> { members.map(&:call) }
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def enum_decoder(node, size)
|
|
122
|
+
labels = node.args.to_h { |label, number| [number, label] }
|
|
123
|
+
-> { labels.fetch(read_signed(size)) }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def datetime64_decoder(node)
|
|
127
|
+
divisor = 10**node.args.fetch(0)
|
|
128
|
+
-> { represent_instant(Time.at(Rational(read_signed(8), divisor))) }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# The wire carries an epoch, so the instant is zone-free; representation
|
|
132
|
+
# follows default_timezone like every built-in adapter (Time.at is local).
|
|
133
|
+
def represent_instant(time)
|
|
134
|
+
ActiveRecord.default_timezone == :local ? time : time.utc
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def decimal_decoder(node)
|
|
138
|
+
precision, scale = node.args
|
|
139
|
+
size = DECIMAL_BYTES.find { |max_precision, _| precision <= max_precision }.last
|
|
140
|
+
-> { BigDecimal("#{read_signed(size)}e-#{scale}") }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def read_varint
|
|
144
|
+
result = 0
|
|
145
|
+
shift = 0
|
|
146
|
+
loop do
|
|
147
|
+
byte = @body.getbyte(advance(1))
|
|
148
|
+
result |= (byte & 0x7f) << shift
|
|
149
|
+
return result if byte < 0x80
|
|
150
|
+
|
|
151
|
+
shift += 7
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def read_string = read_bytes(read_varint)
|
|
156
|
+
|
|
157
|
+
def read_bytes(length)
|
|
158
|
+
@body.byteslice(advance(length), length).force_encoding(Encoding::UTF_8)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def read_unsigned(size)
|
|
162
|
+
return @body.unpack1(INTEGER_FORMATS.fetch("UInt#{size * 8}").first, offset: advance(size)) if size <= 8
|
|
163
|
+
|
|
164
|
+
words = @body.unpack("Q<#{size / 8}", offset: advance(size))
|
|
165
|
+
words.each_with_index.sum { |word, index| word << (64 * index) }
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def read_signed(size)
|
|
169
|
+
return @body.unpack1(INTEGER_FORMATS.fetch("Int#{size * 8}").first, offset: advance(size)) if size <= 8
|
|
170
|
+
|
|
171
|
+
value = read_unsigned(size)
|
|
172
|
+
value >= (1 << ((size * 8) - 1)) ? value - (1 << (size * 8)) : value
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# UUIDs travel as two little-endian UInt64 halves (high, then low).
|
|
176
|
+
def read_uuid
|
|
177
|
+
high, low = @body.unpack("Q<2", offset: advance(16))
|
|
178
|
+
format("%<high>016x%<low>016x", high: high, low: low)
|
|
179
|
+
.insert(20, "-").insert(16, "-").insert(12, "-").insert(8, "-")
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def advance(length)
|
|
183
|
+
position = @position
|
|
184
|
+
@position += length
|
|
185
|
+
position
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module ConnectionAdapters
|
|
5
|
+
module ClickHouse
|
|
6
|
+
class TableDefinition < ConnectionAdapters::TableDefinition
|
|
7
|
+
attr_reader :engine, :order, :partition, :ttl, :table_settings, :primary_key_clause, :sample
|
|
8
|
+
|
|
9
|
+
def initialize(conn, name, engine: "MergeTree", order: nil, partition: nil, ttl: nil,
|
|
10
|
+
settings: nil, primary_key_clause: nil, sample: nil, **)
|
|
11
|
+
@engine = engine
|
|
12
|
+
@order = order
|
|
13
|
+
@partition = partition
|
|
14
|
+
@ttl = ttl
|
|
15
|
+
@table_settings = settings
|
|
16
|
+
@primary_key_clause = primary_key_clause
|
|
17
|
+
@sample = sample
|
|
18
|
+
super(conn, name, **)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Carries the ClickHouse-only column metadata the dumper needs: compression codec
|
|
23
|
+
# and MATERIALIZED/ALIAS server-computed expressions.
|
|
24
|
+
class Column < ConnectionAdapters::Column
|
|
25
|
+
attr_reader :codec, :computed_kind, :computed_expression
|
|
26
|
+
|
|
27
|
+
def initialize(*, codec: nil, computed_kind: nil, computed_expression: nil, **)
|
|
28
|
+
@codec = codec
|
|
29
|
+
@computed_kind = computed_kind
|
|
30
|
+
@computed_expression = computed_expression
|
|
31
|
+
super(*, **)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# A ClickHouse data-skipping index: `using` is the full index type expression
|
|
36
|
+
# ("bloom_filter", "set(100)", ...) and granularity is index blocks per granule.
|
|
37
|
+
class IndexDefinition < ConnectionAdapters::IndexDefinition
|
|
38
|
+
attr_reader :granularity
|
|
39
|
+
|
|
40
|
+
def initialize(table, name, columns:, using:, granularity:)
|
|
41
|
+
@granularity = granularity
|
|
42
|
+
super(table, name, false, columns, using: using)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
class SchemaCreation < ConnectionAdapters::SchemaCreation
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def visit_TableDefinition(o)
|
|
50
|
+
stamp_on_cluster(super, o.name)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def visit_AlterTable(o)
|
|
54
|
+
stamp_on_cluster(super, o.name)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# ON CLUSTER renders directly after the table name; the first mention is the
|
|
58
|
+
# one following CREATE/ALTER TABLE.
|
|
59
|
+
def stamp_on_cluster(sql, table_name)
|
|
60
|
+
clause = @conn.on_cluster_clause
|
|
61
|
+
return sql if clause.empty?
|
|
62
|
+
|
|
63
|
+
quoted = quote_table_name(table_name)
|
|
64
|
+
sql.sub("TABLE #{quoted} ", "TABLE #{quoted}#{clause} ")
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# ClickHouse nullability lives in the type (Nullable/LowCardinality wrappers),
|
|
68
|
+
# not in NOT NULL constraints, and MergeTree requires an ORDER BY clause.
|
|
69
|
+
def visit_ColumnDefinition(o)
|
|
70
|
+
o.sql_type = type_to_sql(o.type, **o.options)
|
|
71
|
+
column_sql = "#{quote_column_name(o.name)} #{wrapped_sql_type(o)}"
|
|
72
|
+
add_column_options!(column_sql, column_options(o))
|
|
73
|
+
column_sql
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def wrapped_sql_type(o)
|
|
77
|
+
sql_type = o.sql_type
|
|
78
|
+
sql_type = "Nullable(#{sql_type})" if o.options[:null]
|
|
79
|
+
sql_type = "LowCardinality(#{sql_type})" if o.options[:low_cardinality]
|
|
80
|
+
sql_type
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def visit_AddColumnDefinition(o)
|
|
84
|
+
"ADD COLUMN #{accept(o.column)}"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Data-skipping indexes are part of CREATE TABLE (supports_indexes_in_create?);
|
|
88
|
+
# the expression passes through verbatim — it may be a function of columns.
|
|
89
|
+
def index_in_create(table_name, column_name, options)
|
|
90
|
+
columns = Array(column_name)
|
|
91
|
+
expression = columns.length == 1 ? columns.first.to_s : "(#{columns.join(", ")})"
|
|
92
|
+
name = options.fetch(:name) { @conn.index_name(table_name, column_name) }
|
|
93
|
+
# Same portability default as add_index: bloom_filter unless told otherwise.
|
|
94
|
+
"INDEX #{quote_column_name(name)} #{expression} " \
|
|
95
|
+
"TYPE #{options.fetch(:using, "bloom_filter")} GRANULARITY #{options.fetch(:granularity, 1)}"
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# DEFAULT / MATERIALIZED / ALIAS are mutually exclusive ways for a column to get
|
|
99
|
+
# its value; CODEC composes with any of them.
|
|
100
|
+
def add_column_options!(sql, options)
|
|
101
|
+
sql << column_value_clause(options).to_s
|
|
102
|
+
sql << " CODEC(#{options[:codec]})" if options[:codec]
|
|
103
|
+
sql
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def column_value_clause(options)
|
|
107
|
+
clauses = []
|
|
108
|
+
if options_include_default?(options)
|
|
109
|
+
default = options[:default]
|
|
110
|
+
clauses << " DEFAULT #{default.is_a?(Proc) ? default.call : @conn.quote(default)}"
|
|
111
|
+
end
|
|
112
|
+
clauses << " MATERIALIZED #{options[:materialized]}" if options[:materialized]
|
|
113
|
+
clauses << " ALIAS #{options[:alias]}" if options[:alias]
|
|
114
|
+
raise ArgumentError, "materialized:, alias: and default: are mutually exclusive" if clauses.many?
|
|
115
|
+
|
|
116
|
+
clauses.first
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def add_table_options!(create_sql, o)
|
|
120
|
+
if o.engine.include?("MergeTree") && o.order.nil?
|
|
121
|
+
raise ArgumentError, "#{o.engine} tables require order: (the sorting key); use order: \"tuple()\" for none"
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
create_sql << " ENGINE = #{o.engine}"
|
|
125
|
+
table_clauses(o).each { |keyword, expression| create_sql << " #{keyword} #{expression}" if expression }
|
|
126
|
+
create_sql
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def table_clauses(o)
|
|
130
|
+
{
|
|
131
|
+
"PARTITION BY" => o.partition,
|
|
132
|
+
"PRIMARY KEY" => o.primary_key_clause,
|
|
133
|
+
"ORDER BY" => o.order,
|
|
134
|
+
"SAMPLE BY" => o.sample,
|
|
135
|
+
"TTL" => o.ttl,
|
|
136
|
+
"SETTINGS" => o.table_settings.present? ? format_settings(o.table_settings) : nil
|
|
137
|
+
}
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def format_settings(settings)
|
|
141
|
+
settings.map { |key, value| "#{key} = #{value}" }.join(", ")
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecord
|
|
4
|
+
module ConnectionAdapters
|
|
5
|
+
module ClickHouse
|
|
6
|
+
# Dumps columns so the schema round-trips through this adapter's DSL: AR-native
|
|
7
|
+
# types only when type_to_sql regenerates the exact server type, everything else
|
|
8
|
+
# as a verbatim ClickHouse type string; Nullable/LowCardinality become column
|
|
9
|
+
# options because absence of null: means NOT NULL here, unlike core Rails.
|
|
10
|
+
class SchemaDumper < ConnectionAdapters::SchemaDumper
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
# Materialized views and dictionaries dump after every table so their sources
|
|
14
|
+
# exist on load (dictionaries bind lazily, but keep the order intuitive).
|
|
15
|
+
def tables(stream)
|
|
16
|
+
super
|
|
17
|
+
materialized_views(stream)
|
|
18
|
+
dictionaries(stream)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
DICTIONARIES_SQL = <<~SQL.squish
|
|
22
|
+
SELECT name, create_table_query FROM system.tables
|
|
23
|
+
WHERE database = currentDatabase() AND engine = 'Dictionary'
|
|
24
|
+
ORDER BY name
|
|
25
|
+
SQL
|
|
26
|
+
private_constant :DICTIONARIES_SQL
|
|
27
|
+
|
|
28
|
+
def dictionaries(stream)
|
|
29
|
+
dumpable = @connection.select_all(DICTIONARIES_SQL, "SCHEMA").reject { |row| ignored?(row["name"]) }
|
|
30
|
+
dumpable.each_with_index do |dictionary, index|
|
|
31
|
+
stream.puts if index.zero?
|
|
32
|
+
stream.puts dictionary_statement(dictionary)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# create_dictionary re-infers columns and re-injects credentials at load time,
|
|
37
|
+
# so only the identity kwargs are parsed back out of the stored DDL.
|
|
38
|
+
def dictionary_statement(dictionary)
|
|
39
|
+
kwargs = dictionary_kwargs(dictionary["create_table_query"])
|
|
40
|
+
rendered = kwargs.map { |keyword, value| "#{keyword}: #{value}" }
|
|
41
|
+
" create_dictionary #{dictionary["name"].inspect}, #{rendered.join(", ")}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def dictionary_kwargs(ddl)
|
|
45
|
+
database = ddl[/\bDB '([^']+)'/, 1]
|
|
46
|
+
{
|
|
47
|
+
source: ddl[/\bTABLE '([^']+)'/, 1].inspect,
|
|
48
|
+
database: (database.inspect if database && database != current_database),
|
|
49
|
+
primary_key: ddl[/\bPRIMARY KEY ([^\s(]+)/, 1].inspect,
|
|
50
|
+
layout: ddl[/\bLAYOUT\((\w+)\(/, 1].downcase.to_sym.inspect,
|
|
51
|
+
lifetime: "#{ddl[/\bLIFETIME\(MIN (\d+)/, 1].to_i}..#{ddl[/\bMAX (\d+)\)/, 1].to_i}"
|
|
52
|
+
}.compact
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Projections dump right after their table as add_projection calls, parsed back
|
|
56
|
+
# out of the stored query text (SELECT ... [GROUP BY ...] [ORDER BY ...]).
|
|
57
|
+
def table(table, stream)
|
|
58
|
+
super
|
|
59
|
+
projections(table).each do |row|
|
|
60
|
+
stream.puts(" #{projection_statement(table, row)}")
|
|
61
|
+
stream.puts
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
PROJECTIONS_SQL = <<~SQL.squish
|
|
66
|
+
SELECT name, query FROM system.projections
|
|
67
|
+
WHERE database = currentDatabase() AND table = %s ORDER BY name
|
|
68
|
+
SQL
|
|
69
|
+
private_constant :PROJECTIONS_SQL
|
|
70
|
+
|
|
71
|
+
def projections(table)
|
|
72
|
+
@connection.select_all(format(PROJECTIONS_SQL, @connection.quote(table)), "SCHEMA").to_a
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def projection_statement(table, row)
|
|
76
|
+
query = row["query"]
|
|
77
|
+
parts = {
|
|
78
|
+
select: query[/\ASELECT\s+(.*?)(?:\s+GROUP BY\s|\s+ORDER BY\s|\z)/m, 1],
|
|
79
|
+
group: query[/\sGROUP BY\s+(.*?)(?:\s+ORDER BY\s|\z)/m, 1],
|
|
80
|
+
order: query[/\sORDER BY\s+(.*)\z/m, 1]
|
|
81
|
+
}.compact
|
|
82
|
+
arguments = parts.map { |keyword, expression| "#{keyword}: #{expression.inspect}" }
|
|
83
|
+
"add_projection #{table.inspect}, #{row["name"].inspect}, #{arguments.join(", ")}"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
MATERIALIZED_VIEW_SQL = <<~SQL.squish
|
|
87
|
+
SELECT name, as_select, create_table_query FROM system.tables
|
|
88
|
+
WHERE database = currentDatabase() AND engine = 'MaterializedView'
|
|
89
|
+
ORDER BY name
|
|
90
|
+
SQL
|
|
91
|
+
private_constant :MATERIALIZED_VIEW_SQL
|
|
92
|
+
|
|
93
|
+
def materialized_views(stream)
|
|
94
|
+
dumpable = @connection.select_all(MATERIALIZED_VIEW_SQL, "SCHEMA").reject { |view| ignored?(view["name"]) }
|
|
95
|
+
dumpable.each_with_index do |view, index|
|
|
96
|
+
stream.puts if index.zero?
|
|
97
|
+
stream.puts materialized_view_statement(view)
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# The server database-qualifies every identifier it stores; strip the current
|
|
102
|
+
# database so the dump loads into any target database.
|
|
103
|
+
def materialized_view_statement(view)
|
|
104
|
+
target = view["create_table_query"][/\bTO\s+(\S+)/, 1]
|
|
105
|
+
select = strip_database_qualifier(view["as_select"])
|
|
106
|
+
" create_materialized_view #{view["name"].inspect}, " \
|
|
107
|
+
"to: #{strip_database_qualifier(target).inspect}, as: #{select.inspect}"
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def strip_database_qualifier(sql)
|
|
111
|
+
database = Regexp.escape(current_database)
|
|
112
|
+
sql.gsub(/(?:`#{database}`|#{database})\./, "")
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def current_database
|
|
116
|
+
@current_database ||= @connection.select_value("SELECT currentDatabase()", "SCHEMA")
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def column_spec(column)
|
|
120
|
+
inner_type, wrappers = unwrap_column_type(column.sql_type)
|
|
121
|
+
type = schema_type(column)
|
|
122
|
+
|
|
123
|
+
if regenerates_exactly?(type, inner_type, column)
|
|
124
|
+
spec = prepare_column_options(column)
|
|
125
|
+
spec[:low_cardinality] = "true" if wrappers.include?(:low_cardinality)
|
|
126
|
+
[type, spec]
|
|
127
|
+
else
|
|
128
|
+
[column.sql_type, verbatim_column_options(column)]
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def prepare_column_options(column)
|
|
133
|
+
spec = super
|
|
134
|
+
spec.delete(:null)
|
|
135
|
+
spec[:null] = "true" if column.null
|
|
136
|
+
spec.merge!(clickhouse_column_options(column))
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def clickhouse_column_options(column)
|
|
140
|
+
spec = {}
|
|
141
|
+
spec[column.computed_kind.to_sym] = column.computed_expression.inspect if column.computed_kind
|
|
142
|
+
spec[:codec] = column.codec.inspect if column.codec
|
|
143
|
+
spec
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Rails omits precision 6 as the datetime default, but this adapter's default is
|
|
147
|
+
# DateTime64(3) — always dump the real precision.
|
|
148
|
+
def schema_precision(column)
|
|
149
|
+
column.precision&.inspect
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
WRAPPER_TYPES = { /\ALowCardinality\((.*)\)\z/m => :low_cardinality, /\ANullable\((.*)\)\z/m => :null }.freeze
|
|
153
|
+
private_constant :WRAPPER_TYPES
|
|
154
|
+
|
|
155
|
+
def unwrap_column_type(sql_type)
|
|
156
|
+
wrappers = []
|
|
157
|
+
inner = sql_type
|
|
158
|
+
while (wrapper = WRAPPER_TYPES.find { |pattern, _| pattern.match?(inner) })
|
|
159
|
+
wrappers << wrapper.last
|
|
160
|
+
inner = inner.match(wrapper.first)[1]
|
|
161
|
+
end
|
|
162
|
+
[inner, wrappers]
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def regenerates_exactly?(type, inner_type, column)
|
|
166
|
+
return false unless type
|
|
167
|
+
|
|
168
|
+
regenerated = @connection.type_to_sql(
|
|
169
|
+
type, limit: column.limit, precision: column.precision, scale: column.scale
|
|
170
|
+
)
|
|
171
|
+
regenerated == inner_type
|
|
172
|
+
rescue ArgumentError
|
|
173
|
+
false
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def verbatim_column_options(column)
|
|
177
|
+
spec = {}
|
|
178
|
+
spec[:default] = schema_default(column)
|
|
179
|
+
spec[:comment] = column.comment.inspect if column.comment.present?
|
|
180
|
+
spec.compact
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def index_parts(index)
|
|
184
|
+
super + ["granularity: #{index.granularity}"]
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Hash#inspect renders {:key=>value} before Ruby 3.4; emit the modern literal
|
|
188
|
+
# so the settings: option dumps identically on every supported Ruby.
|
|
189
|
+
def format_options(options)
|
|
190
|
+
options.map { |key, value| "#{key}: #{format_option_value(value)}" }.join(", ")
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def format_option_value(value)
|
|
194
|
+
return value.inspect unless value.is_a?(Hash)
|
|
195
|
+
|
|
196
|
+
"{#{value.map { |key, entry| "#{key}: #{entry.inspect}" }.join(", ")}}"
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
end
|