clickhouse-activerecord 0.4.6 → 0.4.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: deeb0e0ddf3a6bef04338637ddf814a826f93b4e69779905f28c280a5ca1a400
4
- data.tar.gz: 80cba4b321f42cb978be4d6bf4740b03274a7bd0ff882b35d4ac0760736fe9a8
3
+ metadata.gz: d8d1d60f75e0f889dc5d947df77924f1490c0325f097d634048fb25405eeca70
4
+ data.tar.gz: 309eda1c3f605682c04525fce8633b85a73a38643e367f33d87588939f7815fe
5
5
  SHA512:
6
- metadata.gz: 8c18476f8984ab7e7c834b712eff4f9d8e3995287b3db6ace0e4e819ac19dc6bd31b149fee7da970223807170b1902ce8d2af81ad8ac8a56bc73641e26111d22
7
- data.tar.gz: '094cdb750234da2b59efd755c7fef3e4f4bf3484b124d7d2b74fe999ee2a25cddb435324fd88a49248943cb372459df31a3d151905e93118b456231ae94cbd2c'
6
+ metadata.gz: 5727f1a23c48e04db1052bfbf4c56af412338be88f4b985ed6700e9a833436bcde893ba9cb1ed7b89473dae95b31e4b70de26c6885cce10b8fffcdb9c65fbb3d
7
+ data.tar.gz: 18fad92b704c1c41b97029facbc020111c3b067cbb825d6e5c4d11d6a4b7a220873b922aea7112ec741f1d87a1a5c2217a11047e764959a2edffb98e0b7f2463
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ### Version 0.4.10 (Mar 10, 2021)
2
+
3
+ * Support ClickHouse 20.9+
4
+ * Fix schema create / dump
5
+ * Support all integer types through :limit and :unsigned [@bdevel](https://github.com/bdevel)
6
+
1
7
  ### Version 0.4.4 (Sep 23, 2020)
2
8
 
3
9
  * Full support migration and rollback database
data/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # Clickhouse::Activerecord
2
2
 
3
3
  A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
4
- Support ClickHouse version from 19.14 LTS.
4
+ Support ClickHouse version from 20.9 LTS.
5
5
 
6
6
  ## Installation
7
7
 
@@ -162,6 +162,44 @@ ActionView.maximum(:date)
162
162
  #=> 'Wed, 29 Nov 2017'
163
163
  ```
164
164
 
165
+
166
+ ### Migration Data Types
167
+
168
+ Integer types are unsigned by default. Specify signed values with `:unsigned =>
169
+ false`. The default integer is `UInt32`
170
+
171
+ | Type (bit size) | Range | :limit (byte size) |
172
+ | :--- | :----: | ---: |
173
+ | Int8 | -128 to 127 | 1 |
174
+ | Int16 | -32768 to 32767 | 2 |
175
+ | Int32 | -2147483648 to 2,147,483,647 | 3,4 |
176
+ | Int64 | -9223372036854775808 to 9223372036854775807] | 5,6,7,8 |
177
+ | Int128 | ... | 9 - 15 |
178
+ | Int256 | ... | 16+ |
179
+ | UInt8 | 0 to 255 | 1 |
180
+ | UInt16 | 0 to 65,535 | 2 |
181
+ | UInt32 | 0 to 4,294,967,295 | 3,4 |
182
+ | UInt64 | 0 to 18446744073709551615 | 5,6,7,8 |
183
+ | UInt256 | 0 to ... | 8+ |
184
+
185
+ Example:
186
+
187
+ ``` ruby
188
+ class CreateDataItems < ActiveRecord::Migration
189
+ def change
190
+ create_table "data_items", id: false, options: "VersionedCollapsingMergeTree(sign, version) PARTITION BY toYYYYMM(day) ORDER BY category", force: :cascade do |t|
191
+ t.date "day", null: false
192
+ t.string "category", null: false
193
+ t.integer "value_in", null: false
194
+ t.integer "sign", limit: 1, unsigned: false, default: -> { "CAST(1, 'Int8')" }, null: false
195
+ t.integer "version", limit: 8, default: -> { "CAST(toUnixTimestamp(now()), 'UInt64')" }, null: false
196
+ end
197
+ end
198
+ end
199
+
200
+ ```
201
+
202
+
165
203
  ### Using replica and cluster params in connection parameters
166
204
 
167
205
  ```yml
@@ -39,7 +39,9 @@ module ActiveRecord
39
39
  statements << accept(o.primary_keys) if o.primary_keys
40
40
 
41
41
  create_sql << "(#{statements.join(', ')})" if statements.present?
42
- add_table_options!(create_sql, table_options(o))
42
+ # Attach options for only table or materialized view
43
+ add_table_options!(create_sql, table_options(o)) if !o.view || o.view && o.materialized
44
+
43
45
  create_sql << " AS #{to_sql(o.as)}" if o.as
44
46
  create_sql
45
47
  end
@@ -35,13 +35,31 @@ module ActiveRecord
35
35
  end
36
36
 
37
37
  def integer(*args, **options)
38
- if options[:limit] == 8
39
- args.each { |name| column(name, :big_integer, options.except(:limit)) }
40
- else
41
- super
38
+ # default to unsigned
39
+ unsigned = options[:unsigned]
40
+ unsigned = true if unsigned.nil?
41
+
42
+ kind = :uint32 # default
43
+
44
+ if options[:limit]
45
+ if unsigned
46
+ kind = :uint8 if options[:limit] == 1
47
+ kind = :uint16 if options[:limit] == 2
48
+ kind = :uint32 if [3,4].include?(options[:limit])
49
+ kind = :uint64 if [5,6,7].include?(options[:limit])
50
+ kind = :big_integer if options[:limit] == 8
51
+ kind = :uint256 if options[:limit] > 8
52
+ else
53
+ kind = :int8 if options[:limit] == 1
54
+ kind = :int16 if options[:limit] == 2
55
+ kind = :int32 if [3,4].include?(options[:limit])
56
+ kind = :int64 if options[:limit] > 5 && options[:limit] <= 8
57
+ kind = :int128 if options[:limit] > 8 && options[:limit] <= 16
58
+ kind = :int256 if options[:limit] > 16
59
+ end
42
60
  end
61
+ args.each { |name| column(name, kind, options.except(:limit, :unsigned)) }
43
62
  end
44
-
45
63
  end
46
64
  end
47
65
  end
@@ -36,8 +36,8 @@ module ActiveRecord
36
36
  end
37
37
 
38
38
  def table_options(table)
39
- sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
40
- { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
39
+ sql = show_create_table(table)
40
+ { options: sql.gsub(/^(?:.*?)(?:ENGINE = (.*?))?( AS SELECT .*?)?$/, '\\1').presence, as: sql.match(/^CREATE (?:.*?) AS (SELECT .*?)$/).try(:[], 1) }.compact
41
41
  end
42
42
 
43
43
  # Not indexes on clickhouse
@@ -79,7 +79,6 @@ module ActiveRecord
79
79
 
80
80
  class ClickhouseAdapter < AbstractAdapter
81
81
  ADAPTER_NAME = 'Clickhouse'.freeze
82
-
83
82
  NATIVE_DATABASE_TYPES = {
84
83
  string: { name: 'String' },
85
84
  integer: { name: 'UInt32' },
@@ -88,7 +87,21 @@ module ActiveRecord
88
87
  decimal: { name: 'Decimal' },
89
88
  datetime: { name: 'DateTime' },
90
89
  date: { name: 'Date' },
91
- boolean: { name: 'UInt8' }
90
+ boolean: { name: 'UInt8' },
91
+
92
+ int8: { name: 'Int8' },
93
+ int16: { name: 'Int16' },
94
+ int32: { name: 'Int32' },
95
+ int64: { name: 'Int64' },
96
+ int128: { name: 'Int128' },
97
+ int256: { name: 'Int256' },
98
+
99
+ uint8: { name: 'UInt8' },
100
+ uint16: { name: 'UInt16' },
101
+ uint32: { name: 'UInt32' },
102
+ uint64: { name: 'UInt64' },
103
+ # uint128: { name: 'UInt128' }, not yet implemented in clickhouse
104
+ uint256: { name: 'UInt256' },
92
105
  }.freeze
93
106
 
94
107
  include Clickhouse::SchemaStatements
@@ -139,10 +152,12 @@ module ActiveRecord
139
152
  when /(Nullable)?\(?String\)?/
140
153
  super('String')
141
154
  when /(Nullable)?\(?U?Int8\)?/
142
- super('int2')
143
- when /(Nullable)?\(?U?Int(16|32)\)?/
144
- super('int4')
145
- when /(Nullable)?\(?U?Int(64)\)?/
155
+ 1
156
+ when /(Nullable)?\(?U?Int16\)?/
157
+ 2
158
+ when /(Nullable)?\(?U?Int32\)?/
159
+ nil
160
+ when /(Nullable)?\(?U?Int64\)?/
146
161
  8
147
162
  else
148
163
  super
@@ -154,14 +169,20 @@ module ActiveRecord
154
169
  register_class_with_limit m, %r(String), Type::String
155
170
  register_class_with_limit m, 'Date', Clickhouse::OID::Date
156
171
  register_class_with_limit m, 'DateTime', Clickhouse::OID::DateTime
157
- register_class_with_limit m, %r(Uint8), Type::UnsignedInteger
158
- m.alias_type 'UInt16', 'UInt8'
159
- m.alias_type 'UInt32', 'UInt8'
160
- register_class_with_limit m, %r(UInt64), Type::UnsignedInteger
172
+
161
173
  register_class_with_limit m, %r(Int8), Type::Integer
162
- m.alias_type 'Int16', 'Int8'
163
- m.alias_type 'Int32', 'Int8'
174
+ register_class_with_limit m, %r(Int16), Type::Integer
175
+ register_class_with_limit m, %r(Int32), Type::Integer
164
176
  register_class_with_limit m, %r(Int64), Type::Integer
177
+ register_class_with_limit m, %r(Int128), Type::Integer
178
+ register_class_with_limit m, %r(Int256), Type::Integer
179
+
180
+ register_class_with_limit m, %r(Uint8), Type::UnsignedInteger
181
+ register_class_with_limit m, %r(UInt16), Type::UnsignedInteger
182
+ register_class_with_limit m, %r(UInt32), Type::UnsignedInteger
183
+ register_class_with_limit m, %r(UInt64), Type::UnsignedInteger
184
+ #register_class_with_limit m, %r(UInt128), Type::UnsignedInteger #not implemnted in clickhouse
185
+ register_class_with_limit m, %r(UInt256), Type::UnsignedInteger
165
186
  end
166
187
 
167
188
  # Quoting time without microseconds
@@ -201,6 +222,12 @@ module ActiveRecord
201
222
  ClickhouseActiverecord::SchemaDumper.create(self, options)
202
223
  end
203
224
 
225
+ # @param [String] table
226
+ # @return [String]
227
+ def show_create_table(table)
228
+ do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first).gsub(/[\n\s]+/m, ' ')
229
+ end
230
+
204
231
  # Create a new ClickHouse database.
205
232
  def create_database(name)
206
233
  sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
@@ -7,7 +7,7 @@ module ClickhouseActiverecord
7
7
  unless table_exists?
8
8
  version_options = connection.internal_string_options_for_primary_key
9
9
 
10
- connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
10
+ connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true) do |t|
11
11
  t.string :version, version_options
12
12
  t.column :active, 'Int8', null: false, default: '1'
13
13
  t.datetime :ver, null: false, default: -> { 'now()' }
@@ -27,7 +27,7 @@ module ClickhouseActiverecord
27
27
  unless table_exists?
28
28
  key_options = connection.internal_string_options_for_primary_key
29
29
 
30
- connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
30
+ connection.create_table(table_name, id: false, options: connection.adapter_name.downcase == 'clickhouse' ? 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)' : '', if_not_exists: true) do |t|
31
31
  t.string :key, key_options
32
32
  t.string :value
33
33
  t.timestamps
@@ -33,16 +33,24 @@ module ClickhouseActiverecord
33
33
  HEADER
34
34
  end
35
35
 
36
+ def tables(stream)
37
+ sorted_tables = @connection.tables.sort {|a,b| @connection.show_create_table(a).match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : a <=> b }
38
+
39
+ sorted_tables.each do |table_name|
40
+ table(table_name, stream) unless ignored?(table_name)
41
+ end
42
+ end
43
+
36
44
  def table(table, stream)
37
- if table.match(/^\.inner\./).nil?
45
+ if table.match(/^\.inner/).nil?
38
46
  unless simple
39
47
  stream.puts " # TABLE: #{table}"
40
- sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
48
+ sql = @connection.show_create_table(table)
41
49
  stream.puts " # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
42
50
  # super(table.gsub(/^\.inner\./, ''), stream)
43
51
 
44
52
  # detect view table
45
- match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
53
+ match = sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/)
46
54
  end
47
55
 
48
56
  # Copy from original dumper
@@ -125,5 +133,21 @@ HEADER
125
133
  super
126
134
  end
127
135
  end
136
+
137
+ def schema_limit(column)
138
+ return nil if column.type == :float
139
+ super
140
+ end
141
+
142
+ def schema_unsigned(column)
143
+ return nil unless column.type == :integer && !simple
144
+ (column.sql_type =~ /(Nullable)?\(?UInt\d+\)?/).nil? ? false : nil
145
+ end
146
+
147
+ def prepare_column_options(column)
148
+ spec = {}
149
+ spec[:unsigned] = schema_unsigned(column)
150
+ spec.merge(super).compact
151
+ end
128
152
  end
129
153
  end
@@ -1,3 +1,3 @@
1
1
  module ClickhouseActiverecord
2
- VERSION = '0.4.6'
2
+ VERSION = '0.4.11'
3
3
  end
@@ -20,7 +20,7 @@ namespace :clickhouse do
20
20
 
21
21
  # todo not testing
22
22
  desc 'Load database schema'
23
- task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do |t, args|
23
+ task load: [:load_config, :prepare_internal_metadata_table] do |t, args|
24
24
  simple = ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
25
25
  ClickhouseActiverecord::SchemaMigration.drop_table
26
26
  load("#{Rails.root}/db/clickhouse_schema#{simple}.rb")
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: clickhouse-activerecord
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.6
4
+ version: 0.4.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sergey Odintsov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-09-30 00:00:00.000000000 Z
11
+ date: 2021-03-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
149
149
  - !ruby/object:Gem::Version
150
150
  version: '0'
151
151
  requirements: []
152
- rubygems_version: 3.0.1
152
+ rubygems_version: 3.0.3
153
153
  signing_key:
154
154
  specification_version: 4
155
155
  summary: ClickHouse ActiveRecord