clickhouse-activerecord 0.3.13 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +17 -5
- data/clickhouse-activerecord.gemspec +1 -1
- data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb +20 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb +29 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +21 -5
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +53 -17
- data/lib/clickhouse-activerecord.rb +1 -0
- data/lib/clickhouse-activerecord/migration.rb +92 -0
- data/lib/clickhouse-activerecord/schema.rb +19 -0
- data/lib/clickhouse-activerecord/schema_dumper.rb +93 -3
- data/lib/clickhouse-activerecord/version.rb +1 -1
- data/lib/generators/clickhouse_migration_generator.rb +2 -2
- data/lib/tasks/clickhouse.rake +8 -59
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8020d332d2b7b707fa2776a37fbb8d2c217ec50703962d06938be53eaae22796
|
4
|
+
data.tar.gz: 6979255cb79396c6574e4b58c0e82ae39a42612fe1068b1308d98a1222c58b64
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3aa6d799adf1fc66d49c3dff6d02c5ad0349e467fb1067a1453de8bfcb3316abb46681367b7dfd3ebd8f82f13460cb658c4f167ff23398bba8b9cc459c5b6df1
|
7
|
+
data.tar.gz: a1c8e252279678efb2fe1f8c36a8a7e1c5bfda24896438245921deee5862e94c047dced14b85408b5e1343606ec7055100b813af399568f5ee405a255c268777
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
### Version 0.4.0 (Sep 18, 2020)
|
2
|
+
|
3
|
+
* Full support migration and rollback database
|
4
|
+
* Support cluster and replica. Auto inject to SQL queries.
|
5
|
+
* Fix schema dump/load
|
6
|
+
|
1
7
|
### Version 0.3.10 (Dec 20, 2019)
|
2
8
|
|
3
9
|
* Support structure dump/load [@StoneGod](https://github.com/StoneGod)
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# Clickhouse::Activerecord
|
2
2
|
|
3
3
|
A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
|
4
|
-
|
4
|
+
Support ClickHouse version from 19.14 LTS.
|
5
5
|
|
6
6
|
## Installation
|
7
7
|
|
@@ -31,8 +31,8 @@ default: &default
|
|
31
31
|
ssl: true # optional for using ssl connection
|
32
32
|
debug: true # use for showing in to log technical information
|
33
33
|
migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
|
34
|
-
|
35
|
-
|
34
|
+
cluster_name: 'cluster_name' # optional for creating tables in cluster
|
35
|
+
replica_name: '{replica}' # replica macros name, optional for creating replicated tables
|
36
36
|
```
|
37
37
|
|
38
38
|
## Usage in Rails 5
|
@@ -111,8 +111,7 @@ Migration:
|
|
111
111
|
|
112
112
|
$ rails g clickhouse_migration MIGRATION_NAME COLUMNS
|
113
113
|
$ rake clickhouse:migrate
|
114
|
-
|
115
|
-
Rollback migration not supported!
|
114
|
+
$ rake clickhouse:rollback
|
116
115
|
|
117
116
|
### Dump / Load for multiple using databases
|
118
117
|
|
@@ -159,6 +158,19 @@ ActionView.maximum(:date)
|
|
159
158
|
#=> 'Wed, 29 Nov 2017'
|
160
159
|
```
|
161
160
|
|
161
|
+
### Using replica and cluster params in connection parameters
|
162
|
+
|
163
|
+
```yml
|
164
|
+
default: &default
|
165
|
+
***
|
166
|
+
cluster_name: 'cluster_name'
|
167
|
+
replica_name: '{replica}'
|
168
|
+
```
|
169
|
+
|
170
|
+
`ON CLUSTER cluster_name` will be attach to all queries create / drop.
|
171
|
+
|
172
|
+
Engines `MergeTree` and all support replication engines will be replaced to `Replicated***('/clickhouse/tables/cluster_name/database.table', '{replica}')`
|
173
|
+
|
162
174
|
## Donations
|
163
175
|
|
164
176
|
Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:
|
@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.add_runtime_dependency 'activerecord', '>= 5.2'
|
28
28
|
|
29
29
|
spec.add_development_dependency 'bundler', '~> 1.15'
|
30
|
-
spec.add_development_dependency 'rake', '~>
|
30
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
31
31
|
spec.add_development_dependency 'rspec', '~> 3.4'
|
32
32
|
spec.add_development_dependency 'pry', '~> 0.12'
|
33
33
|
end
|
@@ -27,6 +27,26 @@ module ActiveRecord
|
|
27
27
|
|
28
28
|
create_sql
|
29
29
|
end
|
30
|
+
|
31
|
+
def visit_TableDefinition(o)
|
32
|
+
create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
|
33
|
+
create_sql << "IF NOT EXISTS " if o.if_not_exists
|
34
|
+
create_sql << "#{quote_table_name(o.name)} "
|
35
|
+
|
36
|
+
statements = o.columns.map { |c| accept c }
|
37
|
+
statements << accept(o.primary_keys) if o.primary_keys
|
38
|
+
|
39
|
+
create_sql << "(#{statements.join(', ')})" if statements.present?
|
40
|
+
add_table_options!(create_sql, table_options(o))
|
41
|
+
create_sql << " AS #{to_sql(o.as)}" if o.as
|
42
|
+
create_sql
|
43
|
+
end
|
44
|
+
|
45
|
+
# Returns any SQL string to go between CREATE and TABLE. May be nil.
|
46
|
+
def table_modifier_in_create(o)
|
47
|
+
" TEMPORARY" if o.temporary
|
48
|
+
" MATERIALIZED" if o.materialized
|
49
|
+
end
|
30
50
|
end
|
31
51
|
end
|
32
52
|
end
|
@@ -5,6 +5,35 @@ module ActiveRecord
|
|
5
5
|
module Clickhouse
|
6
6
|
class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
|
7
7
|
|
8
|
+
attr_reader :view, :materialized, :if_not_exists
|
9
|
+
|
10
|
+
def initialize(
|
11
|
+
conn,
|
12
|
+
name,
|
13
|
+
temporary: false,
|
14
|
+
if_not_exists: false,
|
15
|
+
options: nil,
|
16
|
+
as: nil,
|
17
|
+
comment: nil,
|
18
|
+
view: false,
|
19
|
+
materialized: false,
|
20
|
+
**
|
21
|
+
)
|
22
|
+
@conn = conn
|
23
|
+
@columns_hash = {}
|
24
|
+
@indexes = []
|
25
|
+
@foreign_keys = []
|
26
|
+
@primary_keys = nil
|
27
|
+
@temporary = temporary
|
28
|
+
@if_not_exists = if_not_exists
|
29
|
+
@options = options
|
30
|
+
@as = as
|
31
|
+
@name = @conn.apply_cluster(name)
|
32
|
+
@comment = comment
|
33
|
+
@view = view || materialized
|
34
|
+
@materialized = materialized
|
35
|
+
end
|
36
|
+
|
8
37
|
def integer(*args, **options)
|
9
38
|
if options[:limit] == 8
|
10
39
|
args.each { |name| column(name, :big_integer, options.except(:limit)) }
|
@@ -67,6 +67,26 @@ module ActiveRecord
|
|
67
67
|
end
|
68
68
|
end
|
69
69
|
|
70
|
+
def assume_migrated_upto_version(version, migrations_paths = nil)
|
71
|
+
version = version.to_i
|
72
|
+
sm_table = quote_table_name(schema_migration.table_name)
|
73
|
+
|
74
|
+
migrated = migration_context.get_all_versions
|
75
|
+
versions = migration_context.migrations.map(&:version)
|
76
|
+
|
77
|
+
unless migrated.include?(version)
|
78
|
+
exec_insert "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", nil, nil
|
79
|
+
end
|
80
|
+
|
81
|
+
inserting = (versions - migrated).select { |v| v < version }
|
82
|
+
if inserting.any?
|
83
|
+
if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
|
84
|
+
raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
|
85
|
+
end
|
86
|
+
execute insert_versions_sql(inserting)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
70
90
|
private
|
71
91
|
|
72
92
|
def apply_format(sql, format)
|
@@ -95,11 +115,7 @@ module ActiveRecord
|
|
95
115
|
end
|
96
116
|
|
97
117
|
def create_table_definition(*args)
|
98
|
-
|
99
|
-
Clickhouse::TableDefinition.new(self, *args)
|
100
|
-
else
|
101
|
-
Clickhouse::TableDefinition.new(*args)
|
102
|
-
end
|
118
|
+
Clickhouse::TableDefinition.new(self, *args)
|
103
119
|
end
|
104
120
|
|
105
121
|
def new_column_from_field(table_name, field)
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'clickhouse-activerecord/arel/visitors/to_sql'
|
4
4
|
require 'clickhouse-activerecord/arel/table'
|
5
|
+
require 'clickhouse-activerecord/migration'
|
5
6
|
require 'active_record/connection_adapters/clickhouse/oid/date'
|
6
7
|
require 'active_record/connection_adapters/clickhouse/oid/date_time'
|
7
8
|
require 'active_record/connection_adapters/clickhouse/oid/big_integer'
|
@@ -110,17 +111,17 @@ module ActiveRecord
|
|
110
111
|
|
111
112
|
# Support SchemaMigration from v5.2.2 to v6+
|
112
113
|
def schema_migration # :nodoc:
|
113
|
-
|
114
|
-
super
|
115
|
-
else
|
116
|
-
ActiveRecord::SchemaMigration
|
117
|
-
end
|
114
|
+
ClickhouseActiverecord::SchemaMigration
|
118
115
|
end
|
119
116
|
|
120
117
|
def migrations_paths
|
121
118
|
@full_config[:migrations_paths] || 'db/migrate_clickhouse'
|
122
119
|
end
|
123
120
|
|
121
|
+
def migration_context # :nodoc:
|
122
|
+
ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration)
|
123
|
+
end
|
124
|
+
|
124
125
|
def arel_visitor # :nodoc:
|
125
126
|
ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
|
126
127
|
end
|
@@ -209,12 +210,29 @@ module ActiveRecord
|
|
209
210
|
end
|
210
211
|
end
|
211
212
|
|
212
|
-
def
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
213
|
+
def create_view(table_name, **options)
|
214
|
+
options.merge!(view: true)
|
215
|
+
options = apply_replica(table_name, options)
|
216
|
+
td = create_table_definition(table_name, options)
|
217
|
+
yield td if block_given?
|
218
|
+
|
219
|
+
if options[:force]
|
220
|
+
drop_table(table_name, options.merge(if_exists: true))
|
221
|
+
end
|
222
|
+
|
223
|
+
execute schema_creation.accept td
|
224
|
+
end
|
225
|
+
|
226
|
+
def create_table(table_name, **options)
|
227
|
+
options = apply_replica(table_name, options)
|
228
|
+
td = create_table_definition(table_name, options)
|
229
|
+
yield td if block_given?
|
230
|
+
|
231
|
+
if options[:force]
|
232
|
+
drop_table(table_name, options.merge(if_exists: true))
|
233
|
+
end
|
234
|
+
|
235
|
+
execute schema_creation.accept td
|
218
236
|
end
|
219
237
|
|
220
238
|
# Drops a ClickHouse database.
|
@@ -230,6 +248,22 @@ module ActiveRecord
|
|
230
248
|
do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
|
231
249
|
end
|
232
250
|
|
251
|
+
def cluster
|
252
|
+
@full_config[:cluster_name]
|
253
|
+
end
|
254
|
+
|
255
|
+
def replica
|
256
|
+
@full_config[:replica_name]
|
257
|
+
end
|
258
|
+
|
259
|
+
def replica_path(table)
|
260
|
+
"/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
|
261
|
+
end
|
262
|
+
|
263
|
+
def apply_cluster(sql)
|
264
|
+
cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
|
265
|
+
end
|
266
|
+
|
233
267
|
protected
|
234
268
|
|
235
269
|
def last_inserted_id(result)
|
@@ -242,12 +276,14 @@ module ActiveRecord
|
|
242
276
|
@connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
|
243
277
|
end
|
244
278
|
|
245
|
-
def
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
279
|
+
def apply_replica(table, options)
|
280
|
+
if replica && cluster
|
281
|
+
match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
|
282
|
+
if match
|
283
|
+
options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"
|
284
|
+
end
|
285
|
+
end
|
286
|
+
options
|
251
287
|
end
|
252
288
|
end
|
253
289
|
end
|
@@ -4,6 +4,7 @@ require 'active_record/connection_adapters/clickhouse_adapter'
|
|
4
4
|
|
5
5
|
if defined?(Rails::Railtie)
|
6
6
|
require 'clickhouse-activerecord/railtie'
|
7
|
+
require 'clickhouse-activerecord/schema'
|
7
8
|
require 'clickhouse-activerecord/schema_dumper'
|
8
9
|
require 'clickhouse-activerecord/tasks'
|
9
10
|
ActiveRecord::Tasks::DatabaseTasks.register_task(/clickhouse/, "ClickhouseActiverecord::Tasks")
|
@@ -0,0 +1,92 @@
|
|
1
|
+
module ClickhouseActiverecord
|
2
|
+
|
3
|
+
class SchemaMigration < ::ActiveRecord::SchemaMigration
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def create_table
|
7
|
+
unless table_exists?
|
8
|
+
version_options = connection.internal_string_options_for_primary_key
|
9
|
+
|
10
|
+
connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
|
11
|
+
t.string :version, version_options
|
12
|
+
t.column :active, 'Int8', null: false, default: '1'
|
13
|
+
t.datetime :ver, null: false, default: -> { 'now()' }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def all_versions
|
19
|
+
from("#{table_name} FINAL").where(active: 1).order(:version).pluck(:version)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class InternalMetadata < ::ActiveRecord::InternalMetadata
|
25
|
+
class << self
|
26
|
+
def create_table
|
27
|
+
unless table_exists?
|
28
|
+
key_options = connection.internal_string_options_for_primary_key
|
29
|
+
|
30
|
+
connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
|
31
|
+
t.string :key, key_options
|
32
|
+
t.string :value
|
33
|
+
t.timestamps
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
|
41
|
+
attr_reader :migrations_paths, :schema_migration
|
42
|
+
|
43
|
+
def initialize(migrations_paths, schema_migration)
|
44
|
+
@migrations_paths = migrations_paths
|
45
|
+
@schema_migration = schema_migration
|
46
|
+
end
|
47
|
+
|
48
|
+
def down(target_version = nil)
|
49
|
+
selected_migrations = if block_given?
|
50
|
+
migrations.select { |m| yield m }
|
51
|
+
else
|
52
|
+
migrations
|
53
|
+
end
|
54
|
+
|
55
|
+
ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, target_version).migrate
|
56
|
+
end
|
57
|
+
|
58
|
+
def get_all_versions
|
59
|
+
if schema_migration.table_exists?
|
60
|
+
schema_migration.all_versions.map(&:to_i)
|
61
|
+
else
|
62
|
+
[]
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
class Migrator < ::ActiveRecord::Migrator
|
69
|
+
|
70
|
+
def initialize(direction, migrations, schema_migration, target_version = nil)
|
71
|
+
@direction = direction
|
72
|
+
@target_version = target_version
|
73
|
+
@migrated_versions = nil
|
74
|
+
@migrations = migrations
|
75
|
+
@schema_migration = schema_migration
|
76
|
+
|
77
|
+
validate(@migrations)
|
78
|
+
|
79
|
+
@schema_migration.create_table
|
80
|
+
ClickhouseActiverecord::InternalMetadata.create_table
|
81
|
+
end
|
82
|
+
|
83
|
+
def record_version_state_after_migrating(version)
|
84
|
+
if down?
|
85
|
+
migrated.delete(version)
|
86
|
+
@schema_migration.create!(version: version.to_s, active: 0)
|
87
|
+
else
|
88
|
+
super
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ClickhouseActiverecord
|
4
|
+
|
5
|
+
class Schema < ::ActiveRecord::Schema
|
6
|
+
|
7
|
+
def define(info, &block) # :nodoc:
|
8
|
+
instance_eval(&block)
|
9
|
+
|
10
|
+
if info[:version].present?
|
11
|
+
connection.schema_migration.create_table
|
12
|
+
connection.assume_migrated_upto_version(info[:version], ClickhouseActiverecord::Migrator.migrations_paths)
|
13
|
+
end
|
14
|
+
|
15
|
+
ClickhouseActiverecord::InternalMetadata.create_table
|
16
|
+
ClickhouseActiverecord::InternalMetadata[:environment] = connection.migration_context.current_environment
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -1,10 +1,100 @@
|
|
1
1
|
module ClickhouseActiverecord
|
2
2
|
class SchemaDumper < ::ActiveRecord::ConnectionAdapters::SchemaDumper
|
3
3
|
|
4
|
+
def header(stream)
|
5
|
+
stream.puts <<HEADER
|
6
|
+
# This file is auto-generated from the current state of the database. Instead
|
7
|
+
# of editing this file, please use the migrations feature of Active Record to
|
8
|
+
# incrementally modify your database, and then regenerate this schema definition.
|
9
|
+
#
|
10
|
+
# This file is the source Rails uses to define your schema when running `rails
|
11
|
+
# clickhouse:schema:load`. When creating a new database, `rails clickhouse:schema:load` tends to
|
12
|
+
# be faster and is potentially less error prone than running all of your
|
13
|
+
# migrations from scratch. Old migrations may fail to apply correctly if those
|
14
|
+
# migrations use external dependencies or application code.
|
15
|
+
#
|
16
|
+
# It's strongly recommended that you check this file into your version control system.
|
17
|
+
|
18
|
+
ClickhouseActiverecord::Schema.define(#{define_params}) do
|
19
|
+
|
20
|
+
HEADER
|
21
|
+
end
|
22
|
+
|
4
23
|
def table(table, stream)
|
5
|
-
|
6
|
-
|
7
|
-
|
24
|
+
if table.match(/^\.inner\./).nil?
|
25
|
+
stream.puts " # TABLE: #{table}"
|
26
|
+
sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
|
27
|
+
stream.puts " # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
|
28
|
+
# super(table.gsub(/^\.inner\./, ''), stream)
|
29
|
+
|
30
|
+
# detect view table
|
31
|
+
match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
|
32
|
+
|
33
|
+
# Copy from original dumper
|
34
|
+
columns = @connection.columns(table)
|
35
|
+
begin
|
36
|
+
tbl = StringIO.new
|
37
|
+
|
38
|
+
# first dump primary key column
|
39
|
+
pk = @connection.primary_key(table)
|
40
|
+
|
41
|
+
tbl.print " create_table #{remove_prefix_and_suffix(table).inspect}"
|
42
|
+
|
43
|
+
# Add materialize flag
|
44
|
+
tbl.print ', view: true' if match
|
45
|
+
tbl.print ', materialized: true' if match && match[1].presence
|
46
|
+
|
47
|
+
case pk
|
48
|
+
when String
|
49
|
+
tbl.print ", primary_key: #{pk.inspect}" unless pk == "id"
|
50
|
+
pkcol = columns.detect { |c| c.name == pk }
|
51
|
+
pkcolspec = column_spec_for_primary_key(pkcol)
|
52
|
+
if pkcolspec.present?
|
53
|
+
tbl.print ", #{format_colspec(pkcolspec)}"
|
54
|
+
end
|
55
|
+
when Array
|
56
|
+
tbl.print ", primary_key: #{pk.inspect}"
|
57
|
+
else
|
58
|
+
tbl.print ", id: false"
|
59
|
+
end
|
60
|
+
|
61
|
+
table_options = @connection.table_options(table)
|
62
|
+
if table_options.present?
|
63
|
+
tbl.print ", #{format_options(table_options)}"
|
64
|
+
end
|
65
|
+
|
66
|
+
tbl.puts ", force: :cascade do |t|"
|
67
|
+
|
68
|
+
# then dump all non-primary key columns
|
69
|
+
columns.each do |column|
|
70
|
+
raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
|
71
|
+
next if column.name == pk
|
72
|
+
type, colspec = column_spec(column)
|
73
|
+
tbl.print " t.#{type} #{column.name.inspect}"
|
74
|
+
tbl.print ", #{format_colspec(colspec)}" if colspec.present?
|
75
|
+
tbl.puts
|
76
|
+
end
|
77
|
+
|
78
|
+
indexes_in_create(table, tbl)
|
79
|
+
|
80
|
+
tbl.puts " end"
|
81
|
+
tbl.puts
|
82
|
+
|
83
|
+
tbl.rewind
|
84
|
+
stream.print tbl.read
|
85
|
+
rescue => e
|
86
|
+
stream.puts "# Could not dump table #{table.inspect} because of following #{e.class}"
|
87
|
+
stream.puts "# #{e.message}"
|
88
|
+
stream.puts
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def format_options(options)
|
94
|
+
if options && options[:options]
|
95
|
+
options[:options] = options[:options].gsub(/^Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "\\1(\\2)")
|
96
|
+
end
|
97
|
+
super
|
8
98
|
end
|
9
99
|
end
|
10
100
|
end
|
@@ -12,8 +12,8 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato
|
|
12
12
|
private
|
13
13
|
|
14
14
|
def db_migrate_path
|
15
|
-
if defined?(Rails.application) && Rails.application
|
16
|
-
configured_migrate_path
|
15
|
+
if defined?(Rails.application) && Rails.application && respond_to?(:configured_migrate_path, true)
|
16
|
+
configured_migrate_path
|
17
17
|
else
|
18
18
|
default_migrate_path
|
19
19
|
end
|
data/lib/tasks/clickhouse.rake
CHANGED
@@ -3,67 +3,11 @@
|
|
3
3
|
namespace :clickhouse do
|
4
4
|
|
5
5
|
task prepare_schema_migration_table: :environment do
|
6
|
-
|
7
|
-
return if cluster.nil?
|
8
|
-
|
9
|
-
connection = ActiveRecord::Base.connection
|
10
|
-
key_options = connection.internal_string_options_for_primary_key
|
11
|
-
block = Proc.new do |t|
|
12
|
-
t.string :version, key_options
|
13
|
-
end
|
14
|
-
distributed_table_name = ".#{ActiveRecord::SchemaMigration.table_name}_distributed"
|
15
|
-
unless connection.table_exists?(distributed_table_name)
|
16
|
-
options = { id: false }
|
17
|
-
if replica
|
18
|
-
shard = replica.is_a?(String) ? replica : '{shard}'
|
19
|
-
options[:options] = <<-SQL
|
20
|
-
ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
|
21
|
-
PARTITION BY version ORDER BY (version) SETTINGS index_granularity = 8192
|
22
|
-
SQL
|
23
|
-
end
|
24
|
-
connection.create_table("`#{distributed_table_name}`", options, &block)
|
25
|
-
end
|
26
|
-
unless connection.table_exists?(ActiveRecord::SchemaMigration.table_name)
|
27
|
-
connection.create_table(
|
28
|
-
ActiveRecord::SchemaMigration.table_name,
|
29
|
-
id: false,
|
30
|
-
options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(version))",
|
31
|
-
&block
|
32
|
-
)
|
33
|
-
end
|
6
|
+
ClickhouseActiverecord::SchemaMigration.create_table
|
34
7
|
end
|
35
8
|
|
36
9
|
task prepare_internal_metadata_table: :environment do
|
37
|
-
|
38
|
-
return if cluster.nil?
|
39
|
-
|
40
|
-
connection = ActiveRecord::Base.connection
|
41
|
-
key_options = connection.internal_string_options_for_primary_key
|
42
|
-
block = Proc.new do |t|
|
43
|
-
t.string :key, key_options
|
44
|
-
t.string :value
|
45
|
-
t.timestamps
|
46
|
-
end
|
47
|
-
distributed_table_name = ".#{ActiveRecord::InternalMetadata.table_name}_distributed"
|
48
|
-
unless connection.table_exists?(distributed_table_name)
|
49
|
-
options = { id: false }
|
50
|
-
if replica
|
51
|
-
shard = replica.is_a?(String) ? replica : '{shard}'
|
52
|
-
options[:options] = <<-SQL
|
53
|
-
ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
|
54
|
-
PARTITION BY toDate(created_at) ORDER BY (created_at) SETTINGS index_granularity = 8192
|
55
|
-
SQL
|
56
|
-
end
|
57
|
-
connection.create_table("`#{distributed_table_name}`", options, &block)
|
58
|
-
end
|
59
|
-
unless connection.table_exists?(ActiveRecord::InternalMetadata.table_name)
|
60
|
-
connection.create_table(
|
61
|
-
ActiveRecord::InternalMetadata.table_name,
|
62
|
-
id: false,
|
63
|
-
options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(created_at))",
|
64
|
-
&block
|
65
|
-
)
|
66
|
-
end
|
10
|
+
ClickhouseActiverecord::InternalMetadata.create_table
|
67
11
|
end
|
68
12
|
|
69
13
|
task load_config: :environment do
|
@@ -76,7 +20,7 @@ namespace :clickhouse do
|
|
76
20
|
|
77
21
|
# todo not testing
|
78
22
|
desc 'Load database schema'
|
79
|
-
task load: :load_config do
|
23
|
+
task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
|
80
24
|
load("#{Rails.root}/db/clickhouse_schema.rb")
|
81
25
|
end
|
82
26
|
|
@@ -128,4 +72,9 @@ namespace :clickhouse do
|
|
128
72
|
task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
|
129
73
|
Rake::Task['db:migrate'].execute
|
130
74
|
end
|
75
|
+
|
76
|
+
desc 'Rollback the clickhouse database'
|
77
|
+
task rollback: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
|
78
|
+
Rake::Task['db:rollback'].execute
|
79
|
+
end
|
131
80
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: clickhouse-activerecord
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sergey Odintsov
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-09-
|
11
|
+
date: 2020-09-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -58,14 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '13.0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '13.0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rspec
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,7 +122,9 @@ files:
|
|
122
122
|
- lib/clickhouse-activerecord.rb
|
123
123
|
- lib/clickhouse-activerecord/arel/table.rb
|
124
124
|
- lib/clickhouse-activerecord/arel/visitors/to_sql.rb
|
125
|
+
- lib/clickhouse-activerecord/migration.rb
|
125
126
|
- lib/clickhouse-activerecord/railtie.rb
|
127
|
+
- lib/clickhouse-activerecord/schema.rb
|
126
128
|
- lib/clickhouse-activerecord/schema_dumper.rb
|
127
129
|
- lib/clickhouse-activerecord/tasks.rb
|
128
130
|
- lib/clickhouse-activerecord/version.rb
|