clickhouse-activerecord 0.3.13 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +17 -5
- data/clickhouse-activerecord.gemspec +1 -1
- data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb +20 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb +29 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +21 -5
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +53 -17
- data/lib/clickhouse-activerecord.rb +1 -0
- data/lib/clickhouse-activerecord/migration.rb +92 -0
- data/lib/clickhouse-activerecord/schema.rb +19 -0
- data/lib/clickhouse-activerecord/schema_dumper.rb +93 -3
- data/lib/clickhouse-activerecord/version.rb +1 -1
- data/lib/generators/clickhouse_migration_generator.rb +2 -2
- data/lib/tasks/clickhouse.rake +8 -59
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8020d332d2b7b707fa2776a37fbb8d2c217ec50703962d06938be53eaae22796
|
4
|
+
data.tar.gz: 6979255cb79396c6574e4b58c0e82ae39a42612fe1068b1308d98a1222c58b64
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3aa6d799adf1fc66d49c3dff6d02c5ad0349e467fb1067a1453de8bfcb3316abb46681367b7dfd3ebd8f82f13460cb658c4f167ff23398bba8b9cc459c5b6df1
|
7
|
+
data.tar.gz: a1c8e252279678efb2fe1f8c36a8a7e1c5bfda24896438245921deee5862e94c047dced14b85408b5e1343606ec7055100b813af399568f5ee405a255c268777
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
### Version 0.4.0 (Sep 18, 2020)
|
2
|
+
|
3
|
+
* Full support migration and rollback database
|
4
|
+
* Support cluster and replica. Auto inject to SQL queries.
|
5
|
+
* Fix schema dump/load
|
6
|
+
|
1
7
|
### Version 0.3.10 (Dec 20, 2019)
|
2
8
|
|
3
9
|
* Support structure dump/load [@StoneGod](https://github.com/StoneGod)
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# Clickhouse::Activerecord
|
2
2
|
|
3
3
|
A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
|
4
|
-
|
4
|
+
Support ClickHouse version from 19.14 LTS.
|
5
5
|
|
6
6
|
## Installation
|
7
7
|
|
@@ -31,8 +31,8 @@ default: &default
|
|
31
31
|
ssl: true # optional for using ssl connection
|
32
32
|
debug: true # use for showing in to log technical information
|
33
33
|
migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
|
34
|
-
|
35
|
-
|
34
|
+
cluster_name: 'cluster_name' # optional for creating tables in cluster
|
35
|
+
replica_name: '{replica}' # replica macros name, optional for creating replicated tables
|
36
36
|
```
|
37
37
|
|
38
38
|
## Usage in Rails 5
|
@@ -111,8 +111,7 @@ Migration:
|
|
111
111
|
|
112
112
|
$ rails g clickhouse_migration MIGRATION_NAME COLUMNS
|
113
113
|
$ rake clickhouse:migrate
|
114
|
-
|
115
|
-
Rollback migration not supported!
|
114
|
+
$ rake clickhouse:rollback
|
116
115
|
|
117
116
|
### Dump / Load for multiple using databases
|
118
117
|
|
@@ -159,6 +158,19 @@ ActionView.maximum(:date)
|
|
159
158
|
#=> 'Wed, 29 Nov 2017'
|
160
159
|
```
|
161
160
|
|
161
|
+
### Using replica and cluster params in connection parameters
|
162
|
+
|
163
|
+
```yml
|
164
|
+
default: &default
|
165
|
+
***
|
166
|
+
cluster_name: 'cluster_name'
|
167
|
+
replica_name: '{replica}'
|
168
|
+
```
|
169
|
+
|
170
|
+
`ON CLUSTER cluster_name` will be attach to all queries create / drop.
|
171
|
+
|
172
|
+
Engines `MergeTree` and all support replication engines will be replaced to `Replicated***('/clickhouse/tables/cluster_name/database.table', '{replica}')`
|
173
|
+
|
162
174
|
## Donations
|
163
175
|
|
164
176
|
Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:
|
@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.add_runtime_dependency 'activerecord', '>= 5.2'
|
28
28
|
|
29
29
|
spec.add_development_dependency 'bundler', '~> 1.15'
|
30
|
-
spec.add_development_dependency 'rake', '~>
|
30
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
31
31
|
spec.add_development_dependency 'rspec', '~> 3.4'
|
32
32
|
spec.add_development_dependency 'pry', '~> 0.12'
|
33
33
|
end
|
@@ -27,6 +27,26 @@ module ActiveRecord
|
|
27
27
|
|
28
28
|
create_sql
|
29
29
|
end
|
30
|
+
|
31
|
+
def visit_TableDefinition(o)
|
32
|
+
create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
|
33
|
+
create_sql << "IF NOT EXISTS " if o.if_not_exists
|
34
|
+
create_sql << "#{quote_table_name(o.name)} "
|
35
|
+
|
36
|
+
statements = o.columns.map { |c| accept c }
|
37
|
+
statements << accept(o.primary_keys) if o.primary_keys
|
38
|
+
|
39
|
+
create_sql << "(#{statements.join(', ')})" if statements.present?
|
40
|
+
add_table_options!(create_sql, table_options(o))
|
41
|
+
create_sql << " AS #{to_sql(o.as)}" if o.as
|
42
|
+
create_sql
|
43
|
+
end
|
44
|
+
|
45
|
+
# Returns any SQL string to go between CREATE and TABLE. May be nil.
|
46
|
+
def table_modifier_in_create(o)
|
47
|
+
" TEMPORARY" if o.temporary
|
48
|
+
" MATERIALIZED" if o.materialized
|
49
|
+
end
|
30
50
|
end
|
31
51
|
end
|
32
52
|
end
|
@@ -5,6 +5,35 @@ module ActiveRecord
|
|
5
5
|
module Clickhouse
|
6
6
|
class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
|
7
7
|
|
8
|
+
attr_reader :view, :materialized, :if_not_exists
|
9
|
+
|
10
|
+
def initialize(
|
11
|
+
conn,
|
12
|
+
name,
|
13
|
+
temporary: false,
|
14
|
+
if_not_exists: false,
|
15
|
+
options: nil,
|
16
|
+
as: nil,
|
17
|
+
comment: nil,
|
18
|
+
view: false,
|
19
|
+
materialized: false,
|
20
|
+
**
|
21
|
+
)
|
22
|
+
@conn = conn
|
23
|
+
@columns_hash = {}
|
24
|
+
@indexes = []
|
25
|
+
@foreign_keys = []
|
26
|
+
@primary_keys = nil
|
27
|
+
@temporary = temporary
|
28
|
+
@if_not_exists = if_not_exists
|
29
|
+
@options = options
|
30
|
+
@as = as
|
31
|
+
@name = @conn.apply_cluster(name)
|
32
|
+
@comment = comment
|
33
|
+
@view = view || materialized
|
34
|
+
@materialized = materialized
|
35
|
+
end
|
36
|
+
|
8
37
|
def integer(*args, **options)
|
9
38
|
if options[:limit] == 8
|
10
39
|
args.each { |name| column(name, :big_integer, options.except(:limit)) }
|
@@ -67,6 +67,26 @@ module ActiveRecord
|
|
67
67
|
end
|
68
68
|
end
|
69
69
|
|
70
|
+
def assume_migrated_upto_version(version, migrations_paths = nil)
|
71
|
+
version = version.to_i
|
72
|
+
sm_table = quote_table_name(schema_migration.table_name)
|
73
|
+
|
74
|
+
migrated = migration_context.get_all_versions
|
75
|
+
versions = migration_context.migrations.map(&:version)
|
76
|
+
|
77
|
+
unless migrated.include?(version)
|
78
|
+
exec_insert "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", nil, nil
|
79
|
+
end
|
80
|
+
|
81
|
+
inserting = (versions - migrated).select { |v| v < version }
|
82
|
+
if inserting.any?
|
83
|
+
if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
|
84
|
+
raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
|
85
|
+
end
|
86
|
+
execute insert_versions_sql(inserting)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
70
90
|
private
|
71
91
|
|
72
92
|
def apply_format(sql, format)
|
@@ -95,11 +115,7 @@ module ActiveRecord
|
|
95
115
|
end
|
96
116
|
|
97
117
|
def create_table_definition(*args)
|
98
|
-
|
99
|
-
Clickhouse::TableDefinition.new(self, *args)
|
100
|
-
else
|
101
|
-
Clickhouse::TableDefinition.new(*args)
|
102
|
-
end
|
118
|
+
Clickhouse::TableDefinition.new(self, *args)
|
103
119
|
end
|
104
120
|
|
105
121
|
def new_column_from_field(table_name, field)
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'clickhouse-activerecord/arel/visitors/to_sql'
|
4
4
|
require 'clickhouse-activerecord/arel/table'
|
5
|
+
require 'clickhouse-activerecord/migration'
|
5
6
|
require 'active_record/connection_adapters/clickhouse/oid/date'
|
6
7
|
require 'active_record/connection_adapters/clickhouse/oid/date_time'
|
7
8
|
require 'active_record/connection_adapters/clickhouse/oid/big_integer'
|
@@ -110,17 +111,17 @@ module ActiveRecord
|
|
110
111
|
|
111
112
|
# Support SchemaMigration from v5.2.2 to v6+
|
112
113
|
def schema_migration # :nodoc:
|
113
|
-
|
114
|
-
super
|
115
|
-
else
|
116
|
-
ActiveRecord::SchemaMigration
|
117
|
-
end
|
114
|
+
ClickhouseActiverecord::SchemaMigration
|
118
115
|
end
|
119
116
|
|
120
117
|
def migrations_paths
|
121
118
|
@full_config[:migrations_paths] || 'db/migrate_clickhouse'
|
122
119
|
end
|
123
120
|
|
121
|
+
def migration_context # :nodoc:
|
122
|
+
ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration)
|
123
|
+
end
|
124
|
+
|
124
125
|
def arel_visitor # :nodoc:
|
125
126
|
ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
|
126
127
|
end
|
@@ -209,12 +210,29 @@ module ActiveRecord
|
|
209
210
|
end
|
210
211
|
end
|
211
212
|
|
212
|
-
def
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
213
|
+
def create_view(table_name, **options)
|
214
|
+
options.merge!(view: true)
|
215
|
+
options = apply_replica(table_name, options)
|
216
|
+
td = create_table_definition(table_name, options)
|
217
|
+
yield td if block_given?
|
218
|
+
|
219
|
+
if options[:force]
|
220
|
+
drop_table(table_name, options.merge(if_exists: true))
|
221
|
+
end
|
222
|
+
|
223
|
+
execute schema_creation.accept td
|
224
|
+
end
|
225
|
+
|
226
|
+
def create_table(table_name, **options)
|
227
|
+
options = apply_replica(table_name, options)
|
228
|
+
td = create_table_definition(table_name, options)
|
229
|
+
yield td if block_given?
|
230
|
+
|
231
|
+
if options[:force]
|
232
|
+
drop_table(table_name, options.merge(if_exists: true))
|
233
|
+
end
|
234
|
+
|
235
|
+
execute schema_creation.accept td
|
218
236
|
end
|
219
237
|
|
220
238
|
# Drops a ClickHouse database.
|
@@ -230,6 +248,22 @@ module ActiveRecord
|
|
230
248
|
do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
|
231
249
|
end
|
232
250
|
|
251
|
+
def cluster
|
252
|
+
@full_config[:cluster_name]
|
253
|
+
end
|
254
|
+
|
255
|
+
def replica
|
256
|
+
@full_config[:replica_name]
|
257
|
+
end
|
258
|
+
|
259
|
+
def replica_path(table)
|
260
|
+
"/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
|
261
|
+
end
|
262
|
+
|
263
|
+
def apply_cluster(sql)
|
264
|
+
cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
|
265
|
+
end
|
266
|
+
|
233
267
|
protected
|
234
268
|
|
235
269
|
def last_inserted_id(result)
|
@@ -242,12 +276,14 @@ module ActiveRecord
|
|
242
276
|
@connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
|
243
277
|
end
|
244
278
|
|
245
|
-
def
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
279
|
+
def apply_replica(table, options)
|
280
|
+
if replica && cluster
|
281
|
+
match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
|
282
|
+
if match
|
283
|
+
options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"
|
284
|
+
end
|
285
|
+
end
|
286
|
+
options
|
251
287
|
end
|
252
288
|
end
|
253
289
|
end
|
@@ -4,6 +4,7 @@ require 'active_record/connection_adapters/clickhouse_adapter'
|
|
4
4
|
|
5
5
|
if defined?(Rails::Railtie)
|
6
6
|
require 'clickhouse-activerecord/railtie'
|
7
|
+
require 'clickhouse-activerecord/schema'
|
7
8
|
require 'clickhouse-activerecord/schema_dumper'
|
8
9
|
require 'clickhouse-activerecord/tasks'
|
9
10
|
ActiveRecord::Tasks::DatabaseTasks.register_task(/clickhouse/, "ClickhouseActiverecord::Tasks")
|
@@ -0,0 +1,92 @@
|
|
1
|
+
module ClickhouseActiverecord
|
2
|
+
|
3
|
+
class SchemaMigration < ::ActiveRecord::SchemaMigration
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def create_table
|
7
|
+
unless table_exists?
|
8
|
+
version_options = connection.internal_string_options_for_primary_key
|
9
|
+
|
10
|
+
connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
|
11
|
+
t.string :version, version_options
|
12
|
+
t.column :active, 'Int8', null: false, default: '1'
|
13
|
+
t.datetime :ver, null: false, default: -> { 'now()' }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def all_versions
|
19
|
+
from("#{table_name} FINAL").where(active: 1).order(:version).pluck(:version)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class InternalMetadata < ::ActiveRecord::InternalMetadata
|
25
|
+
class << self
|
26
|
+
def create_table
|
27
|
+
unless table_exists?
|
28
|
+
key_options = connection.internal_string_options_for_primary_key
|
29
|
+
|
30
|
+
connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
|
31
|
+
t.string :key, key_options
|
32
|
+
t.string :value
|
33
|
+
t.timestamps
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
|
41
|
+
attr_reader :migrations_paths, :schema_migration
|
42
|
+
|
43
|
+
def initialize(migrations_paths, schema_migration)
|
44
|
+
@migrations_paths = migrations_paths
|
45
|
+
@schema_migration = schema_migration
|
46
|
+
end
|
47
|
+
|
48
|
+
def down(target_version = nil)
|
49
|
+
selected_migrations = if block_given?
|
50
|
+
migrations.select { |m| yield m }
|
51
|
+
else
|
52
|
+
migrations
|
53
|
+
end
|
54
|
+
|
55
|
+
ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, target_version).migrate
|
56
|
+
end
|
57
|
+
|
58
|
+
def get_all_versions
|
59
|
+
if schema_migration.table_exists?
|
60
|
+
schema_migration.all_versions.map(&:to_i)
|
61
|
+
else
|
62
|
+
[]
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
class Migrator < ::ActiveRecord::Migrator
|
69
|
+
|
70
|
+
def initialize(direction, migrations, schema_migration, target_version = nil)
|
71
|
+
@direction = direction
|
72
|
+
@target_version = target_version
|
73
|
+
@migrated_versions = nil
|
74
|
+
@migrations = migrations
|
75
|
+
@schema_migration = schema_migration
|
76
|
+
|
77
|
+
validate(@migrations)
|
78
|
+
|
79
|
+
@schema_migration.create_table
|
80
|
+
ClickhouseActiverecord::InternalMetadata.create_table
|
81
|
+
end
|
82
|
+
|
83
|
+
def record_version_state_after_migrating(version)
|
84
|
+
if down?
|
85
|
+
migrated.delete(version)
|
86
|
+
@schema_migration.create!(version: version.to_s, active: 0)
|
87
|
+
else
|
88
|
+
super
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ClickhouseActiverecord
|
4
|
+
|
5
|
+
class Schema < ::ActiveRecord::Schema
|
6
|
+
|
7
|
+
def define(info, &block) # :nodoc:
|
8
|
+
instance_eval(&block)
|
9
|
+
|
10
|
+
if info[:version].present?
|
11
|
+
connection.schema_migration.create_table
|
12
|
+
connection.assume_migrated_upto_version(info[:version], ClickhouseActiverecord::Migrator.migrations_paths)
|
13
|
+
end
|
14
|
+
|
15
|
+
ClickhouseActiverecord::InternalMetadata.create_table
|
16
|
+
ClickhouseActiverecord::InternalMetadata[:environment] = connection.migration_context.current_environment
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -1,10 +1,100 @@
|
|
1
1
|
module ClickhouseActiverecord
|
2
2
|
class SchemaDumper < ::ActiveRecord::ConnectionAdapters::SchemaDumper
|
3
3
|
|
4
|
+
def header(stream)
|
5
|
+
stream.puts <<HEADER
|
6
|
+
# This file is auto-generated from the current state of the database. Instead
|
7
|
+
# of editing this file, please use the migrations feature of Active Record to
|
8
|
+
# incrementally modify your database, and then regenerate this schema definition.
|
9
|
+
#
|
10
|
+
# This file is the source Rails uses to define your schema when running `rails
|
11
|
+
# clickhouse:schema:load`. When creating a new database, `rails clickhouse:schema:load` tends to
|
12
|
+
# be faster and is potentially less error prone than running all of your
|
13
|
+
# migrations from scratch. Old migrations may fail to apply correctly if those
|
14
|
+
# migrations use external dependencies or application code.
|
15
|
+
#
|
16
|
+
# It's strongly recommended that you check this file into your version control system.
|
17
|
+
|
18
|
+
ClickhouseActiverecord::Schema.define(#{define_params}) do
|
19
|
+
|
20
|
+
HEADER
|
21
|
+
end
|
22
|
+
|
4
23
|
def table(table, stream)
|
5
|
-
|
6
|
-
|
7
|
-
|
24
|
+
if table.match(/^\.inner\./).nil?
|
25
|
+
stream.puts " # TABLE: #{table}"
|
26
|
+
sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
|
27
|
+
stream.puts " # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
|
28
|
+
# super(table.gsub(/^\.inner\./, ''), stream)
|
29
|
+
|
30
|
+
# detect view table
|
31
|
+
match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
|
32
|
+
|
33
|
+
# Copy from original dumper
|
34
|
+
columns = @connection.columns(table)
|
35
|
+
begin
|
36
|
+
tbl = StringIO.new
|
37
|
+
|
38
|
+
# first dump primary key column
|
39
|
+
pk = @connection.primary_key(table)
|
40
|
+
|
41
|
+
tbl.print " create_table #{remove_prefix_and_suffix(table).inspect}"
|
42
|
+
|
43
|
+
# Add materialize flag
|
44
|
+
tbl.print ', view: true' if match
|
45
|
+
tbl.print ', materialized: true' if match && match[1].presence
|
46
|
+
|
47
|
+
case pk
|
48
|
+
when String
|
49
|
+
tbl.print ", primary_key: #{pk.inspect}" unless pk == "id"
|
50
|
+
pkcol = columns.detect { |c| c.name == pk }
|
51
|
+
pkcolspec = column_spec_for_primary_key(pkcol)
|
52
|
+
if pkcolspec.present?
|
53
|
+
tbl.print ", #{format_colspec(pkcolspec)}"
|
54
|
+
end
|
55
|
+
when Array
|
56
|
+
tbl.print ", primary_key: #{pk.inspect}"
|
57
|
+
else
|
58
|
+
tbl.print ", id: false"
|
59
|
+
end
|
60
|
+
|
61
|
+
table_options = @connection.table_options(table)
|
62
|
+
if table_options.present?
|
63
|
+
tbl.print ", #{format_options(table_options)}"
|
64
|
+
end
|
65
|
+
|
66
|
+
tbl.puts ", force: :cascade do |t|"
|
67
|
+
|
68
|
+
# then dump all non-primary key columns
|
69
|
+
columns.each do |column|
|
70
|
+
raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
|
71
|
+
next if column.name == pk
|
72
|
+
type, colspec = column_spec(column)
|
73
|
+
tbl.print " t.#{type} #{column.name.inspect}"
|
74
|
+
tbl.print ", #{format_colspec(colspec)}" if colspec.present?
|
75
|
+
tbl.puts
|
76
|
+
end
|
77
|
+
|
78
|
+
indexes_in_create(table, tbl)
|
79
|
+
|
80
|
+
tbl.puts " end"
|
81
|
+
tbl.puts
|
82
|
+
|
83
|
+
tbl.rewind
|
84
|
+
stream.print tbl.read
|
85
|
+
rescue => e
|
86
|
+
stream.puts "# Could not dump table #{table.inspect} because of following #{e.class}"
|
87
|
+
stream.puts "# #{e.message}"
|
88
|
+
stream.puts
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def format_options(options)
|
94
|
+
if options && options[:options]
|
95
|
+
options[:options] = options[:options].gsub(/^Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "\\1(\\2)")
|
96
|
+
end
|
97
|
+
super
|
8
98
|
end
|
9
99
|
end
|
10
100
|
end
|
@@ -12,8 +12,8 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato
|
|
12
12
|
private
|
13
13
|
|
14
14
|
def db_migrate_path
|
15
|
-
if defined?(Rails.application) && Rails.application
|
16
|
-
configured_migrate_path
|
15
|
+
if defined?(Rails.application) && Rails.application && respond_to?(:configured_migrate_path, true)
|
16
|
+
configured_migrate_path
|
17
17
|
else
|
18
18
|
default_migrate_path
|
19
19
|
end
|
data/lib/tasks/clickhouse.rake
CHANGED
@@ -3,67 +3,11 @@
|
|
3
3
|
namespace :clickhouse do
|
4
4
|
|
5
5
|
task prepare_schema_migration_table: :environment do
|
6
|
-
|
7
|
-
return if cluster.nil?
|
8
|
-
|
9
|
-
connection = ActiveRecord::Base.connection
|
10
|
-
key_options = connection.internal_string_options_for_primary_key
|
11
|
-
block = Proc.new do |t|
|
12
|
-
t.string :version, key_options
|
13
|
-
end
|
14
|
-
distributed_table_name = ".#{ActiveRecord::SchemaMigration.table_name}_distributed"
|
15
|
-
unless connection.table_exists?(distributed_table_name)
|
16
|
-
options = { id: false }
|
17
|
-
if replica
|
18
|
-
shard = replica.is_a?(String) ? replica : '{shard}'
|
19
|
-
options[:options] = <<-SQL
|
20
|
-
ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
|
21
|
-
PARTITION BY version ORDER BY (version) SETTINGS index_granularity = 8192
|
22
|
-
SQL
|
23
|
-
end
|
24
|
-
connection.create_table("`#{distributed_table_name}`", options, &block)
|
25
|
-
end
|
26
|
-
unless connection.table_exists?(ActiveRecord::SchemaMigration.table_name)
|
27
|
-
connection.create_table(
|
28
|
-
ActiveRecord::SchemaMigration.table_name,
|
29
|
-
id: false,
|
30
|
-
options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(version))",
|
31
|
-
&block
|
32
|
-
)
|
33
|
-
end
|
6
|
+
ClickhouseActiverecord::SchemaMigration.create_table
|
34
7
|
end
|
35
8
|
|
36
9
|
task prepare_internal_metadata_table: :environment do
|
37
|
-
|
38
|
-
return if cluster.nil?
|
39
|
-
|
40
|
-
connection = ActiveRecord::Base.connection
|
41
|
-
key_options = connection.internal_string_options_for_primary_key
|
42
|
-
block = Proc.new do |t|
|
43
|
-
t.string :key, key_options
|
44
|
-
t.string :value
|
45
|
-
t.timestamps
|
46
|
-
end
|
47
|
-
distributed_table_name = ".#{ActiveRecord::InternalMetadata.table_name}_distributed"
|
48
|
-
unless connection.table_exists?(distributed_table_name)
|
49
|
-
options = { id: false }
|
50
|
-
if replica
|
51
|
-
shard = replica.is_a?(String) ? replica : '{shard}'
|
52
|
-
options[:options] = <<-SQL
|
53
|
-
ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
|
54
|
-
PARTITION BY toDate(created_at) ORDER BY (created_at) SETTINGS index_granularity = 8192
|
55
|
-
SQL
|
56
|
-
end
|
57
|
-
connection.create_table("`#{distributed_table_name}`", options, &block)
|
58
|
-
end
|
59
|
-
unless connection.table_exists?(ActiveRecord::InternalMetadata.table_name)
|
60
|
-
connection.create_table(
|
61
|
-
ActiveRecord::InternalMetadata.table_name,
|
62
|
-
id: false,
|
63
|
-
options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(created_at))",
|
64
|
-
&block
|
65
|
-
)
|
66
|
-
end
|
10
|
+
ClickhouseActiverecord::InternalMetadata.create_table
|
67
11
|
end
|
68
12
|
|
69
13
|
task load_config: :environment do
|
@@ -76,7 +20,7 @@ namespace :clickhouse do
|
|
76
20
|
|
77
21
|
# todo not testing
|
78
22
|
desc 'Load database schema'
|
79
|
-
task load: :load_config do
|
23
|
+
task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
|
80
24
|
load("#{Rails.root}/db/clickhouse_schema.rb")
|
81
25
|
end
|
82
26
|
|
@@ -128,4 +72,9 @@ namespace :clickhouse do
|
|
128
72
|
task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
|
129
73
|
Rake::Task['db:migrate'].execute
|
130
74
|
end
|
75
|
+
|
76
|
+
desc 'Rollback the clickhouse database'
|
77
|
+
task rollback: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
|
78
|
+
Rake::Task['db:rollback'].execute
|
79
|
+
end
|
131
80
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: clickhouse-activerecord
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sergey Odintsov
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-09-
|
11
|
+
date: 2020-09-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -58,14 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '13.0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '13.0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rspec
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,7 +122,9 @@ files:
|
|
122
122
|
- lib/clickhouse-activerecord.rb
|
123
123
|
- lib/clickhouse-activerecord/arel/table.rb
|
124
124
|
- lib/clickhouse-activerecord/arel/visitors/to_sql.rb
|
125
|
+
- lib/clickhouse-activerecord/migration.rb
|
125
126
|
- lib/clickhouse-activerecord/railtie.rb
|
127
|
+
- lib/clickhouse-activerecord/schema.rb
|
126
128
|
- lib/clickhouse-activerecord/schema_dumper.rb
|
127
129
|
- lib/clickhouse-activerecord/tasks.rb
|
128
130
|
- lib/clickhouse-activerecord/version.rb
|