clickhouse-activerecord 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50dff8ae7219e2f3a04ee530331d19ca0b61396437723819891c1d8b41f0692d
-  data.tar.gz: 75d50e8e2f32fa84f990bc15da6177082b7a0b0d5a508cba822b3c3afbc60c41
+  metadata.gz: 67ea4a63bb140adb5f6de102c38c097ea18f61499a882574952a9982b8d340cf
+  data.tar.gz: a1d2a759dae9062a20cc0a53bbd1cb64142062ce9ef84047f7e953b7d957e9b2
 SHA512:
-  metadata.gz: ebdeb04ba96eb81eb76efdbd09580b51719656136de38b973cea267bc1405dba03cd417058df1909b9bd1e4279c97d845292947de251792f94fc9f0b30cf65cb
-  data.tar.gz: 3a0b0feda885ab74f54f6d61b78ff02efed20d403b0bc4bff99b83ee811d4988e8620c92a84e549bc70e0ccb93c662e665a562bb51d49b5ed552bc850c076696
+  metadata.gz: 243b4d089d6349d8c12c529fbb80078ffbaa03f0033c3c69be77751846b031398895a1efdd08460699636277695a3e03eb228a398b5887a9426f4f3b63496c31
+  data.tar.gz: f790bca5af79492390de98ba2dce363e16207d987093c2e94e9f9ab55b9f769a41bae1c4a12d0b9d3757ee91f92604c1df4db17479dbbc85ced0f48294b297f3

data/README.md

@@ -31,6 +31,8 @@ default: &default
   ssl: true # optional for using ssl connection
   debug: true # use for showing in to log technical information
   migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
+  cluster: 'cluster_name' # optional for creating tables in cluster 
+  replica: '{shard}' # optional for creating system tables for shards
 ```
 
 ## Usage in Rails 5
@@ -99,6 +101,11 @@ Create / drop / purge / reset database:
     $ rake clickhouse:drop
     $ rake clickhouse:purge
     $ rake clickhouse:reset
+
+Prepare system tables for rails:
+
+    $ rake clickhouse:prepare_schema_migration_table
+    $ rake clickhouse:prepare_internal_metadata_table
     
 Migration:
 

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -17,7 +17,7 @@ module ActiveRecord
         def exec_query(sql, name = nil, binds = [], prepare: false)
           result = do_execute(sql, name)
           ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
-        rescue StandartError => _e
+        rescue StandardError => _e
           raise ActiveRecord::ActiveRecordError, "Response: #{result}"
         end
 
@@ -36,7 +36,7 @@ module ActiveRecord
         end
 
         def table_options(table)
-          sql = do_system_execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first)
+          sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
           { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
         end
 
@@ -118,7 +118,7 @@ module ActiveRecord
         protected
 
         def table_structure(table_name)
-          result = do_system_execute("DESCRIBE TABLE #{table_name}", table_name)
+          result = do_system_execute("DESCRIBE TABLE `#{table_name}`", table_name)
           data = result['data']
 
           return data unless data.empty?

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -202,16 +202,24 @@ module ActiveRecord
 
       # Create a new ClickHouse database.
       def create_database(name)
-        sql = "CREATE DATABASE #{quote_table_name(name)}"
+        sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
           res = @connection.post("/?#{@config.except(:database).to_param}", "CREATE DATABASE #{quote_table_name(name)}")
           process_response(res)
         end
       end
 
+      def create_table(table_name, comment: nil, **options)
+        super(
+          apply_cluster(table_name),
+          comment: comment,
+          **options
+        )
+      end
+
       # Drops a ClickHouse database.
       def drop_database(name) #:nodoc:
-        sql = "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
+        sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
           res = @connection.post("/?#{@config.except(:database).to_param}", sql)
           process_response(res)
@@ -219,7 +227,7 @@ module ActiveRecord
       end
 
       def drop_table(table_name, options = {}) # :nodoc:
-        do_execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
+        do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
       end
 
       protected
@@ -233,6 +241,14 @@ module ActiveRecord
       def connect
         @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
       end
+
+      def cluster
+        @full_config[:cluster]
+      end
+
+      def apply_cluster(sql)
+        cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
+      end
     end
   end
 end

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -3,7 +3,7 @@ module ClickhouseActiverecord
 
     def table(table, stream)
       stream.puts "  # TABLE: #{table}"
-      stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE #{table.gsub(/^\.inner\./, '')}")['data'].try(:first).try(:first)}"
+      stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)}"
       super(table.gsub(/^\.inner\./, ''), stream)
     end
   end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.3.12'
+  VERSION = '0.3.13'
 end

data/lib/tasks/clickhouse.rake

@@ -2,6 +2,70 @@
 
 namespace :clickhouse do
 
+  task prepare_schema_migration_table: :environment do
+    cluster, database, replica = ActiveRecord::Base.connection_config.values_at(:cluster, :database, :replica)
+    return if cluster.nil?
+
+    connection = ActiveRecord::Base.connection
+    key_options = connection.internal_string_options_for_primary_key
+    block = Proc.new do |t|
+      t.string :version, key_options
+    end
+    distributed_table_name = ".#{ActiveRecord::SchemaMigration.table_name}_distributed"
+    unless connection.table_exists?(distributed_table_name)
+      options = { id: false }
+      if replica
+        shard = replica.is_a?(String) ? replica : '{shard}'
+        options[:options] = <<-SQL
+          ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
+          PARTITION BY version ORDER BY (version) SETTINGS index_granularity = 8192
+        SQL
+      end
+      connection.create_table("`#{distributed_table_name}`", options, &block)
+    end
+    unless connection.table_exists?(ActiveRecord::SchemaMigration.table_name)
+      connection.create_table(
+        ActiveRecord::SchemaMigration.table_name,
+        id: false,
+        options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(version))",
+        &block
+      )
+    end
+  end
+
+  task prepare_internal_metadata_table: :environment do
+    cluster, database, replica = ActiveRecord::Base.connection_config.values_at(:cluster, :database, :replica)
+    return if cluster.nil?
+
+    connection = ActiveRecord::Base.connection
+    key_options = connection.internal_string_options_for_primary_key
+    block = Proc.new do |t|
+      t.string :key, key_options
+      t.string :value
+      t.timestamps
+    end
+    distributed_table_name = ".#{ActiveRecord::InternalMetadata.table_name}_distributed"
+    unless connection.table_exists?(distributed_table_name)
+      options = { id: false }
+      if replica
+        shard = replica.is_a?(String) ? replica : '{shard}'
+        options[:options] = <<-SQL
+          ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
+          PARTITION BY toDate(created_at) ORDER BY (created_at) SETTINGS index_granularity = 8192
+        SQL
+      end
+      connection.create_table("`#{distributed_table_name}`", options, &block)
+    end
+    unless connection.table_exists?(ActiveRecord::InternalMetadata.table_name)
+      connection.create_table(
+        ActiveRecord::InternalMetadata.table_name,
+        id: false,
+        options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(created_at))",
+        &block
+      )
+    end
+  end
+
   task load_config: :environment do
     ENV['SCHEMA'] = "db/clickhouse_schema.rb"
     ActiveRecord::Migrator.migrations_paths = ["db/migrate_clickhouse"]
@@ -61,7 +125,7 @@ namespace :clickhouse do
   end
 
   desc 'Migrate the clickhouse database'
-  task migrate: :load_config do
+  task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
     Rake::Task['db:migrate'].execute
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-14 00:00:00.000000000 Z
+date: 2020-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler