clickhouse-activerecord 0.3.8 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1d313720b1a43f19f51e6fa9e88ef40d136130e9
-  data.tar.gz: e82dbfb65bd38fa50df9304690bd116eef021a70
+SHA256:
+  metadata.gz: 8020d332d2b7b707fa2776a37fbb8d2c217ec50703962d06938be53eaae22796
+  data.tar.gz: 6979255cb79396c6574e4b58c0e82ae39a42612fe1068b1308d98a1222c58b64
 SHA512:
-  metadata.gz: 8fed861c69428580cddb00c79656ac7614bd26f5c47cdc0a6e4378b0578f4935aec86430b5523f599a97753be329482a8e766471b505d1fd04961ad98ac0415f
-  data.tar.gz: 581d95bdc952a7be4d5645ea0948011f7c67ca6b145c3607774a9770997382fa36279b6029b4128d43909948658b956db64d0d185351eee1f6f756cbaa36bd7c
+  metadata.gz: 3aa6d799adf1fc66d49c3dff6d02c5ad0349e467fb1067a1453de8bfcb3316abb46681367b7dfd3ebd8f82f13460cb658c4f167ff23398bba8b9cc459c5b6df1
+  data.tar.gz: a1c8e252279678efb2fe1f8c36a8a7e1c5bfda24896438245921deee5862e94c047dced14b85408b5e1343606ec7055100b813af399568f5ee405a255c268777

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### Version 0.4.0 (Sep 18, 2020)
+
+ * Full support migration and rollback database
+ * Support cluster and replica. Auto inject to SQL queries.
+ * Fix schema dump/load
+
+### Version 0.3.10 (Dec 20, 2019)
+
+ * Support structure dump/load [@StoneGod](https://github.com/StoneGod)
+
 ### Version 0.3.6 (Sep 2, 2019)
 
  * Support Rails 6.0
@@ -5,18 +15,18 @@
 
 ### Version 0.3.4 (Jun 28, 2019)
 
- * Fix DateTime sql format without microseconds for Rails 5.2 
+ * Fix DateTime sql format without microseconds for Rails 5.2
  * Support ssl connection
  * Migration support
  * Rake tasks for create / drop database
- 
+
 ### Version 0.3.0 (Nov 27, 2018)
 
  * Support materialized view
  * Aggregated functions for view
  * Schema dumper with SQL create table
  * Added migrations support [@Bugagazavr](https://github.com/Bugagazavr)
- 
+
 ### Version 0.2.0 (Oct 3, 2017)
 
  * Support Rails 5.0
@@ -24,7 +34,7 @@
 ### Version 0.1.2 (Sep 27, 2017)
 
  * Fix Big Int type
- 
+
 ### Version 0.1.0 (Aug 31, 2017)
 
  * Initial release

data/README.md

@@ -1,6 +1,7 @@
 # Clickhouse::Activerecord
 
 A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
+Support ClickHouse version from 19.14 LTS.
 
 ## Installation
 
@@ -17,8 +18,24 @@ And then execute:
 Or install it yourself as:
 
     $ gem install clickhouse-activerecord
+    
+## Available database connection parameters
+```yml
+default: &default
+  adapter: clickhouse
+  database: database
+  host: localhost
+  port: 8123
+  username: username
+  password: password
+  ssl: true # optional for using ssl connection
+  debug: true # use for showing in to log technical information
+  migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
+  cluster_name: 'cluster_name' # optional for creating tables in cluster 
+  replica_name: '{replica}' # replica macros name, optional for creating replicated tables
+```
 
-## Usage
+## Usage in Rails 5
 
 Add your `database.yml` connection information with postfix `_clickhouse` for you environment:
 
@@ -26,10 +43,6 @@ Add your `database.yml` connection information with postfix `_clickhouse` for yo
 development_clickhouse:
   adapter: clickhouse
   database: database
-  host: localhost
-  username: username
-  password: password
-  debug: true # use for showing in to log technical information
 ```
 
 Add to your model:
@@ -54,26 +67,55 @@ Or global connection:
 development:
   adapter: clickhouse
   database: database
-  host: localhost
-  username: username
-  password: password
+```
+
+## Usage in Rails 6 with second database
+
+Add your `database.yml` connection information for you environment:
+
+```yml
+development:
+  primary:
+    ...
+    
+  clickhouse:
+    adapter: clickhouse
+    database: database
+```
+
+Connection [Multiple Databases with Active Record](https://guides.rubyonrails.org/active_record_multiple_databases.html) or short example:
+
+```ruby
+class Action < ActiveRecord::Base
+  connects_to database: { writing: :clickhouse, reading: :clickhouse }
+end
 ```
 
 ### Rake tasks
 
+**Note!** For Rails 6 you can use default rake tasks if you configure `migrations_paths` in your `database.yml`, for example: `rake db:migrate`
+
 Create / drop / purge / reset database:
  
     $ rake clickhouse:create
     $ rake clickhouse:drop
     $ rake clickhouse:purge
     $ rake clickhouse:reset
+
+Prepare system tables for rails:
+
+    $ rake clickhouse:prepare_schema_migration_table
+    $ rake clickhouse:prepare_internal_metadata_table
     
 Migration:
 
     $ rails g clickhouse_migration MIGRATION_NAME COLUMNS
     $ rake clickhouse:migrate
-    
-Rollback migration not supported!
+    $ rake clickhouse:rollback
+
+### Dump / Load for multiple using databases
+
+If you using multiple databases, for example: PostgreSQL, Clickhouse.
 
 Schema dump to `db/clickhouse_schema.rb` file:
 
@@ -82,9 +124,24 @@ Schema dump to `db/clickhouse_schema.rb` file:
 Schema load from `db/clickhouse_schema.rb` file:
 
     $ rake clickhouse:schema:load
-    
+
 We use schema for emulate development or tests environment on PostgreSQL adapter.
     
+Structure dump to `db/clickhouse_structure.sql` file:
+
+    $ rake clickhouse:structure:dump
+    
+Structure load from `db/clickhouse_structure.sql` file:
+
+    $ rake clickhouse:structure:load
+
+### Dump / Load for only Clickhouse database using
+
+    $ rake db:schema:dump  
+    $ rake db:schema:load  
+    $ rake db:structure:dump  
+    $ rake db:structure:load  
+    
 ### Insert and select data
 
 ```ruby
@@ -101,6 +158,19 @@ ActionView.maximum(:date)
 #=> 'Wed, 29 Nov 2017'
 ```
 
+### Using replica and cluster params in connection parameters
+
+```yml
+default: &default
+  ***
+  cluster_name: 'cluster_name'
+  replica_name: '{replica}'
+```
+
+`ON CLUSTER cluster_name` will be attach to all queries create / drop.
+
+Engines `MergeTree` and all support replication engines will be replaced to `Replicated***('/clickhouse/tables/cluster_name/database.table', '{replica}')`
+
 ## Donations
 
 Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:

data/clickhouse-activerecord.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'activerecord', '>= 5.2'
 
   spec.add_development_dependency 'bundler', '~> 1.15'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.4'
   spec.add_development_dependency 'pry', '~> 0.12'
 end

data/lib/active_record/connection_adapters/clickhouse/oid/date_time.rb

@@ -8,6 +8,7 @@ module ActiveRecord
 
           def serialize(value)
             value = super
+            return unless value
             return value.strftime('%Y-%m-%d %H:%M:%S') unless value.acts_like?(:time)
 
             value.to_time.strftime('%Y-%m-%d %H:%M:%S')

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -27,6 +27,26 @@ module ActiveRecord
 
           create_sql
         end
+
+        def visit_TableDefinition(o)
+          create_sql = +"CREATE#{table_modifier_in_create(o)} #{o.view ? "VIEW" : "TABLE"} "
+          create_sql << "IF NOT EXISTS " if o.if_not_exists
+          create_sql << "#{quote_table_name(o.name)} "
+
+          statements = o.columns.map { |c| accept c }
+          statements << accept(o.primary_keys) if o.primary_keys
+
+          create_sql << "(#{statements.join(', ')})" if statements.present?
+          add_table_options!(create_sql, table_options(o))
+          create_sql << " AS #{to_sql(o.as)}" if o.as
+          create_sql
+        end
+
+        # Returns any SQL string to go between CREATE and TABLE. May be nil.
+        def table_modifier_in_create(o)
+          " TEMPORARY" if o.temporary
+          " MATERIALIZED" if o.materialized
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -5,6 +5,35 @@ module ActiveRecord
     module Clickhouse
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
 
+        attr_reader :view, :materialized, :if_not_exists
+
+        def initialize(
+            conn,
+            name,
+            temporary: false,
+            if_not_exists: false,
+            options: nil,
+            as: nil,
+            comment: nil,
+            view: false,
+            materialized: false,
+            **
+          )
+          @conn = conn
+          @columns_hash = {}
+          @indexes = []
+          @foreign_keys = []
+          @primary_keys = nil
+          @temporary = temporary
+          @if_not_exists = if_not_exists
+          @options = options
+          @as = as
+          @name = @conn.apply_cluster(name)
+          @comment = comment
+          @view = view || materialized
+          @materialized = materialized
+        end
+
         def integer(*args, **options)
           if options[:limit] == 8
             args.each { |name| column(name, :big_integer, options.except(:limit)) }

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -17,6 +17,8 @@ module ActiveRecord
         def exec_query(sql, name = nil, binds = [], prepare: false)
           result = do_execute(sql, name)
           ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
+        rescue StandardError => _e
+          raise ActiveRecord::ActiveRecordError, "Response: #{result}"
         end
 
         def exec_update(_sql, _name = nil, _binds = [])
@@ -34,41 +36,10 @@ module ActiveRecord
         end
 
         def table_options(table)
-          sql = do_system_execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first)
+          sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
           { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
         end
 
-        # @todo copied from  ActiveRecord::ConnectionAdapters::SchemaStatements v5.2.2
-        # Why version column type of String, but insert to Integer?
-        def assume_migrated_upto_version(version, migrations_paths)
-          migrations_paths = Array(migrations_paths)
-          version = version.to_i
-          sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name)
-
-          migrated = ActiveRecord::SchemaMigration.all_versions.map(&:to_i)
-          versions = migration_context.migration_files.map do |file|
-            migration_context.parse_migration_filename(file).first.to_i
-          end
-
-          unless migrated.include?(version)
-            do_execute( "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", 'SchemaMigration', format: nil)
-          end
-
-          inserting = (versions - migrated).select { |v| v < version }
-          if inserting.any?
-            if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
-              raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
-            end
-            if supports_multi_insert?
-              do_system_execute insert_versions_sql(inserting.map(&:to_s))
-            else
-              inserting.each do |v|
-                do_system_execute insert_versions_sql(v)
-              end
-            end
-          end
-        end
-
         # Not indexes on clickhouse
         def indexes(table_name, name = nil)
           []
@@ -86,21 +57,42 @@ module ActiveRecord
           end
         end
 
-        private
-
-        def apply_format(sql, format)
-          format ? "#{sql} FORMAT #{format}" : sql
-        end
-
-        def do_execute(sql, name = nil, format: 'JSONCompact')
+        def do_execute(sql, name = nil, format: 'JSONCompact', settings: {})
           log(sql, "#{adapter_name} #{name}") do
             formatted_sql = apply_format(sql, format)
-            res = @connection.post("/?#{@config.to_param}", formatted_sql)
+            request_params = @config || {}
+            res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql)
 
             process_response(res)
           end
         end
 
+        def assume_migrated_upto_version(version, migrations_paths = nil)
+          version = version.to_i
+          sm_table = quote_table_name(schema_migration.table_name)
+
+          migrated = migration_context.get_all_versions
+          versions = migration_context.migrations.map(&:version)
+
+          unless migrated.include?(version)
+            exec_insert "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", nil, nil
+          end
+
+          inserting = (versions - migrated).select { |v| v < version }
+          if inserting.any?
+            if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
+              raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
+            end
+            execute insert_versions_sql(inserting)
+          end
+        end
+
+        private
+
+        def apply_format(sql, format)
+          format ? "#{sql} FORMAT #{format}" : sql
+        end
+
         def process_response(res)
           case res.code.to_i
           when 200
@@ -109,6 +101,8 @@ module ActiveRecord
             raise ActiveRecord::ActiveRecordError,
               "Response code: #{res.code}:\n#{res.body}"
           end
+        rescue JSON::ParserError
+          res.body
         end
 
         def log_with_debug(sql, name = nil)
@@ -121,7 +115,7 @@ module ActiveRecord
         end
 
         def create_table_definition(*args)
-          Clickhouse::TableDefinition.new(*args)
+          Clickhouse::TableDefinition.new(self, *args)
         end
 
         def new_column_from_field(table_name, field)
@@ -140,7 +134,7 @@ module ActiveRecord
         protected
 
         def table_structure(table_name)
-          result = do_system_execute("DESCRIBE TABLE #{table_name}", table_name)
+          result = do_system_execute("DESCRIBE TABLE `#{table_name}`", table_name)
           data = result['data']
 
           return data unless data.empty?

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -2,6 +2,7 @@
 
 require 'clickhouse-activerecord/arel/visitors/to_sql'
 require 'clickhouse-activerecord/arel/table'
+require 'clickhouse-activerecord/migration'
 require 'active_record/connection_adapters/clickhouse/oid/date'
 require 'active_record/connection_adapters/clickhouse/oid/date_time'
 require 'active_record/connection_adapters/clickhouse/oid/big_integer'
@@ -18,6 +19,7 @@ module ActiveRecord
         config = config.symbolize_keys
         host = config[:host] || 'localhost'
         port = config[:port] || 8123
+        ssl = config[:ssl].present? ? config[:ssl] : port == 443
 
         if config.key?(:database)
           database = config[:database]
@@ -25,7 +27,7 @@ module ActiveRecord
           raise ArgumentError, 'No database specified. Missing argument: database.'
         end
 
-        ConnectionAdapters::ClickhouseAdapter.new(nil, logger, [host, port], { user: config[:username], password: config[:password], database: database }.compact, config[:debug])
+        ConnectionAdapters::ClickhouseAdapter.new(logger, [host, port, ssl], { user: config[:username], password: config[:password], database: database }.compact, config)
       end
     end
   end
@@ -92,21 +94,34 @@ module ActiveRecord
       include Clickhouse::SchemaStatements
 
       # Initializes and connects a Clickhouse adapter.
-      def initialize(connection, logger, connection_parameters, config, debug = false)
-        super(connection, logger)
+      def initialize(logger, connection_parameters, config, full_config)
+        super(nil, logger)
         @connection_parameters = connection_parameters
         @config = config
-        @debug = debug
+        @debug = full_config[:debug] || false
+        @full_config = full_config
 
-        if ActiveRecord::version >= Gem::Version.new('6')
+        @prepared_statements = false
+        if ActiveRecord::version == Gem::Version.new('6.0.0')
           @prepared_statement_status = Concurrent::ThreadLocalVar.new(false)
-        else
-          @prepared_statements = false
         end
 
         connect
       end
 
+      # Support SchemaMigration from v5.2.2 to v6+
+      def schema_migration # :nodoc:
+        ClickhouseActiverecord::SchemaMigration
+      end
+
+      def migrations_paths
+        @full_config[:migrations_paths] || 'db/migrate_clickhouse'
+      end
+
+      def migration_context # :nodoc:
+        ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration)
+      end
+
       def arel_visitor # :nodoc:
         ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
       end
@@ -188,16 +203,41 @@ module ActiveRecord
 
       # Create a new ClickHouse database.
       def create_database(name)
-        sql = "CREATE DATABASE #{quote_table_name(name)}"
+        sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
           res = @connection.post("/?#{@config.except(:database).to_param}", "CREATE DATABASE #{quote_table_name(name)}")
           process_response(res)
         end
       end
 
+      def create_view(table_name, **options)
+        options.merge!(view: true)
+        options = apply_replica(table_name, options)
+        td = create_table_definition(table_name, options)
+        yield td if block_given?
+
+        if options[:force]
+          drop_table(table_name, options.merge(if_exists: true))
+        end
+
+        execute schema_creation.accept td
+      end
+
+      def create_table(table_name, **options)
+        options = apply_replica(table_name, options)
+        td = create_table_definition(table_name, options)
+        yield td if block_given?
+
+        if options[:force]
+          drop_table(table_name, options.merge(if_exists: true))
+        end
+
+        execute schema_creation.accept td
+      end
+
       # Drops a ClickHouse database.
       def drop_database(name) #:nodoc:
-        sql = "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
+        sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
           res = @connection.post("/?#{@config.except(:database).to_param}", sql)
           process_response(res)
@@ -205,7 +245,23 @@ module ActiveRecord
       end
 
       def drop_table(table_name, options = {}) # :nodoc:
-        do_execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
+        do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
+      end
+
+      def cluster
+        @full_config[:cluster_name]
+      end
+
+      def replica
+        @full_config[:replica_name]
+      end
+
+      def replica_path(table)
+        "/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
+      end
+
+      def apply_cluster(sql)
+        cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
       end
 
       protected
@@ -217,8 +273,17 @@ module ActiveRecord
       private
 
       def connect
-        # for ssl port need use ssl
-        @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[1] == 443)
+        @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
+      end
+
+      def apply_replica(table, options)
+        if replica && cluster
+          match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
+          if match
+            options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"
+          end
+        end
+        options
       end
     end
   end

data/lib/clickhouse-activerecord.rb

@@ -4,6 +4,7 @@ require 'active_record/connection_adapters/clickhouse_adapter'
 
 if defined?(Rails::Railtie)
   require 'clickhouse-activerecord/railtie'
+  require 'clickhouse-activerecord/schema'
   require 'clickhouse-activerecord/schema_dumper'
   require 'clickhouse-activerecord/tasks'
   ActiveRecord::Tasks::DatabaseTasks.register_task(/clickhouse/, "ClickhouseActiverecord::Tasks")

data/lib/clickhouse-activerecord/migration.rb

@@ -0,0 +1,92 @@
+module ClickhouseActiverecord
+
+  class SchemaMigration < ::ActiveRecord::SchemaMigration
+    class << self
+
+      def create_table
+        unless table_exists?
+          version_options = connection.internal_string_options_for_primary_key
+
+          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
+            t.string :version, version_options
+            t.column :active, 'Int8', null: false, default: '1'
+            t.datetime :ver, null: false, default: -> { 'now()' }
+          end
+        end
+      end
+
+      def all_versions
+        from("#{table_name} FINAL").where(active: 1).order(:version).pluck(:version)
+      end
+    end
+  end
+
+  class InternalMetadata < ::ActiveRecord::InternalMetadata
+    class << self
+      def create_table
+        unless table_exists?
+          key_options = connection.internal_string_options_for_primary_key
+
+          connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
+            t.string :key, key_options
+            t.string :value
+            t.timestamps
+          end
+        end
+      end
+    end
+  end
+
+  class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
+    attr_reader :migrations_paths, :schema_migration
+
+    def initialize(migrations_paths, schema_migration)
+      @migrations_paths = migrations_paths
+      @schema_migration = schema_migration
+    end
+
+    def down(target_version = nil)
+      selected_migrations = if block_given?
+        migrations.select { |m| yield m }
+      else
+        migrations
+      end
+
+      ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, target_version).migrate
+    end
+
+    def get_all_versions
+      if schema_migration.table_exists?
+        schema_migration.all_versions.map(&:to_i)
+      else
+        []
+      end
+    end
+
+  end
+
+  class Migrator < ::ActiveRecord::Migrator
+
+    def initialize(direction, migrations, schema_migration, target_version = nil)
+      @direction         = direction
+      @target_version    = target_version
+      @migrated_versions = nil
+      @migrations        = migrations
+      @schema_migration  = schema_migration
+
+      validate(@migrations)
+
+      @schema_migration.create_table
+      ClickhouseActiverecord::InternalMetadata.create_table
+    end
+
+    def record_version_state_after_migrating(version)
+      if down?
+        migrated.delete(version)
+        @schema_migration.create!(version: version.to_s, active: 0)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/clickhouse-activerecord/schema.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ClickhouseActiverecord
+
+  class Schema < ::ActiveRecord::Schema
+
+    def define(info, &block) # :nodoc:
+      instance_eval(&block)
+
+      if info[:version].present?
+        connection.schema_migration.create_table
+        connection.assume_migrated_upto_version(info[:version], ClickhouseActiverecord::Migrator.migrations_paths)
+      end
+
+      ClickhouseActiverecord::InternalMetadata.create_table
+      ClickhouseActiverecord::InternalMetadata[:environment] = connection.migration_context.current_environment
+    end
+  end
+end

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -1,10 +1,100 @@
 module ClickhouseActiverecord
   class SchemaDumper < ::ActiveRecord::ConnectionAdapters::SchemaDumper
 
+    def header(stream)
+      stream.puts <<HEADER
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `rails
+# clickhouse:schema:load`. When creating a new database, `rails clickhouse:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ClickhouseActiverecord::Schema.define(#{define_params}) do
+
+HEADER
+    end
+
     def table(table, stream)
-      stream.puts "  # TABLE: #{table}"
-      stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE #{table.gsub(/^\.inner\./, '')}")['data'].try(:first).try(:first)}"
-      super(table.gsub(/^\.inner\./, ''), stream)
+      if table.match(/^\.inner\./).nil?
+        stream.puts "  # TABLE: #{table}"
+        sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
+        stream.puts "  # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
+        # super(table.gsub(/^\.inner\./, ''), stream)
+
+        # detect view table
+        match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
+
+        # Copy from original dumper
+        columns = @connection.columns(table)
+        begin
+          tbl = StringIO.new
+
+          # first dump primary key column
+          pk = @connection.primary_key(table)
+
+          tbl.print "  create_table #{remove_prefix_and_suffix(table).inspect}"
+
+          # Add materialize flag
+          tbl.print ', view: true' if match
+          tbl.print ', materialized: true' if match && match[1].presence
+
+          case pk
+          when String
+            tbl.print ", primary_key: #{pk.inspect}" unless pk == "id"
+            pkcol = columns.detect { |c| c.name == pk }
+            pkcolspec = column_spec_for_primary_key(pkcol)
+            if pkcolspec.present?
+              tbl.print ", #{format_colspec(pkcolspec)}"
+            end
+          when Array
+            tbl.print ", primary_key: #{pk.inspect}"
+          else
+            tbl.print ", id: false"
+          end
+
+          table_options = @connection.table_options(table)
+          if table_options.present?
+            tbl.print ", #{format_options(table_options)}"
+          end
+
+          tbl.puts ", force: :cascade do |t|"
+
+          # then dump all non-primary key columns
+          columns.each do |column|
+            raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
+            next if column.name == pk
+            type, colspec = column_spec(column)
+            tbl.print "    t.#{type} #{column.name.inspect}"
+            tbl.print ", #{format_colspec(colspec)}" if colspec.present?
+            tbl.puts
+          end
+
+          indexes_in_create(table, tbl)
+
+          tbl.puts "  end"
+          tbl.puts
+
+          tbl.rewind
+          stream.print tbl.read
+        rescue => e
+          stream.puts "# Could not dump table #{table.inspect} because of following #{e.class}"
+          stream.puts "#   #{e.message}"
+          stream.puts
+        end
+      end
+    end
+
+    def format_options(options)
+      if options && options[:options]
+        options[:options] = options[:options].gsub(/^Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "\\1(\\2)")
+      end
+      super
     end
   end
 end

data/lib/clickhouse-activerecord/tasks.rb

@@ -31,13 +31,27 @@ module ClickhouseActiverecord
       create
     end
 
+    def structure_dump(*args)
+      tables = connection.execute("SHOW TABLES FROM #{@configuration['database']}")['data'].flatten
+
+      File.open(args.first, 'w:utf-8') do |file|
+        tables.each do |table|
+          next if table.match(/\.inner/)
+          file.puts connection.execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first).gsub("#{@configuration['database']}.", '') + ";\n\n"
+        end
+      end
+    end
+
+    def structure_load(*args)
+      File.read(args.first).split(";\n\n").each { |sql| connection.execute(sql) }
+    end
+
     def migrate
       check_target_version
 
       verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] != "false" : true
       scope = ENV["SCOPE"]
       verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, verbose
-      binding.pry
       connection.migration_context.migrate(target_version) do |migration|
         scope.blank? || scope == migration.scope
       end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.3.8'
+  VERSION = '0.4.0'
 end

data/lib/generators/clickhouse_migration_generator.rb

@@ -6,6 +6,20 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato
   def create_migration_file
     set_local_assigns!
     validate_file_name!
-    migration_template @migration_template, "db/migrate_clickhouse/#{file_name}.rb"
+    migration_template @migration_template, File.join(db_migrate_path, "#{file_name}.rb")
+  end
+
+  private
+
+  def db_migrate_path
+    if defined?(Rails.application) && Rails.application && respond_to?(:configured_migrate_path, true)
+      configured_migrate_path
+    else
+      default_migrate_path
+    end
+  end
+
+  def default_migrate_path
+    "db/migrate_clickhouse"
   end
 end

data/lib/tasks/clickhouse.rake

@@ -2,6 +2,14 @@
 
 namespace :clickhouse do
 
+  task prepare_schema_migration_table: :environment do
+    ClickhouseActiverecord::SchemaMigration.create_table
+  end
+
+  task prepare_internal_metadata_table: :environment do
+    ClickhouseActiverecord::InternalMetadata.create_table
+  end
+
   task load_config: :environment do
     ENV['SCHEMA'] = "db/clickhouse_schema.rb"
     ActiveRecord::Migrator.migrations_paths = ["db/migrate_clickhouse"]
@@ -12,7 +20,7 @@ namespace :clickhouse do
 
     # todo not testing
     desc 'Load database schema'
-    task load: :load_config do
+    task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
       load("#{Rails.root}/db/clickhouse_schema.rb")
     end
 
@@ -27,6 +35,18 @@ namespace :clickhouse do
 
   end
 
+  namespace :structure do
+    desc 'Load database structure'
+    task load: [:load_config, 'db:check_protected_environments'] do
+      ClickhouseActiverecord::Tasks.new(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"]).structure_load("#{Rails.root}/db/clickhouse_structure.sql")
+    end
+
+    desc 'Dump database structure'
+    task dump: [:load_config, 'db:check_protected_environments'] do
+      ClickhouseActiverecord::Tasks.new(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"]).structure_dump("#{Rails.root}/db/clickhouse_structure.sql")
+    end
+  end
+
   desc 'Creates the database from DATABASE_URL or config/database.yml'
   task create: [:load_config] do
     ActiveRecord::Tasks::DatabaseTasks.create(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"])
@@ -49,7 +69,12 @@ namespace :clickhouse do
   end
 
   desc 'Migrate the clickhouse database'
-  task migrate: :load_config do
+  task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
     Rake::Task['db:migrate'].execute
   end
+
+  desc 'Rollback the clickhouse database'
+  task rollback: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
+    Rake::Task['db:rollback'].execute
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.4.0
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-04 00:00:00.000000000 Z
+date: 2020-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +122,9 @@ files:
 - lib/clickhouse-activerecord.rb
 - lib/clickhouse-activerecord/arel/table.rb
 - lib/clickhouse-activerecord/arel/visitors/to_sql.rb
+- lib/clickhouse-activerecord/migration.rb
 - lib/clickhouse-activerecord/railtie.rb
+- lib/clickhouse-activerecord/schema.rb
 - lib/clickhouse-activerecord/schema_dumper.rb
 - lib/clickhouse-activerecord/tasks.rb
 - lib/clickhouse-activerecord/version.rb
@@ -147,8 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: ClickHouse ActiveRecord