clickhouse-activerecord 0.2.2 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a62437ebce5fafd239daf41a6d83aea137fbd02a
-  data.tar.gz: 1af618f104ee85ded1fc71a5dd28893ef34f5925
+SHA256:
+  metadata.gz: 8a3c9c8eca31fb8287bb89238201bb073223e84af6828fe5331aa4b18b9ed7b2
+  data.tar.gz: 379946b27adc6065995dbafd4fbd65cdbf4a9bebe5e89b2dcb460136b229382a
 SHA512:
-  metadata.gz: b94de9c7bc6c2d9a09841dc14ae95a9602781818f7ea21c2a65d99213c7797d45437fcf6ef82d8e3a8fc9626236d3cad32593fc0f06cffe7be21d7571984ceca
-  data.tar.gz: a1afba5dacfde680fce68a398447c1470ab852892e77fc54c277ea372203168723ca5789d6b7b0d1ab5a11c441459a3ed6e26ec2865bb36e3d14002ab69baacb
+  metadata.gz: 2f8048d478a31674f135a9aebff80a588c31fcfd153b3264fcd28d0c2d36d3dc5785d96056954caa3ab037d33e5e24ff97fd61337058b875baa5f14922abae2d
+  data.tar.gz: 81a6bf828d38da1dfd71fc58bf7408e84407aa1c6156c64412e28247815536b231d79dc84481c417ca75eff563d5e8ebf2d149ac4eba40a717fef9d35abb1450

data/.gitignore

@@ -53,3 +53,4 @@ com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
 fabric.properties
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/CHANGELOG.md

@@ -0,0 +1,18 @@
+### Version 0.3.0 (Nov 27, 2018)
+
+ * Support materialized view
+ * Aggregated functions for view
+ * Schema dumper with SQL create table
+ * Added migrations support [@Bugagazavr](https://github.com/Bugagazavr)
+ 
+### Version 0.2.0 (Oct 3, 2017)
+
+ * Support Rails 5.0
+
+### Version 0.1.2 (Sep 27, 2017)
+
+ * Fix Big Int type
+ 
+### Version 0.1.0 (Aug 31, 2017)
+
+ * Initial release

data/README.md

@@ -1,6 +1,6 @@
 # Clickhouse::Activerecord
 
-A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.0.
+A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
 
 ## Installation
 
@@ -29,6 +29,7 @@ development_clickhouse:
   host: localhost
   username: username
   password: password
+  debug: true # use for showing in to log technical information
 ```
 
 Add to your model:
@@ -39,7 +40,15 @@ class Action < ActiveRecord::Base
 end
 ```
 
-Or global connection, but schema dump don't works:
+For materialized view model add:
+```ruby
+class ActionView < ActiveRecord::Base
+  establish_connection "#{Rails.env}_clickhouse".to_sym
+  self.is_view = true
+end
+```
+
+Or global connection:
 
 ```yml
 development:
@@ -50,21 +59,55 @@ development:
   password: password
 ```
 
-Schema dump:
+### Rake tasks
+
+Create / drop / purge / reset database:
+ 
+    $ rake clickhouse:create
+    $ rake clickhouse:drop
+    $ rake clickhouse:purge
+    $ rake clickhouse:reset
+    
+Migration:
+
+    $ rails g clickhouse_migration MIGRATION_NAME COLUMNS
+    $ rake clickhouse:migrate
+    
+Rollback migration not supported!
+
+Schema dump to `db/clickhouse_schema.rb` file:
 
     $ rake clickhouse:schema:dump
     
+Schema load from `db/clickhouse_schema.rb` file:
+
+    $ rake clickhouse:schema:load
+    
+We use schema for emulate development or tests environment on PostgreSQL adapter.
+    
 ### Insert and select data
 
 ```ruby
 Action.where(url: 'http://example.com', date: Date.current).where.not(name: nil).order(created_at: :desc).limit(10)
-=> #<ActiveRecord::Relation [#<Action *** >]>
+# Clickhouse Action Load (10.3ms)  SELECT  actions.* FROM actions WHERE actions.date = '2017-11-29' AND actions.url = 'http://example.com' AND (actions.name IS NOT NULL)  ORDER BY actions.created_at DESC LIMIT 10
+#=> #<ActiveRecord::Relation [#<Action *** >]>
 
 Action.create(url: 'http://example.com', date: Date.yesterday)
-=> true
+# Clickhouse Action Load (10.8ms)  INSERT INTO actions (url, date) VALUES ('http://example.com', '2017-11-28')
+#=> true
+ 
+ActionView.maximum(:date)
+# Clickhouse (10.3ms)  SELECT maxMerge(actions.date) FROM actions
+#=> 'Wed, 29 Nov 2017'
 ```
 
-NOTE: Creating tables in developing.
+## Donations
+
+Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:
+
+* BTC address: `1Lx2gaJtzfF2dxGFxB65YtY5kNY9xUi6ia`
+* ETH address: `0x6F094365A70fe7836A633d2eE80A1FA9758234d5`
+* XMR address: `42gP71qLB5M43RuDnrQ3vSJFFxis9Kw9VMURhpx9NLQRRwNvaZRjm2TFojAMC8Fk1BQhZNKyWhoyJSn5Ak9kppgZPjE17Zh`
 
 ## Development
 

data/clickhouse-activerecord.gemspec

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
 # coding: utf-8
 
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require File.expand_path('../lib/clickhouse-activerecord/version', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = 'clickhouse-activerecord'
-  spec.version       = '0.2.2'
+  spec.version       = ClickhouseActiverecord::VERSION
   spec.authors       = ['Sergey Odintsov']
   spec.email         = ['nixx.dj@gmail.com']
 
@@ -22,8 +24,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency "bundler", ">= 1.13.4"
-  spec.add_dependency 'rails', '>= 5.0'
+  spec.add_dependency 'activerecord', '>= 5.2'
 
   spec.add_development_dependency 'bundler', '~> 1.15'
   spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'pry'
 end

data/lib/active_record/connection_adapters/clickhouse/oid/big_integer.rb

@@ -1,10 +1,13 @@
+# frozen_string_literal: true
+
 module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
       module OID # :nodoc:
         class BigInteger < Type::BigInteger # :nodoc:
-
-          DEFAULT_LIMIT = 8
+          def type
+            :big_integer
+          end
 
           def limit
             DEFAULT_LIMIT

data/lib/active_record/connection_adapters/clickhouse/oid/date.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ActiveRecord
   module ConnectionAdapters
     module Clickhouse

data/lib/active_record/connection_adapters/clickhouse/oid/date_time.rb

@@ -1,9 +1,18 @@
+# frozen_string_literal: true
+
 module ActiveRecord
   module ConnectionAdapters
     module Clickhouse
       module OID # :nodoc:
         class DateTime < Type::DateTime # :nodoc:
 
+          def serialize(value)
+            value = super
+            return value unless value.acts_like?(:time)
+
+            value.to_time.strftime('%Y-%m-%d %H:%M:%S')
+          end
+
           def type_cast_from_database(value)
             value
           end

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class SchemaCreation < AbstractAdapter::SchemaCreation# :nodoc:
+
+        def visit_AddColumnDefinition(o)
+          +"ADD COLUMN #{accept(o.column)}"
+        end
+
+        def add_column_options!(sql, options)
+          sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
+          if options[:null] || options[:null].nil?
+            sql.gsub!(/\s+(.*)/, ' Nullable(\1)')
+          end
+          sql.gsub!(/(\sString)\(\d+\)/, '\1')
+          sql
+        end
+
+        def add_table_options!(create_sql, options)
+          if options[:options].present?
+            create_sql << " ENGINE = #{options[:options]}"
+          else
+            create_sql << " ENGINE = Log()"
+          end
+
+          create_sql
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
+
+        def integer(*args, **options)
+          if options[:limit] == 8
+            args.each { |name| column(name, :big_integer, options.except(:limit)) }
+          else
+            super
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Clickhouse
+      module SchemaStatements
+        def execute(sql, name = nil)
+          do_execute(sql, name)
+        end
+
+        def exec_insert(sql, name, _binds, _pk = nil, _sequence_name = nil)
+          new_sql = sql.dup.sub(/ (DEFAULT )?VALUES/, " VALUES")
+          do_execute(new_sql, name, format: nil)
+          true
+        end
+
+        def exec_query(sql, name = nil, binds = [], prepare: false)
+          result = do_execute(sql, name)
+          ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
+        end
+
+        def exec_update(_sql, _name = nil, _binds = [])
+          raise ActiveRecord::ActiveRecordError, 'Clickhouse update is not supported'
+        end
+
+        def exec_delete(_sql, _name = nil, _binds = [])
+          raise ActiveRecord::ActiveRecordError, 'Clickhouse delete is not supported'
+        end
+
+        def tables(name = nil)
+          result = do_system_execute('SHOW TABLES', name)
+          return [] if result.nil?
+          result['data'].flatten
+        end
+
+        def table_options(table)
+          sql = do_system_execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first)
+          { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
+        end
+
+        # @todo copied from  ActiveRecord::ConnectionAdapters::SchemaStatements v5.2.2
+        # Why version column type of String, but insert to Integer?
+        def assume_migrated_upto_version(version, migrations_paths)
+          migrations_paths = Array(migrations_paths)
+          version = version.to_i
+          sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name)
+
+          migrated = ActiveRecord::SchemaMigration.all_versions.map(&:to_i)
+          versions = migration_context.migration_files.map do |file|
+            migration_context.parse_migration_filename(file).first.to_i
+          end
+
+          unless migrated.include?(version)
+            do_execute( "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", 'SchemaMigration', format: nil)
+          end
+
+          inserting = (versions - migrated).select { |v| v < version }
+          if inserting.any?
+            if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
+              raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
+            end
+            if supports_multi_insert?
+              do_system_execute insert_versions_sql(inserting)
+            else
+              inserting.each do |v|
+                do_system_execute insert_versions_sql(v)
+              end
+            end
+          end
+        end
+
+        # Not indexes on clickhouse
+        def indexes(table_name, name = nil)
+          []
+        end
+
+        def data_sources
+          tables
+        end
+
+        def do_system_execute(sql, name = nil)
+          log_with_debug(sql, "#{adapter_name} #{name}") do
+            res = @connection.post("/?#{@config.to_param}", "#{sql} FORMAT JSONCompact")
+
+            process_response(res)
+          end
+        end
+
+        private
+
+        def apply_format(sql, format)
+          format ? "#{sql} FORMAT #{format}" : sql
+        end
+
+        def do_execute(sql, name = nil, format: 'JSONCompact')
+          log(sql, "#{adapter_name} #{name}") do
+            formatted_sql = apply_format(sql, format)
+            res = @connection.post("/?#{@config.to_param}", formatted_sql)
+
+            process_response(res)
+          end
+        end
+
+        def process_response(res)
+          case res.code.to_i
+          when 200
+            res.body.presence && JSON.parse(res.body)
+          else
+            raise ActiveRecord::ActiveRecordError,
+              "Response code: #{res.code}:\n#{res.body}"
+          end
+        end
+
+        def log_with_debug(sql, name = nil)
+          return yield unless @debug
+          log(sql, "#{name} (system)") { yield }
+        end
+
+        def schema_creation
+          Clickhouse::SchemaCreation.new(self)
+        end
+
+        def create_table_definition(*args)
+          Clickhouse::TableDefinition.new(*args)
+        end
+
+        def new_column_from_field(table_name, field)
+          sql_type = field[1]
+          type_metadata = fetch_type_metadata(sql_type)
+          default = field[3]
+          default_value = extract_value_from_default(default)
+          default_function = extract_default_function(default_value, default)
+          ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), table_name, default_function)
+        end
+
+        protected
+
+        def table_structure(table_name)
+          result = do_system_execute("DESCRIBE TABLE #{table_name}", table_name)
+          data = result['data']
+
+          return data unless data.empty?
+
+          raise ActiveRecord::StatementInvalid,
+            "Could not find table '#{table_name}'"
+        end
+        alias column_definitions table_structure
+
+        private
+
+        # Extracts the value from a PostgreSQL column default definition.
+        def extract_value_from_default(default)
+          case default
+            # Quoted types
+          when /\Anow\(\)\z/m
+            nil
+            # Boolean types
+          when "true".freeze, "false".freeze
+            default
+            # Object identifier types
+          when /\A-?\d+\z/
+            $1
+          else
+            # Anything else is blank, some user type, or some function
+            # and we can't know the value of that, so return nil.
+            nil
+          end
+        end
+
+        def extract_default_function(default_value, default) # :nodoc:
+          default if has_default_function?(default_value, default)
+        end
+
+        def has_default_function?(default_value, default) # :nodoc:
+          !default_value && (%r{\w+\(.*\)} === default)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -1,7 +1,14 @@
+# frozen_string_literal: true
+
+require 'clickhouse-activerecord/arel/visitors/to_sql'
 require 'active_record/connection_adapters/abstract_adapter'
 require 'active_record/connection_adapters/clickhouse/oid/date'
 require 'active_record/connection_adapters/clickhouse/oid/date_time'
 require 'active_record/connection_adapters/clickhouse/oid/big_integer'
+require 'active_record/connection_adapters/clickhouse/schema_definitions'
+require 'active_record/connection_adapters/clickhouse/schema_creation'
+require 'active_record/connection_adapters/clickhouse/schema_statements'
+require 'net/http'
 
 module ActiveRecord
   class Base
@@ -9,7 +16,7 @@ module ActiveRecord
       # Establishes a connection to the database that's used by all Active Record objects
       def clickhouse_connection(config)
         config = config.symbolize_keys
-        host = config[:host]
+        host = config[:host] || 'localhost'
         port = config[:port] || 8123
 
         if config.key?(:database)
@@ -18,51 +25,25 @@ module ActiveRecord
           raise ArgumentError, 'No database specified. Missing argument: database.'
         end
 
-        ConnectionAdapters::ClickhouseAdapter.new(nil, logger, [host, port], { user: config[:username], password: config[:password], database: database }.compact)
+        ConnectionAdapters::ClickhouseAdapter.new(nil, logger, [host, port], { user: config[:username], password: config[:password], database: database }.compact, config[:debug])
       end
     end
   end
 
-  module ConnectionAdapters
-
-    class ClickhouseColumn < Column
-
-      private
-
-      # Extracts the value from a PostgreSQL column default definition.
-      def extract_value_from_default(default)
-        case default
-          # Quoted types
-        when /\A[\(B]?'(.*)'.*::"?([\w. ]+)"?(?:\[\])?\z/m
-          # The default 'now'::date is CURRENT_DATE
-          if $1 == "now".freeze && $2 == "date".freeze
-            nil
-          else
-            $1.gsub("''".freeze, "'".freeze)
-          end
-          # Boolean types
-        when "true".freeze, "false".freeze
-          default
-          # Numeric types
-        when /\A\(?(-?\d+(\.\d*)?)\)?(::bigint)?\z/
-          $1
-          # Object identifier types
-        when /\A-?\d+\z/
-          $1
-        else
-          # Anything else is blank, some user type, or some function
-          # and we can't know the value of that, so return nil.
-          nil
-        end
+  module ModelSchema
+     module ClassMethods
+      def is_view
+        @is_view || false
       end
-
-      def extract_default_function(default_value, default) # :nodoc:
-        default if has_default_function?(default_value, default)
+       # @param [Boolean] value
+      def is_view=(value)
+        @is_view = value
       end
+    end
+   end
 
-      def has_default_function?(default_value, default) # :nodoc:
-        !default_value && (%r{\w+\(.*\)} === default)
-      end
+  module ConnectionAdapters
+    class ClickhouseColumn < Column
 
     end
 
@@ -70,26 +51,34 @@ module ActiveRecord
       ADAPTER_NAME = 'Clickhouse'.freeze
 
       NATIVE_DATABASE_TYPES = {
-        string: { name: 'String', limit: 255 },
+        string: { name: 'String' },
         integer: { name: 'UInt32' },
-        big_integer: { name: 'UInt64', limit: 8 },
+        big_integer: { name: 'UInt64' },
         float: { name: 'Float32' },
+        decimal: { name: 'Decimal' },
         datetime: { name: 'DateTime' },
         date: { name: 'Date' },
         boolean: { name: 'UInt8' }
       }.freeze
 
+      include Clickhouse::SchemaStatements
+
       # Initializes and connects a Clickhouse adapter.
-      def initialize(connection, logger, connection_parameters, config)
+      def initialize(connection, logger, connection_parameters, config, debug = false)
         super(connection, logger)
         @connection_parameters = connection_parameters
         @config = config
+        @debug = debug
 
         @prepared_statements = false
 
         connect
       end
 
+      def arel_visitor # :nodoc:
+        ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
+      end
+
       def native_database_types #:nodoc:
         NATIVE_DATABASE_TYPES
       end
@@ -100,11 +89,13 @@ module ActiveRecord
 
       def extract_limit(sql_type) # :nodoc:
         case sql_type
-          when 'Nullable(String)'
-            255
-          when /Nullable\(U?Int(8|16)\)/
-            4
-          when /Nullable\(U?Int(32|64)\)/
+          when /(Nullable)?\(?String\)?/
+            super('String')
+          when /(Nullable)?\(?U?Int8\)?/
+            super('int2')
+          when /(Nullable)?\(?U?Int(16|32)\)?/
+            super('int4')
+          when /(Nullable)?\(?U?Int(64)\)?/
             8
           else
             super
@@ -113,93 +104,59 @@ module ActiveRecord
 
       def initialize_type_map(m) # :nodoc:
         super
-        register_class_with_limit m, 'String', Type::String
-        register_class_with_limit m, 'Nullable(String)', Type::String
-        register_class_with_limit m, 'Uint8', Type::UnsignedInteger
+        register_class_with_limit m, %r(String), Type::String
         register_class_with_limit m, 'Date',  Clickhouse::OID::Date
         register_class_with_limit m, 'DateTime',  Clickhouse::OID::DateTime
-        m.alias_type 'UInt16', 'uint4'
-        m.alias_type 'UInt32', 'uint8'
-        m.register_type 'UInt64', Clickhouse::OID::BigInteger.new
-        m.alias_type 'Int8', 'int4'
-        m.alias_type 'Int16', 'int4'
-        m.alias_type 'Int32', 'int8'
-        m.alias_type 'Int64', 'UInt64'
-      end
-
-      # Queries the database and returns the results in an Array-like object
-      def query(sql, _name = nil)
-        res = @connection.post("/?#{@config.to_param}", "#{sql} FORMAT JSONCompact")
-        raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}" unless res.code.to_i == 200
-        JSON.parse res.body
-      end
-
-      def execute(sql, name = nil)
-        log(sql, "#{adapter_name} #{name}") do
-          query sql, name
-        end
-      end
-
-      def exec_query(sql, name = nil, binds = [], prepare: false)
-        result = execute(sql, name)
-        ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
+        register_class_with_limit m, %r(Uint8), Type::UnsignedInteger
+        m.alias_type 'UInt16', 'UInt8'
+        m.alias_type 'UInt32', 'UInt8'
+        register_class_with_limit m, %r(UInt64), Type::UnsignedInteger
+        register_class_with_limit m, %r(Int8), Type::Integer
+        m.alias_type 'Int16', 'Int8'
+        m.alias_type 'Int32', 'Int8'
+        register_class_with_limit m, %r(Int64), Type::Integer
       end
 
       # Executes insert +sql+ statement in the context of this connection using
       # +binds+ as the bind substitutes. +name+ is logged along with
       # the executed +sql+ statement.
-      def exec_insert(sql, name, _binds, _pk = nil, _sequence_name = nil)
-        log(sql, "#{adapter_name} #{name}") do
-          res = @connection.post("/?#{@config.to_param}", sql)
-          raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}" unless res.code.to_i == 200
-          true
-        end
-      end
-
-      def update(_arel, _name = nil, _binds = [])
-        raise ActiveRecord::ActiveRecordError, 'Clickhouse update is not supported'
-      end
-
-      def delete(_arel, _name = nil, _binds = [])
-        raise ActiveRecord::ActiveRecordError, 'Clickhouse delete is not supported'
-      end
 
       # SCHEMA STATEMENTS ========================================
 
-      def tables(name = nil)
-        query('SHOW TABLES', name)['data'].flatten
+      def primary_key(table_name) #:nodoc:
+        pk = table_structure(table_name).first
+        return 'id' if pk.present? && pk[0] == 'id'
+        false
       end
 
-      def new_column_from_field(table_name, field) # :nondoc:
-        sql_type = field[1]
-        type_metadata = fetch_type_metadata(sql_type)
-        ClickhouseColumn.new(field[0], field[3].present? ? field[3] : nil, type_metadata, field[1].include?('Nullable'), table_name, nil)
+      def create_schema_dumper(options) # :nodoc:
+        ClickhouseActiverecord::SchemaDumper.create(self, options)
       end
 
-      def indexes(_table_name, _name = nil) #:nodoc:
-        Rails.logger.warn 'Clickhouse indexes is not supported'
-        []
+      # Create a new ClickHouse database.
+      def create_database(name)
+        sql = "CREATE DATABASE #{quote_table_name(name)}"
+        log_with_debug(sql, adapter_name) do
+          res = @connection.post("/?#{@config.except(:database).to_param}", "CREATE DATABASE #{quote_table_name(name)}")
+          process_response(res)
+        end
       end
 
-      def primary_key(table_name) #:nodoc:
-        pk = table_structure(table_name).first
-        return 'id' if pk.present? && pk[0] == 'id'
-        false
+      # Drops a ClickHouse database.
+      def drop_database(name) #:nodoc:
+        sql = "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
+        log_with_debug(sql, adapter_name) do
+          res = @connection.post("/?#{@config.except(:database).to_param}", sql)
+          process_response(res)
+        end
       end
 
-      def data_sources
-        tables
+      def drop_table(table_name, options = {}) # :nodoc:
+        do_execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
       end
 
       protected
 
-      def table_structure(table_name)
-        data = query("DESCRIBE TABLE #{table_name}", table_name)['data']
-        raise(ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'") if data.empty?
-        data
-      end
-      alias column_definitions table_structure
-
       def last_inserted_id(result)
         result
       end

data/lib/clickhouse-activerecord.rb

@@ -1,6 +1,13 @@
+# frozen_string_literal: true
+
 require 'active_record/connection_adapters/clickhouse_adapter'
 
-require 'clickhouse/railtie' if defined?(Rails::Railtie)
+if defined?(Rails::Railtie)
+  require 'clickhouse-activerecord/railtie'
+  require 'clickhouse-activerecord/schema_dumper'
+  require 'clickhouse-activerecord/tasks'
+  ActiveRecord::Tasks::DatabaseTasks.register_task(/clickhouse/, "ClickhouseActiverecord::Tasks")
+end
 
 module ClickhouseActiverecord
 

data/lib/clickhouse-activerecord/arel/visitors/to_sql.rb

@@ -0,0 +1,21 @@
+require 'arel/visitors/to_sql'
+
+module ClickhouseActiverecord
+  module Arel
+    module Visitors
+      class ToSql < ::Arel::Visitors::ToSql
+
+        def aggregate(name, o, collector)
+          # todo how get model class from request? This method works with only rails 4.2.
+          # replacing function name for materialized view
+          # if o.expressions.first && o.expressions.first != '*' && !o.expressions.first.is_a?(String) && o.expressions.first.relation && o.expressions.first.relation.engine && o.expressions.first.relation.engine.is_view
+          #   super("#{name.downcase}Merge", o, collector)
+          # else
+            super
+          # end
+        end
+
+      end
+    end
+  end
+end

data/lib/{clickhouse → clickhouse-activerecord}/railtie.rb

@@ -1,4 +1,6 @@
-module Clickhouse
+# frozen_string_literal: true
+
+module ClickhouseActiverecord
   require 'rails'
 
   class Railtie < Rails::Railtie

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -0,0 +1,10 @@
+module ClickhouseActiverecord
+  class SchemaDumper < ::ActiveRecord::ConnectionAdapters::SchemaDumper
+
+    def table(table, stream)
+      stream.puts "  # TABLE: #{table}"
+      stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE #{table.gsub(/^\.inner\./, '')}")['data'].try(:first).try(:first)}"
+      super(table.gsub(/^\.inner\./, ''), stream)
+    end
+  end
+end

data/lib/clickhouse-activerecord/tasks.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module ClickhouseActiverecord
+  class Tasks
+
+    delegate :connection, :establish_connection, :clear_active_connections!, to: ActiveRecord::Base
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    def create
+      establish_master_connection
+      connection.create_database @configuration["database"]
+    rescue ActiveRecord::StatementInvalid => e
+      if e.cause.to_s.include?('already exists')
+        raise ActiveRecord::Tasks::DatabaseAlreadyExists
+      else
+        raise
+      end
+    end
+
+    def drop
+      establish_master_connection
+      connection.drop_database @configuration["database"]
+    end
+
+    def purge
+      clear_active_connections!
+      drop
+      create
+    end
+
+    def migrate
+      check_target_version
+
+      verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] != "false" : true
+      scope = ENV["SCOPE"]
+      verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, verbose
+      binding.pry
+      connection.migration_context.migrate(target_version) do |migration|
+        scope.blank? || scope == migration.scope
+      end
+      ActiveRecord::Base.clear_cache!
+    ensure
+      ActiveRecord::Migration.verbose = verbose_was
+    end
+
+    private
+
+    def establish_master_connection
+      establish_connection @configuration
+    end
+
+    def check_target_version
+      if target_version && !(ActiveRecord::Migration::MigrationFilenameRegexp.match?(ENV["VERSION"]) || /\A\d+\z/.match?(ENV["VERSION"]))
+        raise "Invalid format of target version: `VERSION=#{ENV['VERSION']}`"
+      end
+    end
+
+    def target_version
+      ENV["VERSION"].to_i if ENV["VERSION"] && !ENV["VERSION"].empty?
+    end
+  end
+end

data/lib/clickhouse-activerecord/version.rb

@@ -0,0 +1,3 @@
+module ClickhouseActiverecord
+  VERSION = '0.3.1'
+end

data/lib/generators/clickhouse_migration_generator.rb

@@ -0,0 +1,11 @@
+require 'rails/generators/active_record/migration/migration_generator'
+
+class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+  source_root File.join(File.dirname(ActiveRecord::Generators::MigrationGenerator.instance_method(:create_migration_file).source_location.first), "templates")
+
+  def create_migration_file
+    set_local_assigns!
+    validate_file_name!
+    migration_template @migration_template, "db/migrate_clickhouse/#{file_name}.rb"
+  end
+end

data/lib/tasks/clickhouse.rake

@@ -1,11 +1,18 @@
+# frozen_string_literal: true
+
 namespace :clickhouse do
 
+  task load_config: :environment do
+    ENV['SCHEMA'] = "db/clickhouse_schema.rb"
+    ActiveRecord::Migrator.migrations_paths = ["db/migrate_clickhouse"]
+    ActiveRecord::Base.establish_connection(:"#{Rails.env}_clickhouse")
+  end
+
   namespace :schema do
 
     # todo not testing
     desc 'Load database schema'
-    task load: :environment do
-      ActiveRecord::Base.establish_connection(:"#{Rails.env}_clickhouse")
+    task load: :load_config do
       load("#{Rails.root}/db/clickhouse_schema.rb")
     end
 
@@ -14,10 +21,35 @@ namespace :clickhouse do
       filename = "#{Rails.root}/db/clickhouse_schema.rb"
       File.open(filename, 'w:utf-8') do |file|
         ActiveRecord::Base.establish_connection(:"#{Rails.env}_clickhouse")
-        ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
+        ClickhouseActiverecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
       end
     end
 
   end
 
+  desc 'Creates the database from DATABASE_URL or config/database.yml'
+  task create: [:load_config] do
+    ActiveRecord::Tasks::DatabaseTasks.create(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"])
+  end
+
+  desc 'Drops the database from DATABASE_URL or config/database.yml'
+  task drop: [:load_config, 'db:check_protected_environments'] do
+    ActiveRecord::Tasks::DatabaseTasks.drop(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"])
+  end
+
+  desc 'Empty the database from DATABASE_URL or config/database.yml'
+  task purge: [:load_config, 'db:check_protected_environments'] do
+    ActiveRecord::Tasks::DatabaseTasks.purge(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"])
+  end
+
+  # desc 'Resets your database using your migrations for the current environment'
+  task reset: :load_config do
+    Rake::Task['clickhouse:purge'].execute
+    Rake::Task['clickhouse:migrate'].execute
+  end
+
+  desc 'Migrate the clickhouse database'
+  task migrate: :load_config do
+    Rake::Task['db:migrate'].execute
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.1
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-20 00:00:00.000000000 Z
+date: 2019-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -25,19 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.13.4
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ActiveRecord adapter for ClickHouse
 email:
 - nixx.dj@gmail.com
@@ -74,6 +102,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -85,9 +115,17 @@ files:
 - lib/active_record/connection_adapters/clickhouse/oid/big_integer.rb
 - lib/active_record/connection_adapters/clickhouse/oid/date.rb
 - lib/active_record/connection_adapters/clickhouse/oid/date_time.rb
+- lib/active_record/connection_adapters/clickhouse/schema_creation.rb
+- lib/active_record/connection_adapters/clickhouse/schema_definitions.rb
+- lib/active_record/connection_adapters/clickhouse/schema_statements.rb
 - lib/active_record/connection_adapters/clickhouse_adapter.rb
 - lib/clickhouse-activerecord.rb
-- lib/clickhouse/railtie.rb
+- lib/clickhouse-activerecord/arel/visitors/to_sql.rb
+- lib/clickhouse-activerecord/railtie.rb
+- lib/clickhouse-activerecord/schema_dumper.rb
+- lib/clickhouse-activerecord/tasks.rb
+- lib/clickhouse-activerecord/version.rb
+- lib/generators/clickhouse_migration_generator.rb
 - lib/tasks/clickhouse.rake
 homepage: https://github.com/pnixx/clickhouse-activerecord
 licenses:
@@ -108,10 +146,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: ClickHouse ActiveRecord
 test_files: []
-has_rdoc: