clickhouse-activerecord 0.3.4 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b9461371fc0031e2d5f70577857979c6ab9f210028c733a5d1296c22157527c
-  data.tar.gz: b3fb5414957d5c4b9ae3cfe32de5c3f292ef82ba390ac8dea45466663e081733
+  metadata.gz: 50dff8ae7219e2f3a04ee530331d19ca0b61396437723819891c1d8b41f0692d
+  data.tar.gz: 75d50e8e2f32fa84f990bc15da6177082b7a0b0d5a508cba822b3c3afbc60c41
 SHA512:
-  metadata.gz: b72edc5f389e47bc3e6e95bb7ea3e7b239bf35a0d208e721b0fcdc82c6c9e90b68719d0c3e3dc28daf99cea1ee8a053f0ca83173be6dbb9212946f3fc6a30994
-  data.tar.gz: 610bd1ee31541e875f78a2339bbaff8f90f77074a31017f06d8971d9bc00cc27a13c2bda7f9ef2cde2643d7fb971c904aea76bf59181f78af93b5d1597b5f4b8
+  metadata.gz: ebdeb04ba96eb81eb76efdbd09580b51719656136de38b973cea267bc1405dba03cd417058df1909b9bd1e4279c97d845292947de251792f94fc9f0b30cf65cb
+  data.tar.gz: 3a0b0feda885ab74f54f6d61b78ff02efed20d403b0bc4bff99b83ee811d4988e8620c92a84e549bc70e0ccb93c662e665a562bb51d49b5ed552bc850c076696

data/CHANGELOG.md

@@ -1,10 +1,26 @@
+### Version 0.3.10 (Dec 20, 2019)
+
+ * Support structure dump/load [@StoneGod](https://github.com/StoneGod)
+
+### Version 0.3.6 (Sep 2, 2019)
+
+ * Support Rails 6.0
+ * Fix relation `last` method
+
+### Version 0.3.4 (Jun 28, 2019)
+
+ * Fix DateTime sql format without microseconds for Rails 5.2
+ * Support ssl connection
+ * Migration support
+ * Rake tasks for create / drop database
+
 ### Version 0.3.0 (Nov 27, 2018)
 
  * Support materialized view
  * Aggregated functions for view
  * Schema dumper with SQL create table
  * Added migrations support [@Bugagazavr](https://github.com/Bugagazavr)
- 
+
 ### Version 0.2.0 (Oct 3, 2017)
 
  * Support Rails 5.0
@@ -12,7 +28,7 @@
 ### Version 0.1.2 (Sep 27, 2017)
 
  * Fix Big Int type
- 
+
 ### Version 0.1.0 (Aug 31, 2017)
 
  * Initial release

data/README.md

@@ -1,6 +1,7 @@
 # Clickhouse::Activerecord
 
 A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
+Tested on ClickHouse version 18.14.
 
 ## Installation
 
@@ -17,8 +18,22 @@ And then execute:
 Or install it yourself as:
 
     $ gem install clickhouse-activerecord
+    
+## Available database connection parameters
+```yml
+default: &default
+  adapter: clickhouse
+  database: database
+  host: localhost
+  port: 8123
+  username: username
+  password: password
+  ssl: true # optional for using ssl connection
+  debug: true # use for showing in to log technical information
+  migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
+```
 
-## Usage
+## Usage in Rails 5
 
 Add your `database.yml` connection information with postfix `_clickhouse` for you environment:
 
@@ -26,10 +41,6 @@ Add your `database.yml` connection information with postfix `_clickhouse` for yo
 development_clickhouse:
   adapter: clickhouse
   database: database
-  host: localhost
-  username: username
-  password: password
-  debug: true # use for showing in to log technical information
 ```
 
 Add to your model:
@@ -54,13 +65,34 @@ Or global connection:
 development:
   adapter: clickhouse
   database: database
-  host: localhost
-  username: username
-  password: password
+```
+
+## Usage in Rails 6 with second database
+
+Add your `database.yml` connection information for you environment:
+
+```yml
+development:
+  primary:
+    ...
+    
+  clickhouse:
+    adapter: clickhouse
+    database: database
+```
+
+Connection [Multiple Databases with Active Record](https://guides.rubyonrails.org/active_record_multiple_databases.html) or short example:
+
+```ruby
+class Action < ActiveRecord::Base
+  connects_to database: { writing: :clickhouse, reading: :clickhouse }
+end
 ```
 
 ### Rake tasks
 
+**Note!** For Rails 6 you can use default rake tasks if you configure `migrations_paths` in your `database.yml`, for example: `rake db:migrate`
+
 Create / drop / purge / reset database:
  
     $ rake clickhouse:create
@@ -75,6 +107,10 @@ Migration:
     
 Rollback migration not supported!
 
+### Dump / Load for multiple using databases
+
+If you using multiple databases, for example: PostgreSQL, Clickhouse.
+
 Schema dump to `db/clickhouse_schema.rb` file:
 
     $ rake clickhouse:schema:dump
@@ -82,9 +118,24 @@ Schema dump to `db/clickhouse_schema.rb` file:
 Schema load from `db/clickhouse_schema.rb` file:
 
     $ rake clickhouse:schema:load
-    
+
 We use schema for emulate development or tests environment on PostgreSQL adapter.
     
+Structure dump to `db/clickhouse_structure.sql` file:
+
+    $ rake clickhouse:structure:dump
+    
+Structure load from `db/clickhouse_structure.sql` file:
+
+    $ rake clickhouse:structure:load
+
+### Dump / Load for only Clickhouse database using
+
+    $ rake db:schema:dump  
+    $ rake db:schema:load  
+    $ rake db:structure:dump  
+    $ rake db:structure:load  
+    
 ### Insert and select data
 
 ```ruby
@@ -105,7 +156,7 @@ ActionView.maximum(:date)
 
 Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:
 
-* BTC address: `1Lx2gaJtzfF2dxGFxB65YtY5kNY9xUi6ia`
+* BTC address: `1H3rhpf7WEF5JmMZ3PVFMQc7Hm29THgUfN`
 * ETH address: `0x6F094365A70fe7836A633d2eE80A1FA9758234d5`
 * XMR address: `42gP71qLB5M43RuDnrQ3vSJFFxis9Kw9VMURhpx9NLQRRwNvaZRjm2TFojAMC8Fk1BQhZNKyWhoyJSn5Ak9kppgZPjE17Zh`
 

data/clickhouse-activerecord.gemspec

@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'bundler', '~> 1.13', '>= 1.13.4'
-  spec.add_runtime_dependency 'activerecord', '~> 5.2'
+  spec.add_runtime_dependency 'bundler', '>= 1.13.4'
+  spec.add_runtime_dependency 'activerecord', '>= 5.2'
 
   spec.add_development_dependency 'bundler', '~> 1.15'
   spec.add_development_dependency 'rake', '~> 10.0'

data/lib/active_record/connection_adapters/clickhouse/oid/date_time.rb

@@ -8,6 +8,7 @@ module ActiveRecord
 
           def serialize(value)
             value = super
+            return unless value
             return value.strftime('%Y-%m-%d %H:%M:%S') unless value.acts_like?(:time)
 
             value.to_time.strftime('%Y-%m-%d %H:%M:%S')

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -17,6 +17,8 @@ module ActiveRecord
         def exec_query(sql, name = nil, binds = [], prepare: false)
           result = do_execute(sql, name)
           ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
+        rescue StandartError => _e
+          raise ActiveRecord::ActiveRecordError, "Response: #{result}"
         end
 
         def exec_update(_sql, _name = nil, _binds = [])
@@ -38,37 +40,6 @@ module ActiveRecord
           { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
         end
 
-        # @todo copied from  ActiveRecord::ConnectionAdapters::SchemaStatements v5.2.2
-        # Why version column type of String, but insert to Integer?
-        def assume_migrated_upto_version(version, migrations_paths)
-          migrations_paths = Array(migrations_paths)
-          version = version.to_i
-          sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name)
-
-          migrated = ActiveRecord::SchemaMigration.all_versions.map(&:to_i)
-          versions = migration_context.migration_files.map do |file|
-            migration_context.parse_migration_filename(file).first.to_i
-          end
-
-          unless migrated.include?(version)
-            do_execute( "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", 'SchemaMigration', format: nil)
-          end
-
-          inserting = (versions - migrated).select { |v| v < version }
-          if inserting.any?
-            if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
-              raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
-            end
-            if supports_multi_insert?
-              do_system_execute insert_versions_sql(inserting.map(&:to_s))
-            else
-              inserting.each do |v|
-                do_system_execute insert_versions_sql(v)
-              end
-            end
-          end
-        end
-
         # Not indexes on clickhouse
         def indexes(table_name, name = nil)
           []
@@ -86,21 +57,22 @@ module ActiveRecord
           end
         end
 
-        private
-
-        def apply_format(sql, format)
-          format ? "#{sql} FORMAT #{format}" : sql
-        end
-
-        def do_execute(sql, name = nil, format: 'JSONCompact')
+        def do_execute(sql, name = nil, format: 'JSONCompact', settings: {})
           log(sql, "#{adapter_name} #{name}") do
             formatted_sql = apply_format(sql, format)
-            res = @connection.post("/?#{@config.to_param}", formatted_sql)
+            request_params = @config || {}
+            res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql)
 
             process_response(res)
           end
         end
 
+        private
+
+        def apply_format(sql, format)
+          format ? "#{sql} FORMAT #{format}" : sql
+        end
+
         def process_response(res)
           case res.code.to_i
           when 200
@@ -109,6 +81,8 @@ module ActiveRecord
             raise ActiveRecord::ActiveRecordError,
               "Response code: #{res.code}:\n#{res.body}"
           end
+        rescue JSON::ParserError
+          res.body
         end
 
         def log_with_debug(sql, name = nil)
@@ -121,7 +95,11 @@ module ActiveRecord
         end
 
         def create_table_definition(*args)
-          Clickhouse::TableDefinition.new(*args)
+          if ActiveRecord::version >= Gem::Version.new('6')
+            Clickhouse::TableDefinition.new(self, *args)
+          else
+            Clickhouse::TableDefinition.new(*args)
+          end
         end
 
         def new_column_from_field(table_name, field)
@@ -130,7 +108,11 @@ module ActiveRecord
           default = field[3]
           default_value = extract_value_from_default(default)
           default_function = extract_default_function(default_value, default)
-          ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), table_name, default_function)
+          if ActiveRecord::version >= Gem::Version.new('6')
+            ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), default_function)
+          else
+            ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), table_name, default_function)
+          end
         end
 
         protected

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'clickhouse-activerecord/arel/visitors/to_sql'
-require 'active_record/connection_adapters/abstract_adapter'
+require 'clickhouse-activerecord/arel/table'
 require 'active_record/connection_adapters/clickhouse/oid/date'
 require 'active_record/connection_adapters/clickhouse/oid/date_time'
 require 'active_record/connection_adapters/clickhouse/oid/big_integer'
@@ -18,6 +18,7 @@ module ActiveRecord
         config = config.symbolize_keys
         host = config[:host] || 'localhost'
         port = config[:port] || 8123
+        ssl = config[:ssl].present? ? config[:ssl] : port == 443
 
         if config.key?(:database)
           database = config[:database]
@@ -25,7 +26,30 @@ module ActiveRecord
           raise ArgumentError, 'No database specified. Missing argument: database.'
         end
 
-        ConnectionAdapters::ClickhouseAdapter.new(nil, logger, [host, port], { user: config[:username], password: config[:password], database: database }.compact, config[:debug])
+        ConnectionAdapters::ClickhouseAdapter.new(logger, [host, port, ssl], { user: config[:username], password: config[:password], database: database }.compact, config)
+      end
+    end
+  end
+
+  class Relation
+
+    # Replace for only ClickhouseAdapter
+    def reverse_order!
+      orders = order_values.uniq
+      orders.reject!(&:blank?)
+      if self.connection.is_a?(ConnectionAdapters::ClickhouseAdapter) && orders.empty? && !primary_key
+        self.order_values = %w(date created_at).select {|c| column_names.include?(c) }.map{|c| arel_attribute(c).desc }
+      else
+        self.order_values = reverse_sql_order(orders)
+      end
+      self
+    end
+  end
+
+  module TypeCaster
+    class Map
+      def is_view
+        types.is_view
       end
     end
   end
@@ -39,6 +63,11 @@ module ActiveRecord
       def is_view=(value)
         @is_view = value
       end
+
+      def arel_table # :nodoc:
+        @arel_table ||= ClickhouseActiverecord::Arel::Table.new(table_name, type_caster: type_caster)
+      end
+
     end
    end
 
@@ -64,17 +93,34 @@ module ActiveRecord
       include Clickhouse::SchemaStatements
 
       # Initializes and connects a Clickhouse adapter.
-      def initialize(connection, logger, connection_parameters, config, debug = false)
-        super(connection, logger)
+      def initialize(logger, connection_parameters, config, full_config)
+        super(nil, logger)
         @connection_parameters = connection_parameters
         @config = config
-        @debug = debug
+        @debug = full_config[:debug] || false
+        @full_config = full_config
 
         @prepared_statements = false
+        if ActiveRecord::version == Gem::Version.new('6.0.0')
+          @prepared_statement_status = Concurrent::ThreadLocalVar.new(false)
+        end
 
         connect
       end
 
+      # Support SchemaMigration from v5.2.2 to v6+
+      def schema_migration # :nodoc:
+        if ActiveRecord::version >= Gem::Version.new('6')
+          super
+        else
+          ActiveRecord::SchemaMigration
+        end
+      end
+
+      def migrations_paths
+        @full_config[:migrations_paths] || 'db/migrate_clickhouse'
+      end
+
       def arel_visitor # :nodoc:
         ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
       end
@@ -130,6 +176,14 @@ module ActiveRecord
         value.to_s(:db)
       end
 
+      def column_name_for_operation(operation, node) # :nodoc:
+        if ActiveRecord::version >= Gem::Version.new('6')
+          visitor.compile(node)
+        else
+          column_name_from_arel_node(node)
+        end
+      end
+
       # Executes insert +sql+ statement in the context of this connection using
       # +binds+ as the bind substitutes. +name+ is logged along with
       # the executed +sql+ statement.
@@ -177,8 +231,7 @@ module ActiveRecord
       private
 
       def connect
-        # for ssl port need use ssl
-        @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[1] == 443)
+        @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
       end
     end
   end

data/lib/clickhouse-activerecord/arel/table.rb

@@ -0,0 +1,9 @@
+module ClickhouseActiverecord
+  module Arel
+    class Table < ::Arel::Table
+      def is_view
+        type_caster.is_view
+      end
+    end
+  end
+end

data/lib/clickhouse-activerecord/arel/visitors/to_sql.rb

@@ -6,13 +6,12 @@ module ClickhouseActiverecord
       class ToSql < ::Arel::Visitors::ToSql
 
         def aggregate(name, o, collector)
-          # todo how get model class from request? This method works with only rails 4.2.
           # replacing function name for materialized view
-          # if o.expressions.first && o.expressions.first != '*' && !o.expressions.first.is_a?(String) && o.expressions.first.relation && o.expressions.first.relation.engine && o.expressions.first.relation.engine.is_view
-          #   super("#{name.downcase}Merge", o, collector)
-          # else
+          if o.expressions.first && o.expressions.first != '*' && !o.expressions.first.is_a?(String) && o.expressions.first.relation&.is_view
+            super("#{name.downcase}Merge", o, collector)
+          else
             super
-          # end
+          end
         end
 
       end

data/lib/clickhouse-activerecord/tasks.rb

@@ -31,13 +31,27 @@ module ClickhouseActiverecord
       create
     end
 
+    def structure_dump(*args)
+      tables = connection.execute("SHOW TABLES FROM #{@configuration['database']}")['data'].flatten
+
+      File.open(args.first, 'w:utf-8') do |file|
+        tables.each do |table|
+          next if table.match(/\.inner/)
+          file.puts connection.execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first).gsub("#{@configuration['database']}.", '') + ";\n\n"
+        end
+      end
+    end
+
+    def structure_load(*args)
+      File.read(args.first).split(";\n\n").each { |sql| connection.execute(sql) }
+    end
+
     def migrate
       check_target_version
 
       verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] != "false" : true
       scope = ENV["SCOPE"]
       verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, verbose
-      binding.pry
       connection.migration_context.migrate(target_version) do |migration|
         scope.blank? || scope == migration.scope
       end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.3.4'
+  VERSION = '0.3.12'
 end

data/lib/generators/clickhouse_migration_generator.rb

@@ -6,6 +6,20 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato
   def create_migration_file
     set_local_assigns!
     validate_file_name!
-    migration_template @migration_template, "db/migrate_clickhouse/#{file_name}.rb"
+    migration_template @migration_template, File.join(db_migrate_path, "#{file_name}.rb")
+  end
+
+  private
+
+  def db_migrate_path
+    if defined?(Rails.application) && Rails.application
+      configured_migrate_path || default_migrate_path
+    else
+      default_migrate_path
+    end
+  end
+
+  def default_migrate_path
+    "db/migrate_clickhouse"
   end
 end

data/lib/tasks/clickhouse.rake

@@ -27,6 +27,18 @@ namespace :clickhouse do
 
   end
 
+  namespace :structure do
+    desc 'Load database structure'
+    task load: [:load_config, 'db:check_protected_environments'] do
+      ClickhouseActiverecord::Tasks.new(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"]).structure_load("#{Rails.root}/db/clickhouse_structure.sql")
+    end
+
+    desc 'Dump database structure'
+    task dump: [:load_config, 'db:check_protected_environments'] do
+      ClickhouseActiverecord::Tasks.new(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"]).structure_dump("#{Rails.root}/db/clickhouse_structure.sql")
+    end
+  end
+
   desc 'Creates the database from DATABASE_URL or config/database.yml'
   task create: [:load_config] do
     ActiveRecord::Tasks::DatabaseTasks.create(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"])

metadata

@@ -1,22 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.12
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-28 00:00:00.000000000 Z
+date: 2020-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.13.4
@@ -24,9 +21,6 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.13.4
@@ -34,14 +28,14 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.2'
 - !ruby/object:Gem::Dependency
@@ -126,6 +120,7 @@ files:
 - lib/active_record/connection_adapters/clickhouse/schema_statements.rb
 - lib/active_record/connection_adapters/clickhouse_adapter.rb
 - lib/clickhouse-activerecord.rb
+- lib/clickhouse-activerecord/arel/table.rb
 - lib/clickhouse-activerecord/arel/visitors/to_sql.rb
 - lib/clickhouse-activerecord/railtie.rb
 - lib/clickhouse-activerecord/schema_dumper.rb