clickhouse-activerecord 0.6.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6533de4ab9b2415a4df22e72e7ef551d4fb58c5bdc6713ec355b85dbad1174fd
-  data.tar.gz: 1c0a898f494c6dc7511897617ea86c2f12aa42461cf693e28e78124bde3638c6
+  metadata.gz: 823fdbf17f27c65446c01b1599a3c61d9fa9b310d7ce153e9c64a912719e552e
+  data.tar.gz: 5b6dd44cdc44b2d1350e0f1c5937cf08051a5ff7a4f73cf56a5dc7567b4be800
 SHA512:
-  metadata.gz: f6c5c2eba18dce09211cc6b8e5448aaaf2c40ca3eb5cc411aa76874ad6efc00ae257880e5997256808d02771aec7947559edfca941187c219b6724f09b9488f0
-  data.tar.gz: 7b7e5f520c4189b4b5a1587789d0ea2ef8b74e86c077611ac18515c45105306c6e17cfd824623ab574420beeb45d205a9f301e2db9f3befd3c3b92c4d303b978
+  metadata.gz: 5a8d48c078f5bf79a50fddf9b29ade13694019e8666c8c071e5b1ee7ab5b16c7e92116b794a82ce3fa6f927f0bd1186e7e87a65ed5f4eda0f1e6ed113f590de4
+  data.tar.gz: 2be66ae7ac2cb71b20486685f00552890559f2944777d8557aabb60d2318baa868c3c4bc0cf4be82a97eeee920de0049851771253129022e8fbbb0758190f917

data/README.md

@@ -1,6 +1,6 @@
 # Clickhouse::Activerecord
 
-A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
+A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 7.1.
 Support ClickHouse version from 22.0 LTS.
 
 ## Installation
@@ -50,41 +50,31 @@ class ActionView < ActiveRecord::Base
 end
 ```
 
-## Usage in Rails 5
+## Usage in Rails
 
 Add your `database.yml` connection information with postfix `_clickhouse` for you environment:
 
 ```yml
-development_clickhouse:
+development:
   adapter: clickhouse
   database: database
 ```
 
-Add to your model:
+Your model example:
 
 ```ruby
 class Action < ActiveRecord::Base
-  establish_connection "#{Rails.env}_clickhouse".to_sym
 end
 ```
 
 For materialized view model add:
 ```ruby
 class ActionView < ActiveRecord::Base
-  establish_connection "#{Rails.env}_clickhouse".to_sym
   self.is_view = true
 end
 ```
 
-Or global connection:
-
-```yml
-development:
-  adapter: clickhouse
-  database: database
-```
-
-## Usage in Rails 6 with second database
+## Usage in Rails with second database
 
 Add your `database.yml` connection information for you environment:
 
@@ -102,31 +92,31 @@ Connection [Multiple Databases with Active Record](https://guides.rubyonrails.or
 
 ```ruby
 class Action < ActiveRecord::Base
-  connects_to database: { writing: :clickhouse, reading: :clickhouse }
+  establish_connection :clickhouse
 end
 ```
 
 ### Rake tasks
 
-**Note!** For Rails 6 you can use default rake tasks if you configure `migrations_paths` in your `database.yml`, for example: `rake db:migrate`
-
 Create / drop / purge / reset database:
  
-    $ rake clickhouse:create
-    $ rake clickhouse:drop
-    $ rake clickhouse:purge
-    $ rake clickhouse:reset
+    $ rake db:create
+    $ rake db:drop
+    $ rake db:purge
+    $ rake db:reset
 
-Prepare system tables for rails:
+Or with multiple databases:
 
-    $ rake clickhouse:prepare_schema_migration_table
-    $ rake clickhouse:prepare_internal_metadata_table
+    $ rake db:create:clickhouse
+    $ rake db:drop:clickhouse
+    $ rake db:purge:clickhouse
+    $ rake db:reset:clickhouse
     
 Migration:
 
     $ rails g clickhouse_migration MIGRATION_NAME COLUMNS
-    $ rake clickhouse:migrate
-    $ rake clickhouse:rollback
+    $ rake db:migrate
+    $ rake db:rollback
 
 ### Dump / Load for multiple using databases
 
@@ -195,20 +185,20 @@ User.joins(:actions).using(:group_id)
 Integer types are unsigned by default. Specify signed values with `:unsigned =>
 false`. The default integer is `UInt32`
 
-| Type (bit size)    | Range | :limit (byte size) |
-| :---        |    :----:   |          ---: |
-| Int8 | -128 to 127 | 1 | 
-| Int16 | -32768 to 32767 | 2 |
-| Int32 | -2147483648 to 2,147,483,647 | 3,4 |
-| Int64 | -9223372036854775808 to 9223372036854775807] |  5,6,7,8 |
-| Int128 | ... | 9 - 15 |
-| Int256 | ... | 16+ |
-| UInt8 | 0 to 255 | 1 |
-| UInt16 | 0 to 65,535 | 2 |
-| UInt32 | 0 to 4,294,967,295 | 3,4 |
-| UInt64 | 0 to 18446744073709551615 | 5,6,7,8 |
-| UInt256 | 0 to ... | 8+ |
-| Array | ... | ... |
+| Type (bit size) |                    Range                     | :limit (byte size) |
+|:----------------|:--------------------------------------------:|-------------------:|
+| Int8            |                 -128 to 127                  |                  1 | 
+| Int16           |               -32768 to 32767                |                  2 |
+| Int32           |         -2147483648 to 2,147,483,647         |                3,4 |
+| Int64           | -9223372036854775808 to 9223372036854775807] |            5,6,7,8 |
+| Int128          |                     ...                      |             9 - 15 |
+| Int256          |                     ...                      |                16+ |
+| UInt8           |                   0 to 255                   |                  1 |
+| UInt16          |                 0 to 65,535                  |                  2 |
+| UInt32          |              0 to 4,294,967,295              |                3,4 |
+| UInt64          |          0 to 18446744073709551615           |            5,6,7,8 |
+| UInt256         |                   0 to ...                   |                 8+ |
+| Array           |                     ...                      |                ... |
 
 Example:
 

data/clickhouse-activerecord.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'bundler', '>= 1.13.4'
-  spec.add_runtime_dependency 'activerecord', '~> 7.0.0'
+  spec.add_runtime_dependency 'activerecord', '>= 7.1'
 
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.4'

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -10,13 +10,13 @@ module ActiveRecord
           do_execute(sql, name, settings: settings)
         end
 
-        def exec_insert(sql, name, _binds, _pk = nil, _sequence_name = nil)
+        def exec_insert(sql, name, _binds, _pk = nil, _sequence_name = nil, returning: nil)
           new_sql = sql.dup.sub(/ (DEFAULT )?VALUES/, " VALUES")
           do_execute(new_sql, name, format: nil)
           true
         end
 
-        def exec_query(sql, name = nil, binds = [], prepare: false)
+        def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false)
           result = do_execute(sql, name)
           ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'], result['meta'].map { |m| [m['name'], type_map.lookup(m['type'])] }.to_h)
         rescue ActiveRecord::ActiveRecordError => e
@@ -137,7 +137,7 @@ module ActiveRecord
           Clickhouse::TableDefinition.new(self, table_name, **options)
         end
 
-        def new_column_from_field(table_name, field)
+        def new_column_from_field(table_name, field, _definitions)
           sql_type = field[1]
           type_metadata = fetch_type_metadata(sql_type)
           default = field[3]

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -136,7 +136,11 @@ module ActiveRecord
 
       # Support SchemaMigration from v5.2.2 to v6+
       def schema_migration # :nodoc:
-        ClickhouseActiverecord::SchemaMigration
+        ClickhouseActiverecord::SchemaMigration.new(self)
+      end
+
+      def internal_metadata # :nodoc:
+        ClickhouseActiverecord::InternalMetadata.new(self)
       end
 
       def migrations_paths
@@ -144,7 +148,7 @@ module ActiveRecord
       end
 
       def migration_context # :nodoc:
-        ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration)
+        ClickhouseActiverecord::MigrationContext.new(migrations_paths, schema_migration, internal_metadata)
       end
 
       def arel_visitor # :nodoc:

data/lib/arel/visitors/clickhouse.rb

@@ -15,7 +15,7 @@ module Arel
 
       def visit_Arel_Table o, collector
         collector = super
-        collector << ' FINAL ' if o.final
+        collector << ' FINAL' if o.final
         collector
       end
 

data/lib/clickhouse-activerecord/migration.rb

@@ -3,107 +3,96 @@ require 'active_record/migration'
 module ClickhouseActiverecord
 
   class SchemaMigration < ::ActiveRecord::SchemaMigration
-    class << self
+    def create_table
+      return if table_exists?
 
-      def create_table
-        return if table_exists?
+      version_options = connection.internal_string_options_for_primary_key
+      table_options = {
+        id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true
+      }
+      full_config = connection.instance_variable_get(:@full_config) || {}
 
-        version_options = connection.internal_string_options_for_primary_key
-        table_options = {
-          id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true
-        }
-        full_config = connection.instance_variable_get(:@full_config) || {}
+      if full_config[:distributed_service_tables]
+        table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
 
-        if full_config[:distributed_service_tables]
-          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
-
-          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
-        end
-
-        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
-          t.string :version, **version_options
-          t.column :active, 'Int8', null: false, default: '1'
-          t.datetime :ver, null: false, default: -> { 'now()' }
-        end
+        distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
       end
 
-      def all_versions
-        final.where(active: 1).order(:version).pluck(:version)
+      connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
+        t.string :version, **version_options
+        t.column :active, 'Int8', null: false, default: '1'
+        t.datetime :ver, null: false, default: -> { 'now()' }
       end
     end
-  end
 
-  class InternalMetadata < ::ActiveRecord::InternalMetadata
-    class << self
+    def versions
+      table = arel_table.dup
+      table.final = true
+      sm = Arel::SelectManager.new(table)
+      sm.project(arel_table[primary_key])
+      sm.order(arel_table[primary_key].asc)
+      sm.where([arel_table['active'].eq(1)])
 
-      def []=(key, value)
-        row = final.find_by(key: key)
-        if row.nil? || row.value != value
-          create!(key: key, value: value)
-        end
-      end
+      connection.select_values(sm, "#{self.class} Load")
+    end
 
-      def [](key)
-        final.where(key: key).pluck(:value).first
-      end
+    def delete_version(version)
+      im = Arel::InsertManager.new(arel_table)
+      im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
+      connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
+    end
+  end
 
-      def create_table
-        return if table_exists?
+  class InternalMetadata < ::ActiveRecord::InternalMetadata
 
-        key_options = connection.internal_string_options_for_primary_key
-        table_options = {
-          id: false,
-          options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '',
-          if_not_exists: true
-        }
-        full_config = connection.instance_variable_get(:@full_config) || {}
+    def create_table
+      return if table_exists? || !enabled?
 
-        if full_config[:distributed_service_tables]
-          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
+      key_options = connection.internal_string_options_for_primary_key
+      table_options = {
+        id: false,
+        options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '',
+        if_not_exists: true
+      }
+      full_config = connection.instance_variable_get(:@full_config) || {}
 
-          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
-        end
+      if full_config[:distributed_service_tables]
+        table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
 
-        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
-          t.string :key, **key_options
-          t.string :value
-          t.timestamps
-        end
+        distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
       end
-    end
-  end
 
-  class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
-    attr_reader :migrations_paths, :schema_migration
-
-    def initialize(migrations_paths, schema_migration)
-      @migrations_paths = migrations_paths
-      @schema_migration = schema_migration
+      connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
+        t.string :key, **key_options
+        t.string :value
+        t.timestamps
+      end
     end
 
-    def up(target_version = nil)
-      selected_migrations = if block_given?
-        migrations.select { |m| yield m }
-      else
-        migrations
-      end
+    private
 
-      ClickhouseActiverecord::Migrator.new(:up, selected_migrations, schema_migration, target_version).migrate
+    def update_entry(key, new_value)
+      create_entry(key, new_value)
     end
 
-    def down(target_version = nil)
-      selected_migrations = if block_given?
-        migrations.select { |m| yield m }
-      else
-        migrations
-      end
+    def select_entry(key)
+      table = arel_table.dup
+      table.final = true
+      sm = Arel::SelectManager.new(table)
+      sm.project(Arel::Nodes::SqlLiteral.new("*"))
+      sm.where(table[primary_key].eq(Arel::Nodes::BindParam.new(key)))
+      sm.order(table[primary_key].asc)
+      sm.limit = 1
 
-      ClickhouseActiverecord::Migrator.new(:down, selected_migrations, schema_migration, target_version).migrate
+      connection.select_all(sm, "#{self.class} Load").first
     end
+  end
+
+  class MigrationContext < ::ActiveRecord::MigrationContext #:nodoc:
 
     def get_all_versions
       if schema_migration.table_exists?
-        schema_migration.all_versions.map(&:to_i)
+        schema_migration.versions.map(&:to_i)
       else
         []
       end
@@ -111,36 +100,4 @@ module ClickhouseActiverecord
 
   end
 
-  class Migrator < ::ActiveRecord::Migrator
-
-    def initialize(direction, migrations, schema_migration, target_version = nil)
-      @direction         = direction
-      @target_version    = target_version
-      @migrated_versions = nil
-      @migrations        = migrations
-      @schema_migration  = schema_migration
-
-      validate(@migrations)
-
-      @schema_migration.create_table
-      ClickhouseActiverecord::InternalMetadata.create_table
-    end
-
-    def record_version_state_after_migrating(version)
-      if down?
-        migrated.delete(version)
-        @schema_migration.create!(version: version.to_s, active: 0)
-      else
-        super
-      end
-    end
-
-    private
-
-    def record_environment
-      return if down?
-      ClickhouseActiverecord::InternalMetadata[:environment] = ActiveRecord::Base.connection.migration_context.current_environment
-    end
-
-  end
 end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.6.2'
+  VERSION = '1.0.0'
 end

data/lib/core_extensions/active_record/relation.rb

@@ -11,21 +11,65 @@ module CoreExtensions
         self
       end
 
+      # Define settings in the SETTINGS clause of the SELECT query. The setting value is applied only to that query and is reset to the default or previous value after the query is executed.
+      # For example:
+      #
+      #   users = User.settings(optimize_read_in_order: 1, cast_keep_nullable: 1).where(name: 'John')
+      #   # SELECT users.* FROM users WHERE users.name = 'John' SETTINGS optimize_read_in_order = 1, cast_keep_nullable = 1
+      #
+      # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
       # @param [Hash] opts
       def settings(**opts)
+        spawn.settings!(**opts)
+      end
+
+      # @param [Hash] opts
+      def settings!(**opts)
+        assert_mutability!
         check_command('SETTINGS')
         @values[:settings] = (@values[:settings] || {}).merge opts
         self
       end
 
+      # When FINAL is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine.
+      # For example:
+      #
+      #   users = User.final.all
+      #   # SELECT users.* FROM users FINAL
+      #
+      # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
       # @param [Boolean] final
       def final(final = true)
+        spawn.final!(final)
+      end
+
+      # @param [Boolean] final
+      def final!(final = true)
+        assert_mutability!
         check_command('FINAL')
         @table = @table.dup
         @table.final = final
         self
       end
 
+      # The USING clause specifies one or more columns to join, which establishes the equality of these columns. For example:
+      #
+      #   users = User.joins(:joins).using(:event_name, :date)
+      #   # SELECT users.* FROM users INNER JOIN joins USING event_name,date
+      #
+      # An <tt>ActiveRecord::ActiveRecordError</tt> will be raised if database not ClickHouse.
+      # @param [Array] opts
+      def using(*opts)
+        spawn.using!(*opts)
+      end
+
+      # @param [Array] opts
+      def using!(*opts)
+        assert_mutability!
+        @values[:using] = opts
+        self
+      end
+
       private
 
       def check_command(cmd)
@@ -36,6 +80,7 @@ module CoreExtensions
         arel = super
 
         arel.settings(@values[:settings]) if @values[:settings].present?
+        arel.using(@values[:using]) if @values[:using].present?
 
         arel
       end

data/lib/tasks/clickhouse.rake

@@ -2,17 +2,13 @@
 
 namespace :clickhouse do
   task prepare_schema_migration_table: :environment do
-    ClickhouseActiverecord::SchemaMigration.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
+    connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection
+    connection.schema_migration.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
   end
 
   task prepare_internal_metadata_table: :environment do
-    ClickhouseActiverecord::InternalMetadata.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
-  end
-
-  task load_config: :environment do
-    ENV['SCHEMA'] = 'db/clickhouse_schema.rb'
-    ActiveRecord::Migrator.migrations_paths = %w[db/migrate_clickhouse]
-    ActiveRecord::Base.establish_connection(:clickhouse)
+    connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection
+    connection.internal_metadata.create_table unless ENV['simple'] || ARGV.any? { |a| a.include?('--simple') }
   end
 
   namespace :schema do
@@ -37,52 +33,55 @@ namespace :clickhouse do
 
   namespace :structure do
     desc 'Load database structure'
-    task load: [:load_config, 'db:check_protected_environments'] do
+    task load: ['db:check_protected_environments'] do
       config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
       ClickhouseActiverecord::Tasks.new(config).structure_load(Rails.root.join('db/clickhouse_structure.sql'))
     end
 
     desc 'Dump database structure'
-    task dump: [:load_config, 'db:check_protected_environments'] do
+    task dump: ['db:check_protected_environments'] do
       config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
       ClickhouseActiverecord::Tasks.new(config).structure_dump(Rails.root.join('db/clickhouse_structure.sql'))
     end
   end
 
   desc 'Creates the database from DATABASE_URL or config/database.yml'
-  task create: [:load_config] do
+  task create: [] do
     config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
     ActiveRecord::Tasks::DatabaseTasks.create(config)
   end
 
   desc 'Drops the database from DATABASE_URL or config/database.yml'
-  task drop: [:load_config, 'db:check_protected_environments'] do
+  task drop: ['db:check_protected_environments'] do
     config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
     ActiveRecord::Tasks::DatabaseTasks.drop(config)
   end
 
   desc 'Empty the database from DATABASE_URL or config/database.yml'
-  task purge: [:load_config, 'db:check_protected_environments'] do
+  task purge: ['db:check_protected_environments'] do
     config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
     ActiveRecord::Tasks::DatabaseTasks.purge(config)
   end
 
   # desc 'Resets your database using your migrations for the current environment'
-  task reset: :load_config do
+  task :reset  do
     Rake::Task['clickhouse:purge'].execute
     Rake::Task['clickhouse:migrate'].execute
   end
 
   desc 'Migrate the clickhouse database'
-  task migrate: %i[load_config prepare_schema_migration_table prepare_internal_metadata_table] do
-    Rake::Task['db:migrate'].execute
+  task migrate: %i[prepare_schema_migration_table prepare_internal_metadata_table] do
+    Rake::Task['db:migrate:clickhouse'].execute
     if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb"
       Rake::Task['clickhouse:schema:dump'].execute(simple: true)
     end
   end
 
   desc 'Rollback the clickhouse database'
-  task rollback: %i[load_config prepare_schema_migration_table prepare_internal_metadata_table] do
-    Rake::Task['db:rollback'].execute
+  task rollback: %i[prepare_schema_migration_table prepare_internal_metadata_table] do
+    Rake::Task['db:rollback:clickhouse'].execute
+    if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb"
+      Rake::Task['clickhouse:schema:dump'].execute(simple: true)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Sergey Odintsov
@@ -28,16 +28,16 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement