clickhouse-activerecord 0.3.5 → 0.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
 - data/CHANGELOG.md +13 -4
 - data/README.md +68 -10
 - data/clickhouse-activerecord.gemspec +2 -2
 - data/lib/active_record/connection_adapters/clickhouse/oid/date_time.rb +1 -0
 - data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +25 -43
 - data/lib/active_record/connection_adapters/clickhouse_adapter.rb +62 -11
 - data/lib/clickhouse-activerecord/schema_dumper.rb +1 -1
 - data/lib/clickhouse-activerecord/tasks.rb +15 -1
 - data/lib/clickhouse-activerecord/version.rb +1 -1
 - data/lib/generators/clickhouse_migration_generator.rb +15 -1
 - data/lib/tasks/clickhouse.rake +77 -1
 - metadata +5 -12
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
     | 
    
         
            -
             
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 2 
     | 
    
         
            +
            SHA256:
         
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: 67ea4a63bb140adb5f6de102c38c097ea18f61499a882574952a9982b8d340cf
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: a1d2a759dae9062a20cc0a53bbd1cb64142062ce9ef84047f7e953b7d957e9b2
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 243b4d089d6349d8c12c529fbb80078ffbaa03f0033c3c69be77751846b031398895a1efdd08460699636277695a3e03eb228a398b5887a9426f4f3b63496c31
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: f790bca5af79492390de98ba2dce363e16207d987093c2e94e9f9ab55b9f769a41bae1c4a12d0b9d3757ee91f92604c1df4db17479dbbc85ced0f48294b297f3
         
     | 
    
        data/CHANGELOG.md
    CHANGED
    
    | 
         @@ -1,17 +1,26 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            ### Version 0.3.10 (Dec 20, 2019)
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
             * Support structure dump/load [@StoneGod](https://github.com/StoneGod)
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            ### Version 0.3.6 (Sep 2, 2019)
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
             * Support Rails 6.0
         
     | 
| 
      
 8 
     | 
    
         
            +
             * Fix relation `last` method
         
     | 
| 
      
 9 
     | 
    
         
            +
             
     | 
| 
       1 
10 
     | 
    
         
             
            ### Version 0.3.4 (Jun 28, 2019)
         
     | 
| 
       2 
11 
     | 
    
         | 
| 
       3 
     | 
    
         
            -
             * Fix DateTime sql format without microseconds for Rails 5.2 
     | 
| 
      
 12 
     | 
    
         
            +
             * Fix DateTime sql format without microseconds for Rails 5.2
         
     | 
| 
       4 
13 
     | 
    
         
             
             * Support ssl connection
         
     | 
| 
       5 
14 
     | 
    
         
             
             * Migration support
         
     | 
| 
       6 
15 
     | 
    
         
             
             * Rake tasks for create / drop database
         
     | 
| 
       7 
     | 
    
         
            -
             
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
       8 
17 
     | 
    
         
             
            ### Version 0.3.0 (Nov 27, 2018)
         
     | 
| 
       9 
18 
     | 
    
         | 
| 
       10 
19 
     | 
    
         
             
             * Support materialized view
         
     | 
| 
       11 
20 
     | 
    
         
             
             * Aggregated functions for view
         
     | 
| 
       12 
21 
     | 
    
         
             
             * Schema dumper with SQL create table
         
     | 
| 
       13 
22 
     | 
    
         
             
             * Added migrations support [@Bugagazavr](https://github.com/Bugagazavr)
         
     | 
| 
       14 
     | 
    
         
            -
             
     | 
| 
      
 23 
     | 
    
         
            +
             
     | 
| 
       15 
24 
     | 
    
         
             
            ### Version 0.2.0 (Oct 3, 2017)
         
     | 
| 
       16 
25 
     | 
    
         | 
| 
       17 
26 
     | 
    
         
             
             * Support Rails 5.0
         
     | 
| 
         @@ -19,7 +28,7 @@ 
     | 
|
| 
       19 
28 
     | 
    
         
             
            ### Version 0.1.2 (Sep 27, 2017)
         
     | 
| 
       20 
29 
     | 
    
         | 
| 
       21 
30 
     | 
    
         
             
             * Fix Big Int type
         
     | 
| 
       22 
     | 
    
         
            -
             
     | 
| 
      
 31 
     | 
    
         
            +
             
     | 
| 
       23 
32 
     | 
    
         
             
            ### Version 0.1.0 (Aug 31, 2017)
         
     | 
| 
       24 
33 
     | 
    
         | 
| 
       25 
34 
     | 
    
         
             
             * Initial release
         
     | 
    
        data/README.md
    CHANGED
    
    | 
         @@ -1,6 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            # Clickhouse::Activerecord
         
     | 
| 
       2 
2 
     | 
    
         | 
| 
       3 
3 
     | 
    
         
             
            A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
         
     | 
| 
      
 4 
     | 
    
         
            +
            Tested on ClickHouse version 18.14.
         
     | 
| 
       4 
5 
     | 
    
         | 
| 
       5 
6 
     | 
    
         
             
            ## Installation
         
     | 
| 
       6 
7 
     | 
    
         | 
| 
         @@ -17,8 +18,24 @@ And then execute: 
     | 
|
| 
       17 
18 
     | 
    
         
             
            Or install it yourself as:
         
     | 
| 
       18 
19 
     | 
    
         | 
| 
       19 
20 
     | 
    
         
             
                $ gem install clickhouse-activerecord
         
     | 
| 
      
 21 
     | 
    
         
            +
                
         
     | 
| 
      
 22 
     | 
    
         
            +
            ## Available database connection parameters
         
     | 
| 
      
 23 
     | 
    
         
            +
            ```yml
         
     | 
| 
      
 24 
     | 
    
         
            +
            default: &default
         
     | 
| 
      
 25 
     | 
    
         
            +
              adapter: clickhouse
         
     | 
| 
      
 26 
     | 
    
         
            +
              database: database
         
     | 
| 
      
 27 
     | 
    
         
            +
              host: localhost
         
     | 
| 
      
 28 
     | 
    
         
            +
              port: 8123
         
     | 
| 
      
 29 
     | 
    
         
            +
              username: username
         
     | 
| 
      
 30 
     | 
    
         
            +
              password: password
         
     | 
| 
      
 31 
     | 
    
         
            +
              ssl: true # optional for using ssl connection
         
     | 
| 
      
 32 
     | 
    
         
            +
              debug: true # use for showing in to log technical information
         
     | 
| 
      
 33 
     | 
    
         
            +
              migrations_paths: db/clickhouse # optional, default: db/migrate_clickhouse
         
     | 
| 
      
 34 
     | 
    
         
            +
              cluster: 'cluster_name' # optional for creating tables in cluster 
         
     | 
| 
      
 35 
     | 
    
         
            +
              replica: '{shard}' # optional for creating system tables for shards
         
     | 
| 
      
 36 
     | 
    
         
            +
            ```
         
     | 
| 
       20 
37 
     | 
    
         | 
| 
       21 
     | 
    
         
            -
            ## Usage
         
     | 
| 
      
 38 
     | 
    
         
            +
            ## Usage in Rails 5
         
     | 
| 
       22 
39 
     | 
    
         | 
| 
       23 
40 
     | 
    
         
             
            Add your `database.yml` connection information with postfix `_clickhouse` for you environment:
         
     | 
| 
       24 
41 
     | 
    
         | 
| 
         @@ -26,10 +43,6 @@ Add your `database.yml` connection information with postfix `_clickhouse` for yo 
     | 
|
| 
       26 
43 
     | 
    
         
             
            development_clickhouse:
         
     | 
| 
       27 
44 
     | 
    
         
             
              adapter: clickhouse
         
     | 
| 
       28 
45 
     | 
    
         
             
              database: database
         
     | 
| 
       29 
     | 
    
         
            -
              host: localhost
         
     | 
| 
       30 
     | 
    
         
            -
              username: username
         
     | 
| 
       31 
     | 
    
         
            -
              password: password
         
     | 
| 
       32 
     | 
    
         
            -
              debug: true # use for showing in to log technical information
         
     | 
| 
       33 
46 
     | 
    
         
             
            ```
         
     | 
| 
       34 
47 
     | 
    
         | 
| 
       35 
48 
     | 
    
         
             
            Add to your model:
         
     | 
| 
         @@ -54,19 +67,45 @@ Or global connection: 
     | 
|
| 
       54 
67 
     | 
    
         
             
            development:
         
     | 
| 
       55 
68 
     | 
    
         
             
              adapter: clickhouse
         
     | 
| 
       56 
69 
     | 
    
         
             
              database: database
         
     | 
| 
       57 
     | 
    
         
            -
             
     | 
| 
       58 
     | 
    
         
            -
             
     | 
| 
       59 
     | 
    
         
            -
             
     | 
| 
      
 70 
     | 
    
         
            +
            ```
         
     | 
| 
      
 71 
     | 
    
         
            +
             
     | 
| 
      
 72 
     | 
    
         
            +
            ## Usage in Rails 6 with second database
         
     | 
| 
      
 73 
     | 
    
         
            +
             
     | 
| 
      
 74 
     | 
    
         
            +
            Add your `database.yml` connection information for you environment:
         
     | 
| 
      
 75 
     | 
    
         
            +
             
     | 
| 
      
 76 
     | 
    
         
            +
            ```yml
         
     | 
| 
      
 77 
     | 
    
         
            +
            development:
         
     | 
| 
      
 78 
     | 
    
         
            +
              primary:
         
     | 
| 
      
 79 
     | 
    
         
            +
                ...
         
     | 
| 
      
 80 
     | 
    
         
            +
                
         
     | 
| 
      
 81 
     | 
    
         
            +
              clickhouse:
         
     | 
| 
      
 82 
     | 
    
         
            +
                adapter: clickhouse
         
     | 
| 
      
 83 
     | 
    
         
            +
                database: database
         
     | 
| 
      
 84 
     | 
    
         
            +
            ```
         
     | 
| 
      
 85 
     | 
    
         
            +
             
     | 
| 
      
 86 
     | 
    
         
            +
            Connection [Multiple Databases with Active Record](https://guides.rubyonrails.org/active_record_multiple_databases.html) or short example:
         
     | 
| 
      
 87 
     | 
    
         
            +
             
     | 
| 
      
 88 
     | 
    
         
            +
            ```ruby
         
     | 
| 
      
 89 
     | 
    
         
            +
            class Action < ActiveRecord::Base
         
     | 
| 
      
 90 
     | 
    
         
            +
              connects_to database: { writing: :clickhouse, reading: :clickhouse }
         
     | 
| 
      
 91 
     | 
    
         
            +
            end
         
     | 
| 
       60 
92 
     | 
    
         
             
            ```
         
     | 
| 
       61 
93 
     | 
    
         | 
| 
       62 
94 
     | 
    
         
             
            ### Rake tasks
         
     | 
| 
       63 
95 
     | 
    
         | 
| 
      
 96 
     | 
    
         
            +
            **Note!** For Rails 6 you can use default rake tasks if you configure `migrations_paths` in your `database.yml`, for example: `rake db:migrate`
         
     | 
| 
      
 97 
     | 
    
         
            +
             
     | 
| 
       64 
98 
     | 
    
         
             
            Create / drop / purge / reset database:
         
     | 
| 
       65 
99 
     | 
    
         | 
| 
       66 
100 
     | 
    
         
             
                $ rake clickhouse:create
         
     | 
| 
       67 
101 
     | 
    
         
             
                $ rake clickhouse:drop
         
     | 
| 
       68 
102 
     | 
    
         
             
                $ rake clickhouse:purge
         
     | 
| 
       69 
103 
     | 
    
         
             
                $ rake clickhouse:reset
         
     | 
| 
      
 104 
     | 
    
         
            +
             
     | 
| 
      
 105 
     | 
    
         
            +
            Prepare system tables for rails:
         
     | 
| 
      
 106 
     | 
    
         
            +
             
     | 
| 
      
 107 
     | 
    
         
            +
                $ rake clickhouse:prepare_schema_migration_table
         
     | 
| 
      
 108 
     | 
    
         
            +
                $ rake clickhouse:prepare_internal_metadata_table
         
     | 
| 
       70 
109 
     | 
    
         | 
| 
       71 
110 
     | 
    
         
             
            Migration:
         
     | 
| 
       72 
111 
     | 
    
         | 
| 
         @@ -75,6 +114,10 @@ Migration: 
     | 
|
| 
       75 
114 
     | 
    
         | 
| 
       76 
115 
     | 
    
         
             
            Rollback migration not supported!
         
     | 
| 
       77 
116 
     | 
    
         | 
| 
      
 117 
     | 
    
         
            +
            ### Dump / Load for multiple using databases
         
     | 
| 
      
 118 
     | 
    
         
            +
             
     | 
| 
      
 119 
     | 
    
         
            +
            If you using multiple databases, for example: PostgreSQL, Clickhouse.
         
     | 
| 
      
 120 
     | 
    
         
            +
             
     | 
| 
       78 
121 
     | 
    
         
             
            Schema dump to `db/clickhouse_schema.rb` file:
         
     | 
| 
       79 
122 
     | 
    
         | 
| 
       80 
123 
     | 
    
         
             
                $ rake clickhouse:schema:dump
         
     | 
| 
         @@ -82,9 +125,24 @@ Schema dump to `db/clickhouse_schema.rb` file: 
     | 
|
| 
       82 
125 
     | 
    
         
             
            Schema load from `db/clickhouse_schema.rb` file:
         
     | 
| 
       83 
126 
     | 
    
         | 
| 
       84 
127 
     | 
    
         
             
                $ rake clickhouse:schema:load
         
     | 
| 
       85 
     | 
    
         
            -
             
     | 
| 
      
 128 
     | 
    
         
            +
             
     | 
| 
       86 
129 
     | 
    
         
             
            We use schema for emulate development or tests environment on PostgreSQL adapter.
         
     | 
| 
       87 
130 
     | 
    
         | 
| 
      
 131 
     | 
    
         
            +
            Structure dump to `db/clickhouse_structure.sql` file:
         
     | 
| 
      
 132 
     | 
    
         
            +
             
     | 
| 
      
 133 
     | 
    
         
            +
                $ rake clickhouse:structure:dump
         
     | 
| 
      
 134 
     | 
    
         
            +
                
         
     | 
| 
      
 135 
     | 
    
         
            +
            Structure load from `db/clickhouse_structure.sql` file:
         
     | 
| 
      
 136 
     | 
    
         
            +
             
     | 
| 
      
 137 
     | 
    
         
            +
                $ rake clickhouse:structure:load
         
     | 
| 
      
 138 
     | 
    
         
            +
             
     | 
| 
      
 139 
     | 
    
         
            +
            ### Dump / Load for only Clickhouse database using
         
     | 
| 
      
 140 
     | 
    
         
            +
             
     | 
| 
      
 141 
     | 
    
         
            +
                $ rake db:schema:dump  
         
     | 
| 
      
 142 
     | 
    
         
            +
                $ rake db:schema:load  
         
     | 
| 
      
 143 
     | 
    
         
            +
                $ rake db:structure:dump  
         
     | 
| 
      
 144 
     | 
    
         
            +
                $ rake db:structure:load  
         
     | 
| 
      
 145 
     | 
    
         
            +
                
         
     | 
| 
       88 
146 
     | 
    
         
             
            ### Insert and select data
         
     | 
| 
       89 
147 
     | 
    
         | 
| 
       90 
148 
     | 
    
         
             
            ```ruby
         
     | 
| 
         @@ -105,7 +163,7 @@ ActionView.maximum(:date) 
     | 
|
| 
       105 
163 
     | 
    
         | 
| 
       106 
164 
     | 
    
         
             
            Donations to this project are going directly to [PNixx](https://github.com/PNixx), the original author of this project:
         
     | 
| 
       107 
165 
     | 
    
         | 
| 
       108 
     | 
    
         
            -
            * BTC address: ` 
     | 
| 
      
 166 
     | 
    
         
            +
            * BTC address: `1H3rhpf7WEF5JmMZ3PVFMQc7Hm29THgUfN`
         
     | 
| 
       109 
167 
     | 
    
         
             
            * ETH address: `0x6F094365A70fe7836A633d2eE80A1FA9758234d5`
         
     | 
| 
       110 
168 
     | 
    
         
             
            * XMR address: `42gP71qLB5M43RuDnrQ3vSJFFxis9Kw9VMURhpx9NLQRRwNvaZRjm2TFojAMC8Fk1BQhZNKyWhoyJSn5Ak9kppgZPjE17Zh`
         
     | 
| 
       111 
169 
     | 
    
         | 
| 
         @@ -23,8 +23,8 @@ Gem::Specification.new do |spec| 
     | 
|
| 
       23 
23 
     | 
    
         
             
              spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
         
     | 
| 
       24 
24 
     | 
    
         
             
              spec.require_paths = ['lib']
         
     | 
| 
       25 
25 
     | 
    
         | 
| 
       26 
     | 
    
         
            -
              spec.add_runtime_dependency 'bundler', ' 
     | 
| 
       27 
     | 
    
         
            -
              spec.add_runtime_dependency 'activerecord', ' 
     | 
| 
      
 26 
     | 
    
         
            +
              spec.add_runtime_dependency 'bundler', '>= 1.13.4'
         
     | 
| 
      
 27 
     | 
    
         
            +
              spec.add_runtime_dependency 'activerecord', '>= 5.2'
         
     | 
| 
       28 
28 
     | 
    
         | 
| 
       29 
29 
     | 
    
         
             
              spec.add_development_dependency 'bundler', '~> 1.15'
         
     | 
| 
       30 
30 
     | 
    
         
             
              spec.add_development_dependency 'rake', '~> 10.0'
         
     | 
| 
         @@ -17,6 +17,8 @@ module ActiveRecord 
     | 
|
| 
       17 
17 
     | 
    
         
             
                    def exec_query(sql, name = nil, binds = [], prepare: false)
         
     | 
| 
       18 
18 
     | 
    
         
             
                      result = do_execute(sql, name)
         
     | 
| 
       19 
19 
     | 
    
         
             
                      ActiveRecord::Result.new(result['meta'].map { |m| m['name'] }, result['data'])
         
     | 
| 
      
 20 
     | 
    
         
            +
                    rescue StandardError => _e
         
     | 
| 
      
 21 
     | 
    
         
            +
                      raise ActiveRecord::ActiveRecordError, "Response: #{result}"
         
     | 
| 
       20 
22 
     | 
    
         
             
                    end
         
     | 
| 
       21 
23 
     | 
    
         | 
| 
       22 
24 
     | 
    
         
             
                    def exec_update(_sql, _name = nil, _binds = [])
         
     | 
| 
         @@ -34,41 +36,10 @@ module ActiveRecord 
     | 
|
| 
       34 
36 
     | 
    
         
             
                    end
         
     | 
| 
       35 
37 
     | 
    
         | 
| 
       36 
38 
     | 
    
         
             
                    def table_options(table)
         
     | 
| 
       37 
     | 
    
         
            -
                      sql = do_system_execute("SHOW CREATE TABLE  
     | 
| 
      
 39 
     | 
    
         
            +
                      sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
         
     | 
| 
       38 
40 
     | 
    
         
             
                      { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
         
     | 
| 
       39 
41 
     | 
    
         
             
                    end
         
     | 
| 
       40 
42 
     | 
    
         | 
| 
       41 
     | 
    
         
            -
                    # @todo copied from  ActiveRecord::ConnectionAdapters::SchemaStatements v5.2.2
         
     | 
| 
       42 
     | 
    
         
            -
                    # Why version column type of String, but insert to Integer?
         
     | 
| 
       43 
     | 
    
         
            -
                    def assume_migrated_upto_version(version, migrations_paths)
         
     | 
| 
       44 
     | 
    
         
            -
                      migrations_paths = Array(migrations_paths)
         
     | 
| 
       45 
     | 
    
         
            -
                      version = version.to_i
         
     | 
| 
       46 
     | 
    
         
            -
                      sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name)
         
     | 
| 
       47 
     | 
    
         
            -
             
     | 
| 
       48 
     | 
    
         
            -
                      migrated = ActiveRecord::SchemaMigration.all_versions.map(&:to_i)
         
     | 
| 
       49 
     | 
    
         
            -
                      versions = migration_context.migration_files.map do |file|
         
     | 
| 
       50 
     | 
    
         
            -
                        migration_context.parse_migration_filename(file).first.to_i
         
     | 
| 
       51 
     | 
    
         
            -
                      end
         
     | 
| 
       52 
     | 
    
         
            -
             
     | 
| 
       53 
     | 
    
         
            -
                      unless migrated.include?(version)
         
     | 
| 
       54 
     | 
    
         
            -
                        do_execute( "INSERT INTO #{sm_table} (version) VALUES (#{quote(version.to_s)})", 'SchemaMigration', format: nil)
         
     | 
| 
       55 
     | 
    
         
            -
                      end
         
     | 
| 
       56 
     | 
    
         
            -
             
     | 
| 
       57 
     | 
    
         
            -
                      inserting = (versions - migrated).select { |v| v < version }
         
     | 
| 
       58 
     | 
    
         
            -
                      if inserting.any?
         
     | 
| 
       59 
     | 
    
         
            -
                        if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
         
     | 
| 
       60 
     | 
    
         
            -
                          raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
         
     | 
| 
       61 
     | 
    
         
            -
                        end
         
     | 
| 
       62 
     | 
    
         
            -
                        if supports_multi_insert?
         
     | 
| 
       63 
     | 
    
         
            -
                          do_system_execute insert_versions_sql(inserting.map(&:to_s))
         
     | 
| 
       64 
     | 
    
         
            -
                        else
         
     | 
| 
       65 
     | 
    
         
            -
                          inserting.each do |v|
         
     | 
| 
       66 
     | 
    
         
            -
                            do_system_execute insert_versions_sql(v)
         
     | 
| 
       67 
     | 
    
         
            -
                          end
         
     | 
| 
       68 
     | 
    
         
            -
                        end
         
     | 
| 
       69 
     | 
    
         
            -
                      end
         
     | 
| 
       70 
     | 
    
         
            -
                    end
         
     | 
| 
       71 
     | 
    
         
            -
             
     | 
| 
       72 
43 
     | 
    
         
             
                    # Not indexes on clickhouse
         
     | 
| 
       73 
44 
     | 
    
         
             
                    def indexes(table_name, name = nil)
         
     | 
| 
       74 
45 
     | 
    
         
             
                      []
         
     | 
| 
         @@ -86,21 +57,22 @@ module ActiveRecord 
     | 
|
| 
       86 
57 
     | 
    
         
             
                      end
         
     | 
| 
       87 
58 
     | 
    
         
             
                    end
         
     | 
| 
       88 
59 
     | 
    
         | 
| 
       89 
     | 
    
         
            -
                     
     | 
| 
       90 
     | 
    
         
            -
             
     | 
| 
       91 
     | 
    
         
            -
                    def apply_format(sql, format)
         
     | 
| 
       92 
     | 
    
         
            -
                      format ? "#{sql} FORMAT #{format}" : sql
         
     | 
| 
       93 
     | 
    
         
            -
                    end
         
     | 
| 
       94 
     | 
    
         
            -
             
     | 
| 
       95 
     | 
    
         
            -
                    def do_execute(sql, name = nil, format: 'JSONCompact')
         
     | 
| 
      
 60 
     | 
    
         
            +
                    def do_execute(sql, name = nil, format: 'JSONCompact', settings: {})
         
     | 
| 
       96 
61 
     | 
    
         
             
                      log(sql, "#{adapter_name} #{name}") do
         
     | 
| 
       97 
62 
     | 
    
         
             
                        formatted_sql = apply_format(sql, format)
         
     | 
| 
       98 
     | 
    
         
            -
                         
     | 
| 
      
 63 
     | 
    
         
            +
                        request_params = @config || {}
         
     | 
| 
      
 64 
     | 
    
         
            +
                        res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql)
         
     | 
| 
       99 
65 
     | 
    
         | 
| 
       100 
66 
     | 
    
         
             
                        process_response(res)
         
     | 
| 
       101 
67 
     | 
    
         
             
                      end
         
     | 
| 
       102 
68 
     | 
    
         
             
                    end
         
     | 
| 
       103 
69 
     | 
    
         | 
| 
      
 70 
     | 
    
         
            +
                    private
         
     | 
| 
      
 71 
     | 
    
         
            +
             
     | 
| 
      
 72 
     | 
    
         
            +
                    def apply_format(sql, format)
         
     | 
| 
      
 73 
     | 
    
         
            +
                      format ? "#{sql} FORMAT #{format}" : sql
         
     | 
| 
      
 74 
     | 
    
         
            +
                    end
         
     | 
| 
      
 75 
     | 
    
         
            +
             
     | 
| 
       104 
76 
     | 
    
         
             
                    def process_response(res)
         
     | 
| 
       105 
77 
     | 
    
         
             
                      case res.code.to_i
         
     | 
| 
       106 
78 
     | 
    
         
             
                      when 200
         
     | 
| 
         @@ -109,6 +81,8 @@ module ActiveRecord 
     | 
|
| 
       109 
81 
     | 
    
         
             
                        raise ActiveRecord::ActiveRecordError,
         
     | 
| 
       110 
82 
     | 
    
         
             
                          "Response code: #{res.code}:\n#{res.body}"
         
     | 
| 
       111 
83 
     | 
    
         
             
                      end
         
     | 
| 
      
 84 
     | 
    
         
            +
                    rescue JSON::ParserError
         
     | 
| 
      
 85 
     | 
    
         
            +
                      res.body
         
     | 
| 
       112 
86 
     | 
    
         
             
                    end
         
     | 
| 
       113 
87 
     | 
    
         | 
| 
       114 
88 
     | 
    
         
             
                    def log_with_debug(sql, name = nil)
         
     | 
| 
         @@ -121,7 +95,11 @@ module ActiveRecord 
     | 
|
| 
       121 
95 
     | 
    
         
             
                    end
         
     | 
| 
       122 
96 
     | 
    
         | 
| 
       123 
97 
     | 
    
         
             
                    def create_table_definition(*args)
         
     | 
| 
       124 
     | 
    
         
            -
                       
     | 
| 
      
 98 
     | 
    
         
            +
                      if ActiveRecord::version >= Gem::Version.new('6')
         
     | 
| 
      
 99 
     | 
    
         
            +
                        Clickhouse::TableDefinition.new(self, *args)
         
     | 
| 
      
 100 
     | 
    
         
            +
                      else
         
     | 
| 
      
 101 
     | 
    
         
            +
                        Clickhouse::TableDefinition.new(*args)
         
     | 
| 
      
 102 
     | 
    
         
            +
                      end
         
     | 
| 
       125 
103 
     | 
    
         
             
                    end
         
     | 
| 
       126 
104 
     | 
    
         | 
| 
       127 
105 
     | 
    
         
             
                    def new_column_from_field(table_name, field)
         
     | 
| 
         @@ -130,13 +108,17 @@ module ActiveRecord 
     | 
|
| 
       130 
108 
     | 
    
         
             
                      default = field[3]
         
     | 
| 
       131 
109 
     | 
    
         
             
                      default_value = extract_value_from_default(default)
         
     | 
| 
       132 
110 
     | 
    
         
             
                      default_function = extract_default_function(default_value, default)
         
     | 
| 
       133 
     | 
    
         
            -
                       
     | 
| 
      
 111 
     | 
    
         
            +
                      if ActiveRecord::version >= Gem::Version.new('6')
         
     | 
| 
      
 112 
     | 
    
         
            +
                        ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), default_function)
         
     | 
| 
      
 113 
     | 
    
         
            +
                      else
         
     | 
| 
      
 114 
     | 
    
         
            +
                        ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), table_name, default_function)
         
     | 
| 
      
 115 
     | 
    
         
            +
                      end
         
     | 
| 
       134 
116 
     | 
    
         
             
                    end
         
     | 
| 
       135 
117 
     | 
    
         | 
| 
       136 
118 
     | 
    
         
             
                    protected
         
     | 
| 
       137 
119 
     | 
    
         | 
| 
       138 
120 
     | 
    
         
             
                    def table_structure(table_name)
         
     | 
| 
       139 
     | 
    
         
            -
                      result = do_system_execute("DESCRIBE TABLE  
     | 
| 
      
 121 
     | 
    
         
            +
                      result = do_system_execute("DESCRIBE TABLE `#{table_name}`", table_name)
         
     | 
| 
       140 
122 
     | 
    
         
             
                      data = result['data']
         
     | 
| 
       141 
123 
     | 
    
         | 
| 
       142 
124 
     | 
    
         
             
                      return data unless data.empty?
         
     | 
| 
         @@ -2,7 +2,6 @@ 
     | 
|
| 
       2 
2 
     | 
    
         | 
| 
       3 
3 
     | 
    
         
             
            require 'clickhouse-activerecord/arel/visitors/to_sql'
         
     | 
| 
       4 
4 
     | 
    
         
             
            require 'clickhouse-activerecord/arel/table'
         
     | 
| 
       5 
     | 
    
         
            -
            require 'active_record/connection_adapters/abstract_adapter'
         
     | 
| 
       6 
5 
     | 
    
         
             
            require 'active_record/connection_adapters/clickhouse/oid/date'
         
     | 
| 
       7 
6 
     | 
    
         
             
            require 'active_record/connection_adapters/clickhouse/oid/date_time'
         
     | 
| 
       8 
7 
     | 
    
         
             
            require 'active_record/connection_adapters/clickhouse/oid/big_integer'
         
     | 
| 
         @@ -19,6 +18,7 @@ module ActiveRecord 
     | 
|
| 
       19 
18 
     | 
    
         
             
                    config = config.symbolize_keys
         
     | 
| 
       20 
19 
     | 
    
         
             
                    host = config[:host] || 'localhost'
         
     | 
| 
       21 
20 
     | 
    
         
             
                    port = config[:port] || 8123
         
     | 
| 
      
 21 
     | 
    
         
            +
                    ssl = config[:ssl].present? ? config[:ssl] : port == 443
         
     | 
| 
       22 
22 
     | 
    
         | 
| 
       23 
23 
     | 
    
         
             
                    if config.key?(:database)
         
     | 
| 
       24 
24 
     | 
    
         
             
                      database = config[:database]
         
     | 
| 
         @@ -26,11 +26,26 @@ module ActiveRecord 
     | 
|
| 
       26 
26 
     | 
    
         
             
                      raise ArgumentError, 'No database specified. Missing argument: database.'
         
     | 
| 
       27 
27 
     | 
    
         
             
                    end
         
     | 
| 
       28 
28 
     | 
    
         | 
| 
       29 
     | 
    
         
            -
                    ConnectionAdapters::ClickhouseAdapter.new( 
     | 
| 
      
 29 
     | 
    
         
            +
                    ConnectionAdapters::ClickhouseAdapter.new(logger, [host, port, ssl], { user: config[:username], password: config[:password], database: database }.compact, config)
         
     | 
| 
       30 
30 
     | 
    
         
             
                  end
         
     | 
| 
       31 
31 
     | 
    
         
             
                end
         
     | 
| 
       32 
32 
     | 
    
         
             
              end
         
     | 
| 
       33 
33 
     | 
    
         | 
| 
      
 34 
     | 
    
         
            +
              class Relation
         
     | 
| 
      
 35 
     | 
    
         
            +
             
     | 
| 
      
 36 
     | 
    
         
            +
                # Replace for only ClickhouseAdapter
         
     | 
| 
      
 37 
     | 
    
         
            +
                def reverse_order!
         
     | 
| 
      
 38 
     | 
    
         
            +
                  orders = order_values.uniq
         
     | 
| 
      
 39 
     | 
    
         
            +
                  orders.reject!(&:blank?)
         
     | 
| 
      
 40 
     | 
    
         
            +
                  if self.connection.is_a?(ConnectionAdapters::ClickhouseAdapter) && orders.empty? && !primary_key
         
     | 
| 
      
 41 
     | 
    
         
            +
                    self.order_values = %w(date created_at).select {|c| column_names.include?(c) }.map{|c| arel_attribute(c).desc }
         
     | 
| 
      
 42 
     | 
    
         
            +
                  else
         
     | 
| 
      
 43 
     | 
    
         
            +
                    self.order_values = reverse_sql_order(orders)
         
     | 
| 
      
 44 
     | 
    
         
            +
                  end
         
     | 
| 
      
 45 
     | 
    
         
            +
                  self
         
     | 
| 
      
 46 
     | 
    
         
            +
                end
         
     | 
| 
      
 47 
     | 
    
         
            +
              end
         
     | 
| 
      
 48 
     | 
    
         
            +
             
     | 
| 
       34 
49 
     | 
    
         
             
              module TypeCaster
         
     | 
| 
       35 
50 
     | 
    
         
             
                class Map
         
     | 
| 
       36 
51 
     | 
    
         
             
                  def is_view
         
     | 
| 
         @@ -78,17 +93,34 @@ module ActiveRecord 
     | 
|
| 
       78 
93 
     | 
    
         
             
                  include Clickhouse::SchemaStatements
         
     | 
| 
       79 
94 
     | 
    
         | 
| 
       80 
95 
     | 
    
         
             
                  # Initializes and connects a Clickhouse adapter.
         
     | 
| 
       81 
     | 
    
         
            -
                  def initialize( 
     | 
| 
       82 
     | 
    
         
            -
                    super( 
     | 
| 
      
 96 
     | 
    
         
            +
                  def initialize(logger, connection_parameters, config, full_config)
         
     | 
| 
      
 97 
     | 
    
         
            +
                    super(nil, logger)
         
     | 
| 
       83 
98 
     | 
    
         
             
                    @connection_parameters = connection_parameters
         
     | 
| 
       84 
99 
     | 
    
         
             
                    @config = config
         
     | 
| 
       85 
     | 
    
         
            -
                    @debug = debug
         
     | 
| 
      
 100 
     | 
    
         
            +
                    @debug = full_config[:debug] || false
         
     | 
| 
      
 101 
     | 
    
         
            +
                    @full_config = full_config
         
     | 
| 
       86 
102 
     | 
    
         | 
| 
       87 
103 
     | 
    
         
             
                    @prepared_statements = false
         
     | 
| 
      
 104 
     | 
    
         
            +
                    if ActiveRecord::version == Gem::Version.new('6.0.0')
         
     | 
| 
      
 105 
     | 
    
         
            +
                      @prepared_statement_status = Concurrent::ThreadLocalVar.new(false)
         
     | 
| 
      
 106 
     | 
    
         
            +
                    end
         
     | 
| 
       88 
107 
     | 
    
         | 
| 
       89 
108 
     | 
    
         
             
                    connect
         
     | 
| 
       90 
109 
     | 
    
         
             
                  end
         
     | 
| 
       91 
110 
     | 
    
         | 
| 
      
 111 
     | 
    
         
            +
                  # Support SchemaMigration from v5.2.2 to v6+
         
     | 
| 
      
 112 
     | 
    
         
            +
                  def schema_migration # :nodoc:
         
     | 
| 
      
 113 
     | 
    
         
            +
                    if ActiveRecord::version >= Gem::Version.new('6')
         
     | 
| 
      
 114 
     | 
    
         
            +
                      super
         
     | 
| 
      
 115 
     | 
    
         
            +
                    else
         
     | 
| 
      
 116 
     | 
    
         
            +
                      ActiveRecord::SchemaMigration
         
     | 
| 
      
 117 
     | 
    
         
            +
                    end
         
     | 
| 
      
 118 
     | 
    
         
            +
                  end
         
     | 
| 
      
 119 
     | 
    
         
            +
             
     | 
| 
      
 120 
     | 
    
         
            +
                  def migrations_paths
         
     | 
| 
      
 121 
     | 
    
         
            +
                    @full_config[:migrations_paths] || 'db/migrate_clickhouse'
         
     | 
| 
      
 122 
     | 
    
         
            +
                  end
         
     | 
| 
      
 123 
     | 
    
         
            +
             
     | 
| 
       92 
124 
     | 
    
         
             
                  def arel_visitor # :nodoc:
         
     | 
| 
       93 
125 
     | 
    
         
             
                    ClickhouseActiverecord::Arel::Visitors::ToSql.new(self)
         
     | 
| 
       94 
126 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -145,7 +177,11 @@ module ActiveRecord 
     | 
|
| 
       145 
177 
     | 
    
         
             
                  end
         
     | 
| 
       146 
178 
     | 
    
         | 
| 
       147 
179 
     | 
    
         
             
                  def column_name_for_operation(operation, node) # :nodoc:
         
     | 
| 
       148 
     | 
    
         
            -
                     
     | 
| 
      
 180 
     | 
    
         
            +
                    if ActiveRecord::version >= Gem::Version.new('6')
         
     | 
| 
      
 181 
     | 
    
         
            +
                      visitor.compile(node)
         
     | 
| 
      
 182 
     | 
    
         
            +
                    else
         
     | 
| 
      
 183 
     | 
    
         
            +
                      column_name_from_arel_node(node)
         
     | 
| 
      
 184 
     | 
    
         
            +
                    end
         
     | 
| 
       149 
185 
     | 
    
         
             
                  end
         
     | 
| 
       150 
186 
     | 
    
         | 
| 
       151 
187 
     | 
    
         
             
                  # Executes insert +sql+ statement in the context of this connection using
         
     | 
| 
         @@ -166,16 +202,24 @@ module ActiveRecord 
     | 
|
| 
       166 
202 
     | 
    
         | 
| 
       167 
203 
     | 
    
         
             
                  # Create a new ClickHouse database.
         
     | 
| 
       168 
204 
     | 
    
         
             
                  def create_database(name)
         
     | 
| 
       169 
     | 
    
         
            -
                    sql = "CREATE DATABASE #{quote_table_name(name)}"
         
     | 
| 
      
 205 
     | 
    
         
            +
                    sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
         
     | 
| 
       170 
206 
     | 
    
         
             
                    log_with_debug(sql, adapter_name) do
         
     | 
| 
       171 
207 
     | 
    
         
             
                      res = @connection.post("/?#{@config.except(:database).to_param}", "CREATE DATABASE #{quote_table_name(name)}")
         
     | 
| 
       172 
208 
     | 
    
         
             
                      process_response(res)
         
     | 
| 
       173 
209 
     | 
    
         
             
                    end
         
     | 
| 
       174 
210 
     | 
    
         
             
                  end
         
     | 
| 
       175 
211 
     | 
    
         | 
| 
      
 212 
     | 
    
         
            +
                  def create_table(table_name, comment: nil, **options)
         
     | 
| 
      
 213 
     | 
    
         
            +
                    super(
         
     | 
| 
      
 214 
     | 
    
         
            +
                      apply_cluster(table_name),
         
     | 
| 
      
 215 
     | 
    
         
            +
                      comment: comment,
         
     | 
| 
      
 216 
     | 
    
         
            +
                      **options
         
     | 
| 
      
 217 
     | 
    
         
            +
                    )
         
     | 
| 
      
 218 
     | 
    
         
            +
                  end
         
     | 
| 
      
 219 
     | 
    
         
            +
             
     | 
| 
       176 
220 
     | 
    
         
             
                  # Drops a ClickHouse database.
         
     | 
| 
       177 
221 
     | 
    
         
             
                  def drop_database(name) #:nodoc:
         
     | 
| 
       178 
     | 
    
         
            -
                    sql = "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
         
     | 
| 
      
 222 
     | 
    
         
            +
                    sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
         
     | 
| 
       179 
223 
     | 
    
         
             
                    log_with_debug(sql, adapter_name) do
         
     | 
| 
       180 
224 
     | 
    
         
             
                      res = @connection.post("/?#{@config.except(:database).to_param}", sql)
         
     | 
| 
       181 
225 
     | 
    
         
             
                      process_response(res)
         
     | 
| 
         @@ -183,7 +227,7 @@ module ActiveRecord 
     | 
|
| 
       183 
227 
     | 
    
         
             
                  end
         
     | 
| 
       184 
228 
     | 
    
         | 
| 
       185 
229 
     | 
    
         
             
                  def drop_table(table_name, options = {}) # :nodoc:
         
     | 
| 
       186 
     | 
    
         
            -
                    do_execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
         
     | 
| 
      
 230 
     | 
    
         
            +
                    do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
         
     | 
| 
       187 
231 
     | 
    
         
             
                  end
         
     | 
| 
       188 
232 
     | 
    
         | 
| 
       189 
233 
     | 
    
         
             
                  protected
         
     | 
| 
         @@ -195,8 +239,15 @@ module ActiveRecord 
     | 
|
| 
       195 
239 
     | 
    
         
             
                  private
         
     | 
| 
       196 
240 
     | 
    
         | 
| 
       197 
241 
     | 
    
         
             
                  def connect
         
     | 
| 
       198 
     | 
    
         
            -
                     
     | 
| 
       199 
     | 
    
         
            -
             
     | 
| 
      
 242 
     | 
    
         
            +
                    @connection = Net::HTTP.start(@connection_parameters[0], @connection_parameters[1], use_ssl: @connection_parameters[2], verify_mode: OpenSSL::SSL::VERIFY_NONE)
         
     | 
| 
      
 243 
     | 
    
         
            +
                  end
         
     | 
| 
      
 244 
     | 
    
         
            +
             
     | 
| 
      
 245 
     | 
    
         
            +
                  def cluster
         
     | 
| 
      
 246 
     | 
    
         
            +
                    @full_config[:cluster]
         
     | 
| 
      
 247 
     | 
    
         
            +
                  end
         
     | 
| 
      
 248 
     | 
    
         
            +
             
     | 
| 
      
 249 
     | 
    
         
            +
                  def apply_cluster(sql)
         
     | 
| 
      
 250 
     | 
    
         
            +
                    cluster ? "#{sql} ON CLUSTER #{cluster}" : sql
         
     | 
| 
       200 
251 
     | 
    
         
             
                  end
         
     | 
| 
       201 
252 
     | 
    
         
             
                end
         
     | 
| 
       202 
253 
     | 
    
         
             
              end
         
     | 
| 
         @@ -3,7 +3,7 @@ module ClickhouseActiverecord 
     | 
|
| 
       3 
3 
     | 
    
         | 
| 
       4 
4 
     | 
    
         
             
                def table(table, stream)
         
     | 
| 
       5 
5 
     | 
    
         
             
                  stream.puts "  # TABLE: #{table}"
         
     | 
| 
       6 
     | 
    
         
            -
                  stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE  
     | 
| 
      
 6 
     | 
    
         
            +
                  stream.puts "  # SQL: #{@connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)}"
         
     | 
| 
       7 
7 
     | 
    
         
             
                  super(table.gsub(/^\.inner\./, ''), stream)
         
     | 
| 
       8 
8 
     | 
    
         
             
                end
         
     | 
| 
       9 
9 
     | 
    
         
             
              end
         
     | 
| 
         @@ -31,13 +31,27 @@ module ClickhouseActiverecord 
     | 
|
| 
       31 
31 
     | 
    
         
             
                  create
         
     | 
| 
       32 
32 
     | 
    
         
             
                end
         
     | 
| 
       33 
33 
     | 
    
         | 
| 
      
 34 
     | 
    
         
            +
                def structure_dump(*args)
         
     | 
| 
      
 35 
     | 
    
         
            +
                  tables = connection.execute("SHOW TABLES FROM #{@configuration['database']}")['data'].flatten
         
     | 
| 
      
 36 
     | 
    
         
            +
             
     | 
| 
      
 37 
     | 
    
         
            +
                  File.open(args.first, 'w:utf-8') do |file|
         
     | 
| 
      
 38 
     | 
    
         
            +
                    tables.each do |table|
         
     | 
| 
      
 39 
     | 
    
         
            +
                      next if table.match(/\.inner/)
         
     | 
| 
      
 40 
     | 
    
         
            +
                      file.puts connection.execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first).gsub("#{@configuration['database']}.", '') + ";\n\n"
         
     | 
| 
      
 41 
     | 
    
         
            +
                    end
         
     | 
| 
      
 42 
     | 
    
         
            +
                  end
         
     | 
| 
      
 43 
     | 
    
         
            +
                end
         
     | 
| 
      
 44 
     | 
    
         
            +
             
     | 
| 
      
 45 
     | 
    
         
            +
                def structure_load(*args)
         
     | 
| 
      
 46 
     | 
    
         
            +
                  File.read(args.first).split(";\n\n").each { |sql| connection.execute(sql) }
         
     | 
| 
      
 47 
     | 
    
         
            +
                end
         
     | 
| 
      
 48 
     | 
    
         
            +
             
     | 
| 
       34 
49 
     | 
    
         
             
                def migrate
         
     | 
| 
       35 
50 
     | 
    
         
             
                  check_target_version
         
     | 
| 
       36 
51 
     | 
    
         | 
| 
       37 
52 
     | 
    
         
             
                  verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] != "false" : true
         
     | 
| 
       38 
53 
     | 
    
         
             
                  scope = ENV["SCOPE"]
         
     | 
| 
       39 
54 
     | 
    
         
             
                  verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, verbose
         
     | 
| 
       40 
     | 
    
         
            -
                  binding.pry
         
     | 
| 
       41 
55 
     | 
    
         
             
                  connection.migration_context.migrate(target_version) do |migration|
         
     | 
| 
       42 
56 
     | 
    
         
             
                    scope.blank? || scope == migration.scope
         
     | 
| 
       43 
57 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -6,6 +6,20 @@ class ClickhouseMigrationGenerator < ActiveRecord::Generators::MigrationGenerato 
     | 
|
| 
       6 
6 
     | 
    
         
             
              def create_migration_file
         
     | 
| 
       7 
7 
     | 
    
         
             
                set_local_assigns!
         
     | 
| 
       8 
8 
     | 
    
         
             
                validate_file_name!
         
     | 
| 
       9 
     | 
    
         
            -
                migration_template @migration_template, " 
     | 
| 
      
 9 
     | 
    
         
            +
                migration_template @migration_template, File.join(db_migrate_path, "#{file_name}.rb")
         
     | 
| 
      
 10 
     | 
    
         
            +
              end
         
     | 
| 
      
 11 
     | 
    
         
            +
             
     | 
| 
      
 12 
     | 
    
         
            +
              private
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
              def db_migrate_path
         
     | 
| 
      
 15 
     | 
    
         
            +
                if defined?(Rails.application) && Rails.application
         
     | 
| 
      
 16 
     | 
    
         
            +
                  configured_migrate_path || default_migrate_path
         
     | 
| 
      
 17 
     | 
    
         
            +
                else
         
     | 
| 
      
 18 
     | 
    
         
            +
                  default_migrate_path
         
     | 
| 
      
 19 
     | 
    
         
            +
                end
         
     | 
| 
      
 20 
     | 
    
         
            +
              end
         
     | 
| 
      
 21 
     | 
    
         
            +
             
     | 
| 
      
 22 
     | 
    
         
            +
              def default_migrate_path
         
     | 
| 
      
 23 
     | 
    
         
            +
                "db/migrate_clickhouse"
         
     | 
| 
       10 
24 
     | 
    
         
             
              end
         
     | 
| 
       11 
25 
     | 
    
         
             
            end
         
     | 
    
        data/lib/tasks/clickhouse.rake
    CHANGED
    
    | 
         @@ -2,6 +2,70 @@ 
     | 
|
| 
       2 
2 
     | 
    
         | 
| 
       3 
3 
     | 
    
         
             
            namespace :clickhouse do
         
     | 
| 
       4 
4 
     | 
    
         | 
| 
      
 5 
     | 
    
         
            +
              task prepare_schema_migration_table: :environment do
         
     | 
| 
      
 6 
     | 
    
         
            +
                cluster, database, replica = ActiveRecord::Base.connection_config.values_at(:cluster, :database, :replica)
         
     | 
| 
      
 7 
     | 
    
         
            +
                return if cluster.nil?
         
     | 
| 
      
 8 
     | 
    
         
            +
             
     | 
| 
      
 9 
     | 
    
         
            +
                connection = ActiveRecord::Base.connection
         
     | 
| 
      
 10 
     | 
    
         
            +
                key_options = connection.internal_string_options_for_primary_key
         
     | 
| 
      
 11 
     | 
    
         
            +
                block = Proc.new do |t|
         
     | 
| 
      
 12 
     | 
    
         
            +
                  t.string :version, key_options
         
     | 
| 
      
 13 
     | 
    
         
            +
                end
         
     | 
| 
      
 14 
     | 
    
         
            +
                distributed_table_name = ".#{ActiveRecord::SchemaMigration.table_name}_distributed"
         
     | 
| 
      
 15 
     | 
    
         
            +
                unless connection.table_exists?(distributed_table_name)
         
     | 
| 
      
 16 
     | 
    
         
            +
                  options = { id: false }
         
     | 
| 
      
 17 
     | 
    
         
            +
                  if replica
         
     | 
| 
      
 18 
     | 
    
         
            +
                    shard = replica.is_a?(String) ? replica : '{shard}'
         
     | 
| 
      
 19 
     | 
    
         
            +
                    options[:options] = <<-SQL
         
     | 
| 
      
 20 
     | 
    
         
            +
                      ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
         
     | 
| 
      
 21 
     | 
    
         
            +
                      PARTITION BY version ORDER BY (version) SETTINGS index_granularity = 8192
         
     | 
| 
      
 22 
     | 
    
         
            +
                    SQL
         
     | 
| 
      
 23 
     | 
    
         
            +
                  end
         
     | 
| 
      
 24 
     | 
    
         
            +
                  connection.create_table("`#{distributed_table_name}`", options, &block)
         
     | 
| 
      
 25 
     | 
    
         
            +
                end
         
     | 
| 
      
 26 
     | 
    
         
            +
                unless connection.table_exists?(ActiveRecord::SchemaMigration.table_name)
         
     | 
| 
      
 27 
     | 
    
         
            +
                  connection.create_table(
         
     | 
| 
      
 28 
     | 
    
         
            +
                    ActiveRecord::SchemaMigration.table_name,
         
     | 
| 
      
 29 
     | 
    
         
            +
                    id: false,
         
     | 
| 
      
 30 
     | 
    
         
            +
                    options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(version))",
         
     | 
| 
      
 31 
     | 
    
         
            +
                    &block
         
     | 
| 
      
 32 
     | 
    
         
            +
                  )
         
     | 
| 
      
 33 
     | 
    
         
            +
                end
         
     | 
| 
      
 34 
     | 
    
         
            +
              end
         
     | 
| 
      
 35 
     | 
    
         
            +
             
     | 
| 
      
 36 
     | 
    
         
            +
              task prepare_internal_metadata_table: :environment do
         
     | 
| 
      
 37 
     | 
    
         
            +
                cluster, database, replica = ActiveRecord::Base.connection_config.values_at(:cluster, :database, :replica)
         
     | 
| 
      
 38 
     | 
    
         
            +
                return if cluster.nil?
         
     | 
| 
      
 39 
     | 
    
         
            +
             
     | 
| 
      
 40 
     | 
    
         
            +
                connection = ActiveRecord::Base.connection
         
     | 
| 
      
 41 
     | 
    
         
            +
                key_options = connection.internal_string_options_for_primary_key
         
     | 
| 
      
 42 
     | 
    
         
            +
                block = Proc.new do |t|
         
     | 
| 
      
 43 
     | 
    
         
            +
                  t.string :key, key_options
         
     | 
| 
      
 44 
     | 
    
         
            +
                  t.string :value
         
     | 
| 
      
 45 
     | 
    
         
            +
                  t.timestamps
         
     | 
| 
      
 46 
     | 
    
         
            +
                end
         
     | 
| 
      
 47 
     | 
    
         
            +
                distributed_table_name = ".#{ActiveRecord::InternalMetadata.table_name}_distributed"
         
     | 
| 
      
 48 
     | 
    
         
            +
                unless connection.table_exists?(distributed_table_name)
         
     | 
| 
      
 49 
     | 
    
         
            +
                  options = { id: false }
         
     | 
| 
      
 50 
     | 
    
         
            +
                  if replica
         
     | 
| 
      
 51 
     | 
    
         
            +
                    shard = replica.is_a?(String) ? replica : '{shard}'
         
     | 
| 
      
 52 
     | 
    
         
            +
                    options[:options] = <<-SQL
         
     | 
| 
      
 53 
     | 
    
         
            +
                      ReplicatedMergeTree('/clickhouse/tables/{cluster}/#{shard}/#{database}.`#{distributed_table_name}`', '{replica}')
         
     | 
| 
      
 54 
     | 
    
         
            +
                      PARTITION BY toDate(created_at) ORDER BY (created_at) SETTINGS index_granularity = 8192
         
     | 
| 
      
 55 
     | 
    
         
            +
                    SQL
         
     | 
| 
      
 56 
     | 
    
         
            +
                  end
         
     | 
| 
      
 57 
     | 
    
         
            +
                  connection.create_table("`#{distributed_table_name}`", options, &block)
         
     | 
| 
      
 58 
     | 
    
         
            +
                end
         
     | 
| 
      
 59 
     | 
    
         
            +
                unless connection.table_exists?(ActiveRecord::InternalMetadata.table_name)
         
     | 
| 
      
 60 
     | 
    
         
            +
                  connection.create_table(
         
     | 
| 
      
 61 
     | 
    
         
            +
                    ActiveRecord::InternalMetadata.table_name,
         
     | 
| 
      
 62 
     | 
    
         
            +
                    id: false,
         
     | 
| 
      
 63 
     | 
    
         
            +
                    options: "Distributed(#{cluster},#{database},`#{distributed_table_name}`,sipHash64(created_at))",
         
     | 
| 
      
 64 
     | 
    
         
            +
                    &block
         
     | 
| 
      
 65 
     | 
    
         
            +
                  )
         
     | 
| 
      
 66 
     | 
    
         
            +
                end
         
     | 
| 
      
 67 
     | 
    
         
            +
              end
         
     | 
| 
      
 68 
     | 
    
         
            +
             
     | 
| 
       5 
69 
     | 
    
         
             
              task load_config: :environment do
         
     | 
| 
       6 
70 
     | 
    
         
             
                ENV['SCHEMA'] = "db/clickhouse_schema.rb"
         
     | 
| 
       7 
71 
     | 
    
         
             
                ActiveRecord::Migrator.migrations_paths = ["db/migrate_clickhouse"]
         
     | 
| 
         @@ -27,6 +91,18 @@ namespace :clickhouse do 
     | 
|
| 
       27 
91 
     | 
    
         | 
| 
       28 
92 
     | 
    
         
             
              end
         
     | 
| 
       29 
93 
     | 
    
         | 
| 
      
 94 
     | 
    
         
            +
              namespace :structure do
         
     | 
| 
      
 95 
     | 
    
         
            +
                desc 'Load database structure'
         
     | 
| 
      
 96 
     | 
    
         
            +
                task load: [:load_config, 'db:check_protected_environments'] do
         
     | 
| 
      
 97 
     | 
    
         
            +
                  ClickhouseActiverecord::Tasks.new(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"]).structure_load("#{Rails.root}/db/clickhouse_structure.sql")
         
     | 
| 
      
 98 
     | 
    
         
            +
                end
         
     | 
| 
      
 99 
     | 
    
         
            +
             
     | 
| 
      
 100 
     | 
    
         
            +
                desc 'Dump database structure'
         
     | 
| 
      
 101 
     | 
    
         
            +
                task dump: [:load_config, 'db:check_protected_environments'] do
         
     | 
| 
      
 102 
     | 
    
         
            +
                  ClickhouseActiverecord::Tasks.new(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"]).structure_dump("#{Rails.root}/db/clickhouse_structure.sql")
         
     | 
| 
      
 103 
     | 
    
         
            +
                end
         
     | 
| 
      
 104 
     | 
    
         
            +
              end
         
     | 
| 
      
 105 
     | 
    
         
            +
             
     | 
| 
       30 
106 
     | 
    
         
             
              desc 'Creates the database from DATABASE_URL or config/database.yml'
         
     | 
| 
       31 
107 
     | 
    
         
             
              task create: [:load_config] do
         
     | 
| 
       32 
108 
     | 
    
         
             
                ActiveRecord::Tasks::DatabaseTasks.create(ActiveRecord::Base.configurations["#{Rails.env}_clickhouse"])
         
     | 
| 
         @@ -49,7 +125,7 @@ namespace :clickhouse do 
     | 
|
| 
       49 
125 
     | 
    
         
             
              end
         
     | 
| 
       50 
126 
     | 
    
         | 
| 
       51 
127 
     | 
    
         
             
              desc 'Migrate the clickhouse database'
         
     | 
| 
       52 
     | 
    
         
            -
              task migrate: :load_config do
         
     | 
| 
      
 128 
     | 
    
         
            +
              task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
         
     | 
| 
       53 
129 
     | 
    
         
             
                Rake::Task['db:migrate'].execute
         
     | 
| 
       54 
130 
     | 
    
         
             
              end
         
     | 
| 
       55 
131 
     | 
    
         
             
            end
         
     | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,22 +1,19 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: clickhouse-activerecord
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 0.3. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 0.3.13
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - Sergey Odintsov
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire: 
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: exe
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date:  
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2020-09-11 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies:
         
     | 
| 
       13 
13 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       14 
14 
     | 
    
         
             
              name: bundler
         
     | 
| 
       15 
15 
     | 
    
         
             
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
       16 
16 
     | 
    
         
             
                requirements:
         
     | 
| 
       17 
     | 
    
         
            -
                - - "~>"
         
     | 
| 
       18 
     | 
    
         
            -
                  - !ruby/object:Gem::Version
         
     | 
| 
       19 
     | 
    
         
            -
                    version: '1.13'
         
     | 
| 
       20 
17 
     | 
    
         
             
                - - ">="
         
     | 
| 
       21 
18 
     | 
    
         
             
                  - !ruby/object:Gem::Version
         
     | 
| 
       22 
19 
     | 
    
         
             
                    version: 1.13.4
         
     | 
| 
         @@ -24,9 +21,6 @@ dependencies: 
     | 
|
| 
       24 
21 
     | 
    
         
             
              prerelease: false
         
     | 
| 
       25 
22 
     | 
    
         
             
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
       26 
23 
     | 
    
         
             
                requirements:
         
     | 
| 
       27 
     | 
    
         
            -
                - - "~>"
         
     | 
| 
       28 
     | 
    
         
            -
                  - !ruby/object:Gem::Version
         
     | 
| 
       29 
     | 
    
         
            -
                    version: '1.13'
         
     | 
| 
       30 
24 
     | 
    
         
             
                - - ">="
         
     | 
| 
       31 
25 
     | 
    
         
             
                  - !ruby/object:Gem::Version
         
     | 
| 
       32 
26 
     | 
    
         
             
                    version: 1.13.4
         
     | 
| 
         @@ -34,14 +28,14 @@ dependencies: 
     | 
|
| 
       34 
28 
     | 
    
         
             
              name: activerecord
         
     | 
| 
       35 
29 
     | 
    
         
             
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
       36 
30 
     | 
    
         
             
                requirements:
         
     | 
| 
       37 
     | 
    
         
            -
                - - " 
     | 
| 
      
 31 
     | 
    
         
            +
                - - ">="
         
     | 
| 
       38 
32 
     | 
    
         
             
                  - !ruby/object:Gem::Version
         
     | 
| 
       39 
33 
     | 
    
         
             
                    version: '5.2'
         
     | 
| 
       40 
34 
     | 
    
         
             
              type: :runtime
         
     | 
| 
       41 
35 
     | 
    
         
             
              prerelease: false
         
     | 
| 
       42 
36 
     | 
    
         
             
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
       43 
37 
     | 
    
         
             
                requirements:
         
     | 
| 
       44 
     | 
    
         
            -
                - - " 
     | 
| 
      
 38 
     | 
    
         
            +
                - - ">="
         
     | 
| 
       45 
39 
     | 
    
         
             
                  - !ruby/object:Gem::Version
         
     | 
| 
       46 
40 
     | 
    
         
             
                    version: '5.2'
         
     | 
| 
       47 
41 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
         @@ -153,8 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement 
     | 
|
| 
       153 
147 
     | 
    
         
             
                - !ruby/object:Gem::Version
         
     | 
| 
       154 
148 
     | 
    
         
             
                  version: '0'
         
     | 
| 
       155 
149 
     | 
    
         
             
            requirements: []
         
     | 
| 
       156 
     | 
    
         
            -
             
     | 
| 
       157 
     | 
    
         
            -
            rubygems_version: 2.5.2.3
         
     | 
| 
      
 150 
     | 
    
         
            +
            rubygems_version: 3.0.1
         
     | 
| 
       158 
151 
     | 
    
         
             
            signing_key: 
         
     | 
| 
       159 
152 
     | 
    
         
             
            specification_version: 4
         
     | 
| 
       160 
153 
     | 
    
         
             
            summary: ClickHouse ActiveRecord
         
     |