clickhouse-activerecord 0.4.6 → 0.4.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: deeb0e0ddf3a6bef04338637ddf814a826f93b4e69779905f28c280a5ca1a400
-  data.tar.gz: 80cba4b321f42cb978be4d6bf4740b03274a7bd0ff882b35d4ac0760736fe9a8
+  metadata.gz: d8d1d60f75e0f889dc5d947df77924f1490c0325f097d634048fb25405eeca70
+  data.tar.gz: 309eda1c3f605682c04525fce8633b85a73a38643e367f33d87588939f7815fe
 SHA512:
-  metadata.gz: 8c18476f8984ab7e7c834b712eff4f9d8e3995287b3db6ace0e4e819ac19dc6bd31b149fee7da970223807170b1902ce8d2af81ad8ac8a56bc73641e26111d22
-  data.tar.gz: '094cdb750234da2b59efd755c7fef3e4f4bf3484b124d7d2b74fe999ee2a25cddb435324fd88a49248943cb372459df31a3d151905e93118b456231ae94cbd2c'
+  metadata.gz: 5727f1a23c48e04db1052bfbf4c56af412338be88f4b985ed6700e9a833436bcde893ba9cb1ed7b89473dae95b31e4b70de26c6885cce10b8fffcdb9c65fbb3d
+  data.tar.gz: 18fad92b704c1c41b97029facbc020111c3b067cbb825d6e5c4d11d6a4b7a220873b922aea7112ec741f1d87a1a5c2217a11047e764959a2edffb98e0b7f2463

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+### Version 0.4.10 (Mar 10, 2021)
+
+ * Support ClickHouse 20.9+
+ * Fix schema create / dump
+ * Support all integer types through :limit and :unsigned [@bdevel](https://github.com/bdevel)
+
 ### Version 0.4.4 (Sep 23, 2020)
 
  * Full support migration and rollback database

data/README.md

@@ -1,7 +1,7 @@
 # Clickhouse::Activerecord
 
 A Ruby database ActiveRecord driver for ClickHouse. Support Rails >= 5.2.
-Support ClickHouse version from 19.14 LTS.
+Support ClickHouse version from 20.9 LTS.
 
 ## Installation
 
@@ -162,6 +162,44 @@ ActionView.maximum(:date)
 #=> 'Wed, 29 Nov 2017'
 ```
 
+
+### Migration Data Types
+
+Integer types are unsigned by default. Specify signed values with `:unsigned =>
+false`. The default integer is `UInt32`
+
+| Type (bit size)    | Range | :limit (byte size) |
+| :---        |    :----:   |          ---: |
+| Int8 | -128 to 127 | 1 | 
+| Int16 | -32768 to 32767 | 2 |
+| Int32 | -2147483648 to 2,147,483,647 | 3,4 |
+| Int64 | -9223372036854775808 to 9223372036854775807] |  5,6,7,8 |
+| Int128 | ... | 9 - 15 |
+| Int256 | ... | 16+ |
+| UInt8 | 0 to 255 | 1 |
+| UInt16 | 0 to 65,535 | 2 |
+| UInt32 | 0 to 4,294,967,295 | 3,4 |
+| UInt64 | 0 to 18446744073709551615 | 5,6,7,8 |
+| UInt256 | 0 to ... | 8+ |
+
+Example:
+
+``` ruby
+class CreateDataItems < ActiveRecord::Migration
+  def change
+    create_table "data_items", id: false, options: "VersionedCollapsingMergeTree(sign, version) PARTITION BY toYYYYMM(day) ORDER BY category", force: :cascade do |t|
+      t.date "day", null: false
+      t.string "category", null: false
+      t.integer "value_in", null: false
+      t.integer "sign", limit: 1, unsigned: false, default: -> { "CAST(1, 'Int8')" }, null: false
+      t.integer "version", limit: 8, default: -> { "CAST(toUnixTimestamp(now()), 'UInt64')" }, null: false
+    end
+  end
+end
+      
+```
+
+
 ### Using replica and cluster params in connection parameters
 
 ```yml

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -39,7 +39,9 @@ module ActiveRecord
           statements << accept(o.primary_keys) if o.primary_keys
 
           create_sql << "(#{statements.join(', ')})" if statements.present?
-          add_table_options!(create_sql, table_options(o))
+          # Attach options for only table or materialized view
+          add_table_options!(create_sql, table_options(o)) if !o.view || o.view && o.materialized
+
           create_sql << " AS #{to_sql(o.as)}" if o.as
           create_sql
         end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -35,13 +35,31 @@ module ActiveRecord
         end
 
         def integer(*args, **options)
-          if options[:limit] == 8
-            args.each { |name| column(name, :big_integer, options.except(:limit)) }
-          else
-            super
+          # default to unsigned
+          unsigned = options[:unsigned]
+          unsigned = true if unsigned.nil?
+
+          kind = :uint32 # default
+
+          if options[:limit]
+            if unsigned
+              kind = :uint8       if options[:limit] == 1
+              kind = :uint16      if options[:limit] == 2
+              kind = :uint32      if [3,4].include?(options[:limit])
+              kind = :uint64      if [5,6,7].include?(options[:limit])
+              kind = :big_integer if options[:limit] == 8
+              kind = :uint256     if options[:limit] > 8
+            else
+              kind = :int8       if options[:limit] == 1
+              kind = :int16      if options[:limit] == 2
+              kind = :int32      if [3,4].include?(options[:limit])
+              kind = :int64     if options[:limit] > 5 && options[:limit] <= 8
+              kind = :int128     if options[:limit] > 8 && options[:limit] <= 16
+              kind = :int256     if options[:limit] > 16
+            end
           end
+          args.each { |name| column(name, kind, options.except(:limit, :unsigned)) }
         end
-
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -36,8 +36,8 @@ module ActiveRecord
         end
 
         def table_options(table)
-          sql = do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first)
-          { options: sql.gsub(/^(?:.*?)ENGINE = (.*?)$/, '\\1') }
+          sql = show_create_table(table)
+          { options: sql.gsub(/^(?:.*?)(?:ENGINE = (.*?))?( AS SELECT .*?)?$/, '\\1').presence, as: sql.match(/^CREATE (?:.*?) AS (SELECT .*?)$/).try(:[], 1) }.compact
         end
 
         # Not indexes on clickhouse

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -79,7 +79,6 @@ module ActiveRecord
 
     class ClickhouseAdapter < AbstractAdapter
       ADAPTER_NAME = 'Clickhouse'.freeze
-
       NATIVE_DATABASE_TYPES = {
         string: { name: 'String' },
         integer: { name: 'UInt32' },
@@ -88,7 +87,21 @@ module ActiveRecord
         decimal: { name: 'Decimal' },
         datetime: { name: 'DateTime' },
         date: { name: 'Date' },
-        boolean: { name: 'UInt8' }
+        boolean: { name: 'UInt8' },
+
+        int8:  { name: 'Int8' },
+        int16: { name: 'Int16' },
+        int32: { name: 'Int32' },
+        int64:  { name: 'Int64' },
+        int128: { name: 'Int128' },
+        int256: { name: 'Int256' },
+
+        uint8: { name: 'UInt8' },
+        uint16: { name: 'UInt16' },
+        uint32: { name: 'UInt32' },
+        uint64: { name: 'UInt64' },
+        # uint128: { name: 'UInt128' }, not yet implemented in clickhouse
+        uint256: { name: 'UInt256' },
       }.freeze
 
       include Clickhouse::SchemaStatements
@@ -139,10 +152,12 @@ module ActiveRecord
           when /(Nullable)?\(?String\)?/
             super('String')
           when /(Nullable)?\(?U?Int8\)?/
-            super('int2')
-          when /(Nullable)?\(?U?Int(16|32)\)?/
-            super('int4')
-          when /(Nullable)?\(?U?Int(64)\)?/
+            1
+          when /(Nullable)?\(?U?Int16\)?/
+            2
+          when /(Nullable)?\(?U?Int32\)?/
+            nil
+          when /(Nullable)?\(?U?Int64\)?/
             8
           else
             super
@@ -154,14 +169,20 @@ module ActiveRecord
         register_class_with_limit m, %r(String), Type::String
         register_class_with_limit m, 'Date',  Clickhouse::OID::Date
         register_class_with_limit m, 'DateTime',  Clickhouse::OID::DateTime
-        register_class_with_limit m, %r(Uint8), Type::UnsignedInteger
-        m.alias_type 'UInt16', 'UInt8'
-        m.alias_type 'UInt32', 'UInt8'
-        register_class_with_limit m, %r(UInt64), Type::UnsignedInteger
+
         register_class_with_limit m, %r(Int8), Type::Integer
-        m.alias_type 'Int16', 'Int8'
-        m.alias_type 'Int32', 'Int8'
+        register_class_with_limit m, %r(Int16), Type::Integer
+        register_class_with_limit m, %r(Int32), Type::Integer
         register_class_with_limit m, %r(Int64), Type::Integer
+        register_class_with_limit m, %r(Int128), Type::Integer
+        register_class_with_limit m, %r(Int256), Type::Integer
+
+        register_class_with_limit m, %r(Uint8), Type::UnsignedInteger
+        register_class_with_limit m, %r(UInt16), Type::UnsignedInteger
+        register_class_with_limit m, %r(UInt32), Type::UnsignedInteger
+        register_class_with_limit m, %r(UInt64), Type::UnsignedInteger
+        #register_class_with_limit m, %r(UInt128), Type::UnsignedInteger #not implemnted in clickhouse
+        register_class_with_limit m, %r(UInt256), Type::UnsignedInteger
       end
 
       # Quoting time without microseconds
@@ -201,6 +222,12 @@ module ActiveRecord
         ClickhouseActiverecord::SchemaDumper.create(self, options)
       end
 
+      # @param [String] table
+      # @return [String]
+      def show_create_table(table)
+        do_system_execute("SHOW CREATE TABLE `#{table}`")['data'].try(:first).try(:first).gsub(/[\n\s]+/m, ' ')
+      end
+
       # Create a new ClickHouse database.
       def create_database(name)
         sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"

data/lib/clickhouse-activerecord/migration.rb

@@ -7,7 +7,7 @@ module ClickhouseActiverecord
         unless table_exists?
           version_options = connection.internal_string_options_for_primary_key
 
-          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
+          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true) do |t|
             t.string :version, version_options
             t.column :active, 'Int8', null: false, default: '1'
             t.datetime :ver, null: false, default: -> { 'now()' }
@@ -27,7 +27,7 @@ module ClickhouseActiverecord
         unless table_exists?
           key_options = connection.internal_string_options_for_primary_key
 
-          connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
+          connection.create_table(table_name, id: false, options: connection.adapter_name.downcase == 'clickhouse' ? 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)' : '', if_not_exists: true) do |t|
             t.string :key, key_options
             t.string :value
             t.timestamps

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -33,16 +33,24 @@ module ClickhouseActiverecord
 HEADER
     end
 
+    def tables(stream)
+      sorted_tables = @connection.tables.sort {|a,b| @connection.show_create_table(a).match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : a <=> b }
+
+      sorted_tables.each do |table_name|
+        table(table_name, stream) unless ignored?(table_name)
+      end
+    end
+
     def table(table, stream)
-      if table.match(/^\.inner\./).nil?
+      if table.match(/^\.inner/).nil?
         unless simple
           stream.puts "  # TABLE: #{table}"
-          sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
+          sql = @connection.show_create_table(table)
           stream.puts "  # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
           # super(table.gsub(/^\.inner\./, ''), stream)
 
           # detect view table
-          match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
+          match = sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/)
         end
 
         # Copy from original dumper
@@ -125,5 +133,21 @@ HEADER
         super
       end
     end
+
+    def schema_limit(column)
+      return nil if column.type == :float
+      super
+    end
+
+    def schema_unsigned(column)
+      return nil unless column.type == :integer && !simple
+      (column.sql_type =~ /(Nullable)?\(?UInt\d+\)?/).nil? ? false : nil
+    end
+
+    def prepare_column_options(column)
+      spec = {}
+      spec[:unsigned] = schema_unsigned(column)
+      spec.merge(super).compact
+    end
   end
 end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.4.6'
+  VERSION = '0.4.11'
 end

data/lib/tasks/clickhouse.rake

@@ -20,7 +20,7 @@ namespace :clickhouse do
 
     # todo not testing
     desc 'Load database schema'
-    task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do |t, args|
+    task load: [:load_config, :prepare_internal_metadata_table] do |t, args|
       simple = ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
       ClickhouseActiverecord::SchemaMigration.drop_table
       load("#{Rails.root}/db/clickhouse_schema#{simple}.rb")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.11
 platform: ruby
 authors:
 - Sergey Odintsov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-30 00:00:00.000000000 Z
+date: 2021-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: ClickHouse ActiveRecord