clickhouse-activerecord 0.4.0 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8020d332d2b7b707fa2776a37fbb8d2c217ec50703962d06938be53eaae22796
-  data.tar.gz: 6979255cb79396c6574e4b58c0e82ae39a42612fe1068b1308d98a1222c58b64
+  metadata.gz: ed292c2d626374fcb26194a4ca28ca34d60f8c49105b7217bfc7654a5a1b08f0
+  data.tar.gz: 97d4074d47e641e8202fd8796ea049f78bf76d86906ce08877f3eb96b236b116
 SHA512:
-  metadata.gz: 3aa6d799adf1fc66d49c3dff6d02c5ad0349e467fb1067a1453de8bfcb3316abb46681367b7dfd3ebd8f82f13460cb658c4f167ff23398bba8b9cc459c5b6df1
-  data.tar.gz: a1c8e252279678efb2fe1f8c36a8a7e1c5bfda24896438245921deee5862e94c047dced14b85408b5e1343606ec7055100b813af399568f5ee405a255c268777
+  metadata.gz: 23c4d834c3c3bbaa000b8bd9202064cbabf2c366a8984d1b2fd92dc51e4c6d2811301e39cd8d2b80141581dcedb8cb6647dcbe59d90eb0b0987afdd5f1b22efa
+  data.tar.gz: 0ac017b6614a3072fdad422db66a36201d5e3a62269230f102fbe8b6c7085f48a139cd8c0753adab11ca53a25e91e6eb327298036ebc385eb26f302c4e5178f8

data/CHANGELOG.md

@@ -1,8 +1,9 @@
-### Version 0.4.0 (Sep 18, 2020)
+### Version 0.4.4 (Sep 23, 2020)
 
  * Full support migration and rollback database
  * Support cluster and replica. Auto inject to SQL queries.
  * Fix schema dump/load
+ * Can dump schema for using PostgreSQL
 
 ### Version 0.3.10 (Dec 20, 2019)
 

data/README.md

@@ -125,7 +125,11 @@ Schema load from `db/clickhouse_schema.rb` file:
 
     $ rake clickhouse:schema:load
 
-We use schema for emulate development or tests environment on PostgreSQL adapter.
+For export schema to PostgreSQL, you need use:
+
+    $ rake clickhouse:schema:dump -- --simple
+
+Schema will be dump to `db/clickhouse_schema_simple.rb`. If default file exists, it will be auto update after migration.
     
 Structure dump to `db/clickhouse_structure.sql` file:
 

data/lib/active_record/connection_adapters/clickhouse/schema_creation.rb

@@ -6,15 +6,17 @@ module ActiveRecord
       class SchemaCreation < AbstractAdapter::SchemaCreation# :nodoc:
 
         def visit_AddColumnDefinition(o)
-          +"ADD COLUMN #{accept(o.column)}"
+          sql = +"ADD COLUMN #{accept(o.column)}"
+          sql << " AFTER " + quote_column_name(o.column.options[:after]) if o.column.options.key?(:after)
+          sql
         end
 
         def add_column_options!(sql, options)
-          sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           if options[:null] || options[:null].nil?
             sql.gsub!(/\s+(.*)/, ' Nullable(\1)')
           end
           sql.gsub!(/(\sString)\(\d+\)/, '\1')
+          sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options)
           sql
         end
 
@@ -47,6 +49,25 @@ module ActiveRecord
           " TEMPORARY" if o.temporary
           " MATERIALIZED" if o.materialized
         end
+
+        def visit_ChangeColumnDefinition(o)
+          column = o.column
+          column.sql_type = type_to_sql(column.type, column.options)
+          options = column_options(column)
+
+          quoted_column_name = quote_column_name(o.name)
+          type = column.sql_type
+          type = "Nullable(#{type})" if options[:null]
+          change_column_sql = +"MODIFY COLUMN #{quoted_column_name} #{type}"
+
+          if options.key?(:default)
+            quoted_default = quote_default_expression(options[:default], column)
+            change_column_sql << " DEFAULT #{quoted_default}"
+          end
+
+          change_column_sql
+        end
+
       end
     end
   end

data/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb

@@ -28,7 +28,7 @@ module ActiveRecord
           @if_not_exists = if_not_exists
           @options = options
           @as = as
-          @name = @conn.apply_cluster(name)
+          @name = name
           @comment = comment
           @view = view || materialized
           @materialized = materialized

data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb

@@ -156,6 +156,8 @@ module ActiveRecord
           when "true".freeze, "false".freeze
             default
             # Object identifier types
+          when "''"
+            ''
           when /\A-?\d+\z/
             $1
           else

data/lib/active_record/connection_adapters/clickhouse_adapter.rb

@@ -205,7 +205,7 @@ module ActiveRecord
       def create_database(name)
         sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
         log_with_debug(sql, adapter_name) do
-          res = @connection.post("/?#{@config.except(:database).to_param}", "CREATE DATABASE #{quote_table_name(name)}")
+          res = @connection.post("/?#{@config.except(:database).to_param}", sql)
           process_response(res)
         end
       end
@@ -213,7 +213,7 @@ module ActiveRecord
       def create_view(table_name, **options)
         options.merge!(view: true)
         options = apply_replica(table_name, options)
-        td = create_table_definition(table_name, options)
+        td = create_table_definition(apply_cluster(table_name), options)
         yield td if block_given?
 
         if options[:force]
@@ -225,7 +225,7 @@ module ActiveRecord
 
       def create_table(table_name, **options)
         options = apply_replica(table_name, options)
-        td = create_table_definition(table_name, options)
+        td = create_table_definition(apply_cluster(table_name), options)
         yield td if block_given?
 
         if options[:force]
@@ -244,10 +244,29 @@ module ActiveRecord
         end
       end
 
+      def rename_table(table_name, new_name)
+        do_execute apply_cluster "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}"
+      end
+
       def drop_table(table_name, options = {}) # :nodoc:
         do_execute apply_cluster "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}"
       end
 
+      def change_column(table_name, column_name, type, options = {})
+        result = do_execute "ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}"
+        raise "Error parse json response: #{result}" if result.presence && !result.is_a?(Hash)
+      end
+
+      def change_column_null(table_name, column_name, null, default = nil)
+        structure = table_structure(table_name).select{|v| v[0] == column_name.to_s}.first
+        raise "Column #{column_name} not found in table #{table_name}" if structure.nil?
+        change_column table_name, column_name, structure[1].gsub(/(Nullable\()?(.*?)\)?/, '\2'), {null: null, default: default}.compact
+      end
+
+      def change_column_default(table_name, column_name, default)
+        change_column table_name, column_name, nil, {default: default}.compact
+      end
+
       def cluster
         @full_config[:cluster_name]
       end
@@ -270,6 +289,12 @@ module ActiveRecord
         result
       end
 
+      def change_column_for_alter(table_name, column_name, type, options = {})
+        td = create_table_definition(table_name)
+        cd = td.new_column_definition(column_name, type, options)
+        schema_creation.accept(ChangeColumnDefinition.new(cd, column_name))
+      end
+
       private
 
       def connect
@@ -277,7 +302,7 @@ module ActiveRecord
       end
 
       def apply_replica(table, options)
-        if replica && cluster
+        if replica && cluster && options[:options]
           match = options[:options].match(/^(.*?MergeTree)\(([^\)]*)\)(.*?)$/)
           if match
             options[:options] = "Replicated#{match[1]}(#{([replica_path(table), replica].map{|v| "'#{v}'"} + [match[2].presence]).compact.join(', ')})#{match[3]}"

data/lib/clickhouse-activerecord/migration.rb

@@ -7,7 +7,7 @@ module ClickhouseActiverecord
         unless table_exists?
           version_options = connection.internal_string_options_for_primary_key
 
-          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)') do |t|
+          connection.create_table(table_name, id: false, options: 'ReplacingMergeTree(ver) PARTITION BY version ORDER BY (version)', if_not_exists: true) do |t|
             t.string :version, version_options
             t.column :active, 'Int8', null: false, default: '1'
             t.datetime :ver, null: false, default: -> { 'now()' }
@@ -27,7 +27,7 @@ module ClickhouseActiverecord
         unless table_exists?
           key_options = connection.internal_string_options_for_primary_key
 
-          connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)') do |t|
+          connection.create_table(table_name, id: false, options: 'MergeTree() PARTITION BY toDate(created_at) ORDER BY (created_at)', if_not_exists: true) do |t|
             t.string :key, key_options
             t.string :value
             t.timestamps

data/lib/clickhouse-activerecord/schema_dumper.rb

@@ -1,6 +1,19 @@
 module ClickhouseActiverecord
   class SchemaDumper < ::ActiveRecord::ConnectionAdapters::SchemaDumper
 
+    attr_accessor :simple
+
+    class << self
+      def dump(connection = ActiveRecord::Base.connection, stream = STDOUT, config = ActiveRecord::Base, default = false)
+        dumper = connection.create_schema_dumper(generate_options(config))
+        dumper.simple = default
+        dumper.dump(stream)
+        stream
+      end
+    end
+
+    private
+
     def header(stream)
       stream.puts <<HEADER
 # This file is auto-generated from the current state of the database. Instead
@@ -8,27 +21,29 @@ module ClickhouseActiverecord
 # incrementally modify your database, and then regenerate this schema definition.
 #
 # This file is the source Rails uses to define your schema when running `rails
-# clickhouse:schema:load`. When creating a new database, `rails clickhouse:schema:load` tends to
+# #{simple ? 'db' : 'clickhouse'}:schema:load`. When creating a new database, `rails #{simple ? 'db' : 'clickhouse'}:schema:load` tends to
 # be faster and is potentially less error prone than running all of your
 # migrations from scratch. Old migrations may fail to apply correctly if those
 # migrations use external dependencies or application code.
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ClickhouseActiverecord::Schema.define(#{define_params}) do
+#{simple ? 'ActiveRecord' : 'ClickhouseActiverecord'}::Schema.define(#{define_params}) do
 
 HEADER
     end
 
     def table(table, stream)
       if table.match(/^\.inner\./).nil?
-        stream.puts "  # TABLE: #{table}"
-        sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
-        stream.puts "  # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
-        # super(table.gsub(/^\.inner\./, ''), stream)
-
-        # detect view table
-        match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
+        unless simple
+          stream.puts "  # TABLE: #{table}"
+          sql = @connection.do_system_execute("SHOW CREATE TABLE `#{table.gsub(/^\.inner\./, '')}`")['data'].try(:first).try(:first)
+          stream.puts "  # SQL: #{sql.gsub(/ENGINE = Replicated(.*?)\('[^']+',\s*'[^']+',?\s?([^\)]*)?\)/, "ENGINE = \\1(\\2)")}" if sql
+          # super(table.gsub(/^\.inner\./, ''), stream)
+
+          # detect view table
+          match = sql.match(/^CREATE\s+(MATERIALIZED)\s+VIEW/)
+        end
 
         # Copy from original dumper
         columns = @connection.columns(table)
@@ -40,9 +55,11 @@ HEADER
 
           tbl.print "  create_table #{remove_prefix_and_suffix(table).inspect}"
 
-          # Add materialize flag
-          tbl.print ', view: true' if match
-          tbl.print ', materialized: true' if match && match[1].presence
+          unless simple
+            # Add materialize flag
+            tbl.print ', view: true' if match
+            tbl.print ', materialized: true' if match && match[1].presence
+          end
 
           case pk
           when String
@@ -58,21 +75,25 @@ HEADER
             tbl.print ", id: false"
           end
 
-          table_options = @connection.table_options(table)
-          if table_options.present?
-            tbl.print ", #{format_options(table_options)}"
+          unless simple
+            table_options = @connection.table_options(table)
+            if table_options.present?
+              tbl.print ", #{format_options(table_options)}"
+            end
           end
 
           tbl.puts ", force: :cascade do |t|"
 
           # then dump all non-primary key columns
-          columns.each do |column|
-            raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
-            next if column.name == pk
-            type, colspec = column_spec(column)
-            tbl.print "    t.#{type} #{column.name.inspect}"
-            tbl.print ", #{format_colspec(colspec)}" if colspec.present?
-            tbl.puts
+          if simple || !match
+            columns.each do |column|
+              raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" unless @connection.valid_type?(column.type)
+              next if column.name == pk
+              type, colspec = column_spec(column)
+              tbl.print "    t.#{type} #{column.name.inspect}"
+              tbl.print ", #{format_colspec(colspec)}" if colspec.present?
+              tbl.puts
+            end
           end
 
           indexes_in_create(table, tbl)
@@ -96,5 +117,13 @@ HEADER
       end
       super
     end
+
+    def format_colspec(colspec)
+      if simple
+        super.gsub(/CAST\('?([^,']*)'?,\s?'.*?'\)/, "\\1")
+      else
+        super
+      end
+    end
   end
 end

data/lib/clickhouse-activerecord/version.rb

@@ -1,3 +1,3 @@
 module ClickhouseActiverecord
-  VERSION = '0.4.0'
+  VERSION = '0.4.7'
 end

data/lib/tasks/clickhouse.rake

@@ -3,11 +3,11 @@
 namespace :clickhouse do
 
   task prepare_schema_migration_table: :environment do
-    ClickhouseActiverecord::SchemaMigration.create_table
+    ClickhouseActiverecord::SchemaMigration.create_table unless ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any?
   end
 
   task prepare_internal_metadata_table: :environment do
-    ClickhouseActiverecord::InternalMetadata.create_table
+    ClickhouseActiverecord::InternalMetadata.create_table unless ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any?
   end
 
   task load_config: :environment do
@@ -20,16 +20,19 @@ namespace :clickhouse do
 
     # todo not testing
     desc 'Load database schema'
-    task load: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
-      load("#{Rails.root}/db/clickhouse_schema.rb")
+    task load: [:load_config, :prepare_internal_metadata_table] do |t, args|
+      simple = ENV['simple'] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
+      ClickhouseActiverecord::SchemaMigration.drop_table
+      load("#{Rails.root}/db/clickhouse_schema#{simple}.rb")
     end
 
     desc 'Dump database schema'
-    task dump: :environment do
-      filename = "#{Rails.root}/db/clickhouse_schema.rb"
+    task dump: :environment do |t, args|
+      simple = ENV['simple'] || args[:simple] || ARGV.map{|a| a.include?('--simple') ? true : nil}.compact.any? ? '_simple' : nil
+      filename = "#{Rails.root}/db/clickhouse_schema#{simple}.rb"
       File.open(filename, 'w:utf-8') do |file|
         ActiveRecord::Base.establish_connection(:"#{Rails.env}_clickhouse")
-        ClickhouseActiverecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
+        ClickhouseActiverecord::SchemaDumper.dump(ActiveRecord::Base.connection, file, ActiveRecord::Base, !!simple)
       end
     end
 
@@ -71,6 +74,9 @@ namespace :clickhouse do
   desc 'Migrate the clickhouse database'
   task migrate: [:load_config, :prepare_schema_migration_table, :prepare_internal_metadata_table] do
     Rake::Task['db:migrate'].execute
+    if File.exists? "#{Rails.root}/db/clickhouse_schema_simple.rb"
+      Rake::Task['clickhouse:schema:dump'].execute(simple: true)
+    end
   end
 
   desc 'Rollback the clickhouse database'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clickhouse-activerecord
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.7
 platform: ruby
 authors:
 - Sergey Odintsov
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-18 00:00:00.000000000 Z
+date: 2020-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -134,7 +134,7 @@ homepage: https://github.com/pnixx/clickhouse-activerecord
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -150,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: ClickHouse ActiveRecord
 test_files: []