activerecord-cockroachdb-adapter 6.0.0beta3 → 6.1.0beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2fb5df1ff24503630d2700365d1c6e9b8b8f84306ce5d2b331712f86adea1f5
-  data.tar.gz: ffa086c2093b8e80424e5b85847d6f0a7a3503437df8168e72c1abab0a042528
+  metadata.gz: 5d0954fd425d90342f04c1b6559b78e398be9e523e67779476d81f925a1aa611
+  data.tar.gz: 1f990c34784394bafa77933eb0d49965a0eac81151f2468c92a66c1bc9beb78a
 SHA512:
-  metadata.gz: b8e3520c981683e242469bcef709c715aa5fe9240b1b6a49fb930dfb8d9f9f395ee4acfae615d06b477e70fc10f8ba1c98bf4bb3489484ea4a2a7f4ec4f5fd54
-  data.tar.gz: 6368fb8ed0c73348538dc978585beb495838002cea456071ddd4026c2435d3ff146626c19f3101070c1d2fba84efc222aab17a57bba3410130e213eb6379b2ea
+  metadata.gz: 3074426e81a83bb4ae2e50047c6db97dc75d9b8a822c3fd1153fb3403457bb56e7f36acb42802e5dd777c056dd0864db7dc26e875e7a644cb158f0a1c8001f3a
+  data.tar.gz: 8ff7c637812c1b262452ca1ef6ad8308a3f620a47e3798c6ef7b93096fedb362ce52372f6c0b870af1b475a31cf83f670f4e6d65498c231f5759a475820568af

data/CONTRIBUTING.md

@@ -78,6 +78,26 @@ RAILS_SOURCE="path/to/local_copy" bundle exec rake test
 
 `test/config.yml` assumes CockroachDB will be running at localhost:26257 with a root user. Make changes to `test/config.yml` as needed.
 
+### Run Tests from a Backup
+
+Loading the full test schema every time a test runs can take a while, so for cases where loading the schema sequentially is unimportant, it is possible to use a backup to set up the database. This is significantly faster than the standard method and is provided to run individual tests faster, but should not be used to validate a build.
+
+First create the template database.
+
+```bash
+bundle exec rake db:create_test_template
+```
+
+This will create a template database for the current version (ex. `activerecord_test_template611` for version 6.1.1) and create a `BACKUP` in the `nodelocal://self/activerecord-crdb-adapter/#{activerecord_version}` directory.
+
+To load from the template, use the `COCKROACH_LOAD_FROM_TEMPLATE` flag.
+
+```bash
+COCKROACH_LOAD_FROM_TEMPLATE=1 TEST_FILES="test/cases/adapters/postgresql/ddl_test.rb" bundle exec rake test
+```
+
+And the `activerecord_unittest` database will use the `RESTORE` command to load the schema from the template database.
+
 # Improvements
 
 

data/Rakefile

@@ -2,10 +2,30 @@ require "bundler/gem_tasks"
 require "rake/testtask"
 require_relative 'test/support/paths_cockroachdb'
 require_relative 'test/support/rake_helpers'
+require_relative 'test/support/template_creator'
 
 task test: ["test:cockroachdb"]
 task default: [:test]
 
+namespace :db do
+  task "create_test_template" do
+    ENV['DEBUG_COCKROACHDB_ADAPTER'] = "1"
+    ENV['COCKROACH_SKIP_LOAD_SCHEMA'] = "1"
+    ENV["ARCONN"] = "cockroachdb"
+
+    TemplateCreator.connect
+    require_relative 'test/cases/helper'
+
+    # TODO: look into this more, but for some reason the blob alias
+    # is not defined while running this task.
+    ActiveRecord::ConnectionAdapters::CockroachDB::TableDefinition.class_eval do
+      alias :blob :binary
+    end
+
+    TemplateCreator.create_test_template
+  end
+end
+
 namespace :test do
   Rake::TestTask.new("cockroachdb") do |t|
     t.libs = ARTest::CockroachDB.test_load_paths

data/activerecord-cockroachdb-adapter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "activerecord-cockroachdb-adapter"
-  spec.version       = "6.0.0beta3"
+  spec.version       = "6.1.0beta1"
   spec.licenses      = ["Apache-2.0"]
   spec.authors       = ["Cockroach Labs"]
   spec.email         = ["cockroach-db@googlegroups.com"]
@@ -13,8 +13,8 @@ Gem::Specification.new do |spec|
   spec.description   = "Allows the use of CockroachDB as a backend for ActiveRecord and Rails apps."
   spec.homepage      = "https://github.com/cockroachdb/activerecord-cockroachdb-adapter"
 
-  spec.add_dependency "activerecord", "~> 6.0.3"
-  spec.add_dependency "pg", ">= 0.20"
+  spec.add_dependency "activerecord", "~> 6.1"
+  spec.add_dependency "pg", "~> 1.2"
   spec.add_dependency "rgeo-activerecord", "~> 7.0.0"
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'

data/lib/active_record/connection_adapters/cockroachdb/oid/interval.rb

@@ -0,0 +1,126 @@
+# frozen-string-literal: true
+
+require "active_support/duration"
+
+module ActiveRecord
+  module ConnectionAdapters
+    module CockroachDB
+      module OID
+        module Interval # :nodoc:
+          DEFAULT_PRECISION = 6 # microseconds
+
+          def cast_value(value)
+            case value
+            when ::ActiveSupport::Duration
+              value
+            when ::String
+              begin
+                PostgresqlInterval::Parser.parse(value)
+              rescue PostgresqlInterval::ParseError
+                # Try ISO 8601
+                super
+              end
+            else
+              super
+            end
+          end
+
+          def serialize(value)
+            precision = self.precision || DEFAULT_PRECISION
+            case value
+            when ::ActiveSupport::Duration
+              serialize_duration(value, precision)
+            when ::Numeric
+              serialize_duration(value.seconds, precision)
+            else
+              super
+            end
+          end
+
+          def type_cast_for_schema(value)
+            serialize(value).inspect
+          end
+
+          private
+
+            # Convert an ActiveSupport::Duration to
+            # the postgres interval style
+            # ex. 1 year 2 mons 3 days 4 hours 5 minutes 6 seconds
+            def serialize_duration(value, precision)
+              yrs = value.parts.fetch(:years, 0)
+              mons = value.parts.fetch(:months, 0)
+              days = value.parts.fetch(:days, 0)
+              hrs = value.parts.fetch(:hours, 0)
+              mins = value.parts.fetch(:minutes, 0)
+              secs = value.parts.fetch(:seconds, 0).round(precision)
+
+              "#{yrs} years #{mons} mons #{days} days #{hrs} hours #{mins} minutes #{secs} seconds"
+            end
+        end
+
+        PostgreSQL::OID::Interval.prepend(Interval)
+      end
+
+      module PostgresqlInterval
+        class Parser
+          PARTS = ActiveSupport::Duration::PARTS
+          PARTS_IN_SECONDS = ActiveSupport::Duration::PARTS_IN_SECONDS
+
+          # modified regex from https://github.com/jeremyevans/sequel/blob/master/lib/sequel/extensions/pg_interval.rb#L86
+          REGEX = /\A([+-]?\d+ years?\s?)?([+-]?\d+ mons?\s?)?([+-]?\d+ days?\s?)?(?:([+-])?(\d{2,10}):(\d\d):(\d\d(\.\d+)?))?\z/
+
+          def self.parse(string)
+            matches = REGEX.match(string)
+            raise(ParseError) unless matches
+
+            # 1 => years, 2 => months, 3 => days, 4 => nil, 5 => hours,
+            # 6 => minutes, 7 => seconds with fraction digits, 8 => fractional portion of 7
+            duration = 0
+            parts = {}
+
+            if matches[1]
+              val = matches[1].to_i
+              duration += val * PARTS_IN_SECONDS[:years]
+              parts[:years] = val
+            end
+
+            if matches[2]
+              val = matches[2].to_i
+              duration += val * PARTS_IN_SECONDS[:months]
+              parts[:months] = val
+            end
+
+            if matches[3]
+              val = matches[3].to_i
+              duration += val * PARTS_IN_SECONDS[:days]
+              parts[:days] = val
+            end
+
+            if matches[5]
+              val = matches[5].to_i
+              duration += val * PARTS_IN_SECONDS[:hours]
+              parts[:hours] = val
+            end
+
+            if matches[6]
+              val = matches[6].to_i
+              duration += val * PARTS_IN_SECONDS[:minutes]
+              parts[:minutes] = val
+            end
+
+            if matches[7]
+              val = matches[7].to_f
+              duration += val * PARTS_IN_SECONDS[:seconds]
+              parts[:seconds] = val
+            end
+
+            ActiveSupport::Duration.new(duration, parts)
+          end
+        end
+
+        class ParseError < StandardError
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/cockroachdb/schema_statements.rb

@@ -4,7 +4,7 @@ module ActiveRecord
       module SchemaStatements
         include ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaStatements
 
-        def add_index(table_name, column_name, options = {})
+        def add_index(table_name, column_name, **options)
           super
         rescue ActiveRecord::StatementInvalid => error
           if debugging? && error.cause.class == PG::FeatureNotSupported

data/lib/active_record/connection_adapters/cockroachdb/transaction_manager.rb

@@ -9,7 +9,7 @@ module ActiveRecord
         # transactions will be retried until they pass or the max retry limit is
         # exceeded.
         def within_new_transaction(isolation: nil, joinable: true, attempts: 0)
-          super
+          super(isolation: isolation, joinable: joinable)
         rescue ActiveRecord::StatementInvalid => error
           raise unless retryable? error
           raise if attempts >= @connection.max_transaction_retries

data/lib/active_record/connection_adapters/cockroachdb/type.rb

@@ -1,12 +1,10 @@
 module ActiveRecord
   module Type
     class << self
-      private
-
       # Return :postgresql instead of :cockroachdb for current_adapter_name so
       # we can continue using the ActiveRecord::Types defined in
       # PostgreSQLAdapter.
-      def current_adapter_name
+      def adapter_name_from(_model)
         :postgresql
       end
     end

data/lib/active_record/connection_adapters/cockroachdb_adapter.rb

@@ -1,6 +1,6 @@
 require "rgeo/active_record"
 
-require 'active_record/connection_adapters/postgresql_adapter'
+require "active_record/connection_adapters/postgresql_adapter"
 require "active_record/connection_adapters/cockroachdb/column_methods"
 require "active_record/connection_adapters/cockroachdb/schema_statements"
 require "active_record/connection_adapters/cockroachdb/referential_integrity"
@@ -14,6 +14,7 @@ require "active_record/connection_adapters/cockroachdb/column"
 require "active_record/connection_adapters/cockroachdb/spatial_column_info"
 require "active_record/connection_adapters/cockroachdb/setup"
 require "active_record/connection_adapters/cockroachdb/oid/spatial"
+require "active_record/connection_adapters/cockroachdb/oid/interval"
 require "active_record/connection_adapters/cockroachdb/arel_tosql"
 
 # Run to ignore spatial tables that will break schemna dumper.
@@ -24,25 +25,29 @@ module ActiveRecord
   module ConnectionHandling
     def cockroachdb_connection(config)
       # This is copied from the PostgreSQL adapter.
-      conn_params = config.symbolize_keys
-
-      conn_params.delete_if { |_, v| v.nil? }
+      conn_params = config.symbolize_keys.compact
 
       # Map ActiveRecords param names to PGs.
       conn_params[:user] = conn_params.delete(:username) if conn_params[:username]
       conn_params[:dbname] = conn_params.delete(:database) if conn_params[:database]
 
       # Forward only valid config params to PG::Connection.connect.
-      valid_conn_param_keys = PG::Connection.conndefaults_hash.keys + [:sslmode, :application_name]
+      valid_conn_param_keys = PG::Connection.conndefaults_hash.keys + [:requiressl]
       conn_params.slice!(*valid_conn_param_keys)
 
-      conn = PG.connect(conn_params)
-      ConnectionAdapters::CockroachDBAdapter.new(conn, logger, conn_params, config)
-    rescue ::PG::Error, ActiveRecord::ActiveRecordError  => error
-      if error.message.include?("does not exist")
+      ConnectionAdapters::CockroachDBAdapter.new(
+        ConnectionAdapters::CockroachDBAdapter.new_client(conn_params),
+        logger,
+        conn_params,
+        config
+      )
+    # This rescue flow appears in new_client, but it is needed here as well
+    # since Cockroach will sometimes not raise until a query is made.
+    rescue ActiveRecord::StatementInvalid => error
+      if conn_params && conn_params[:dbname] && error.cause.message.include?(conn_params[:dbname])
         raise ActiveRecord::NoDatabaseError
       else
-        raise
+        raise ActiveRecord::ConnectionNotEstablished, error.message
       end
     end
   end
@@ -76,56 +81,6 @@ module ActiveRecord
       include CockroachDB::DatabaseStatements
       include CockroachDB::Quoting
 
-      # override
-      # This method makes a sql query to gather information about columns
-      # in a table. It returns an array of arrays (one for each col) and
-      # passes each to the SchemaStatements#new_column_from_field method
-      # as the field parameter. This data is then used to format the column
-      # objects for the model and sent to the OID for data casting.
-      #
-      # The issue with the default method is that the sql_type field is 
-      # retrieved with the `format_type` function, but this is implemented
-      # differently in CockroachDB than PostGIS, so geometry/geography
-      # types are missing information which makes parsing them impossible.
-      # Below is an example of what `format_type` returns for a geometry
-      # column.
-      #
-      # column_type: geometry(POINT, 4326)
-      # Expected: geometry(POINT, 4326)
-      # Actual: geometry
-      #
-      # The solution is to make the default query with super, then
-      # iterate through the columns and if it is a spatial type,
-      # access the proper column_type with the information_schema.columns 
-      # table.
-      #
-      # @see: https://github.com/rails/rails/blob/8695b028261bdd244e254993255c6641bdbc17a5/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L829
-      def column_definitions(table_name)
-        fields = super
-        # iterate through and identify all spatial fields based on format_type
-        # being geometry or geography, then query for the information_schema.column
-        # column_type because that contains the necessary information.
-        fields.map do |field|
-          dtype = field[1]
-          if dtype == 'geometry' || dtype == 'geography'
-            col_name = field[0]
-            data_type = \
-            query(<<~SQL, "SCHEMA")
-              SELECT c.data_type
-                FROM information_schema.columns c
-              WHERE c.table_name = #{quote(table_name)}
-                AND c.column_name = #{quote(col_name)}
-            SQL
-            field[1] = data_type[0][0]
-          end
-          field
-        end
-      end
-
-      def arel_visitor
-        Arel::Visitors::CockroachDB.new(self)
-      end
-
       def self.spatial_column_options(key)
         SPATIAL_COLUMN_OPTIONS[key]
       end
@@ -276,19 +231,31 @@ module ActiveRecord
             end
           end
 
-          # NOTE(joey): PostgreSQL intervals have a precision.
-          # CockroachDB intervals do not, so overide the type
-          # definition. Returning a ArgumentError may not be correct.
-          # This needs to be tested.
-          m.register_type "interval" do |_, _, sql_type|
+          # Belongs after other types are defined because of issues described
+          # in this https://github.com/rails/rails/pull/38571
+          # Once that PR is merged, we can call super at the top.
+          super(m)
+
+          # Override numeric type. This is almost identical to the default,
+          # except that the conditional based on the fmod is changed.
+          m.register_type "numeric" do |_, fmod, sql_type|
             precision = extract_precision(sql_type)
-            if precision
-              raise(ArgumentError, "CockroachDB does not support precision on intervals, but got precision: #{precision}")
+            scale = extract_scale(sql_type)
+
+            # TODO(#178) this should never use DecimalWithoutScale since scale
+            # is assumed to be 0 if it is not explicitly defined.
+            #
+            # If fmod is -1, that means that precision is defined but not
+            # scale, or neither is defined.
+            if fmod && fmod == -1
+              # Below comment is from ActiveRecord
+              # FIXME: Remove this class, and the second argument to
+              # lookups on PG
+              Type::DecimalWithoutScale.new(precision: precision)
+            else
+              OID::Decimal.new(precision: precision, scale: scale)
             end
-            OID::SpecializedString.new(:interval, precision: precision)
           end
-
-          super(m)
         end
 
         # Configures the encoding, verbosity, schema search path, and time zone of the connection.
@@ -398,6 +365,84 @@ module ActiveRecord
           return "{}"
         end
 
+        # override
+        # This method makes a query to gather information about columns
+        # in a table. It returns an array of arrays (one for each col) and
+        # passes each to the SchemaStatements#new_column_from_field method
+        # as the field parameter. This data is then used to format the column
+        # objects for the model and sent to the OID for data casting.
+        #
+        # Sometimes there are differences between how data is formatted
+        # in Postgres and CockroachDB, so additional queries for certain types
+        # may be necessary to properly form the column definition.
+        #
+        # @see: https://github.com/rails/rails/blob/8695b028261bdd244e254993255c6641bdbc17a5/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L829
+        def column_definitions(table_name)
+          fields = super
+
+          # Use regex comparison because if a type is an array it will
+          # have [] appended to the end of it.
+          target_types = [
+            /geometry/,
+            /geography/,
+            /interval/,
+            /numeric/
+          ]
+          re = Regexp.union(target_types)
+          fields.map do |field|
+            dtype = field[1]
+            if re.match(dtype)
+              crdb_column_definition(field, table_name)
+            else
+              field
+            end
+          end
+        end
+
+        # Use the crdb_sql_type instead of the sql_type returned by
+        # column_definitions. This will include limit,
+        # precision, and scale information in the type.
+        # Ex. geometry -> geometry(point, 4326)
+        def crdb_column_definition(field, table_name)
+          col_name = field[0]
+          data_type = \
+          query(<<~SQL, "SCHEMA")
+            SELECT c.crdb_sql_type
+              FROM information_schema.columns c
+            WHERE c.table_name = #{quote(table_name)}
+              AND c.column_name = #{quote(col_name)}
+          SQL
+          field[1] = data_type[0][0].downcase
+          field
+        end
+
+        # override
+        # This method is used to determine if a
+        # FEATURE_NOT_SUPPORTED error from the PG gem should
+        # be an ActiveRecord::PreparedStatementCacheExpired
+        # error.
+        #
+        # ActiveRecord handles this by checking that the sql state matches the
+        # FEATURE_NOT_SUPPORTED code and that the source function
+        # is "RevalidateCachedQuery" since that is the only function
+        # in postgres that will create this error.
+        #
+        # That method will not work for CockroachDB because the error
+        # originates from the "runExecBuilder" function, so we need
+        # to modify the original to match the CockroachDB behavior.
+        def is_cached_plan_failure?(e)
+          pgerror = e.cause
+
+          pgerror.result.result_error_field(PG::PG_DIAG_SQLSTATE) == FEATURE_NOT_SUPPORTED &&
+            pgerror.result.result_error_field(PG::PG_DIAG_SOURCE_FUNCTION) == "runExecBuilder"
+        rescue
+          false
+        end
+
+        def arel_visitor
+          Arel::Visitors::CockroachDB.new(self)
+        end
+
       # end private
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-cockroachdb-adapter
 version: !ruby/object:Gem::Version
-  version: 6.0.0beta3
+  version: 6.1.0beta1
 platform: ruby
 authors:
 - Cockroach Labs
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-20 00:00:00.000000000 Z
+date: 2021-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 6.0.3
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 6.0.3
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rgeo-activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -81,6 +81,7 @@ files:
 - lib/active_record/connection_adapters/cockroachdb/column_methods.rb
 - lib/active_record/connection_adapters/cockroachdb/database_statements.rb
 - lib/active_record/connection_adapters/cockroachdb/database_tasks.rb
+- lib/active_record/connection_adapters/cockroachdb/oid/interval.rb
 - lib/active_record/connection_adapters/cockroachdb/oid/spatial.rb
 - lib/active_record/connection_adapters/cockroachdb/quoting.rb
 - lib/active_record/connection_adapters/cockroachdb/referential_integrity.rb