activerecord-cockroachdb-adapter 5.2.2 → 6.0.0.pre.beta.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef8a3ba83448f71d7a2d0e5a4d21f065531f17a862aa8a33e7ed4e9470927f0a
-  data.tar.gz: ae23824599ad3bc37773f99e08a143b7242e81094098674d22499df7153c5856
+  metadata.gz: e242232ac6fcfc87b6bddc17eeb852099a959d3a024064ed3a32e85bd80ad786
+  data.tar.gz: 5ab114df9f5fee7fb5c18c5cf0eb0dfdc822f7513f19d79312f65e0b55e1e459
 SHA512:
-  metadata.gz: 4100c35b51262901f1f087be1d0a5b4864f45a89585878e5d00aa12919863cdc66cd8ffddfcd0f3b16037bd6b21539845ddbe18c7901fd2f48f7d7e807f61f0b
-  data.tar.gz: accca94dcb97b512e261a676d031657c8cc34d22590866744ffd6dbaf757a67386d9c642638d77aa4354d5281437e7a5e8c5c5dc141335f441422e20a3605aee
+  metadata.gz: a2de1e036b53ebeb11185eaeb10bd465f9a1103a34307354bc4bbbe94b8beb4c5a2313938e01703db8b68f5a689ee6b9b778b8282bceac3bb852b6569340351b
+  data.tar.gz: 4138ea2ccad0bac070c277adb9ec4aa837be440900f72fe8f2728bac78d58a96d96ed7df9a830841b3389e5f51678dba9377e0598b3aa96d682d36c609dcedfb

data/CHANGELOG.md

@@ -0,0 +1,25 @@
+# Changelog
+
+## 6.0.0-beta.5 - 2021-04-02
+
+- Added a configuration option named `use_follower_reads_for_type_introspection`.
+  If true, it improves the speed of type introspection by allowing potentially stale
+  type metadata to be read. Defaults to false.
+
+## 6.0.0-beta.4 - 2021-03-06
+
+- Improved connection performance by refactoring an introspection
+  that loads types.
+- Changed version numbers to semver.
+
+## 6.0.0beta3
+
+- Added support for spatial features.
+
+## 6.0.0beta2
+
+- Updated transaction retry logic to work with Rails 6.
+
+## 6.0.0beta1
+
+- Initial support for Rails 6.

data/README.md

@@ -22,6 +22,12 @@ development:
   user: <username>
 ```
 
+## Configuration
+
+In addition to the standard adapter settings, CockroachDB also supports the following:
+
+- `use_follower_reads_for_type_introspection`: Use follower reads on queries to the `pg_type` catalog when set to `true`. This helps to speed up initialization by reading historical data, but may not find recently created user-defined types.
+
 ## Working with Spatial Data
 
 The adapter uses [RGeo](https://github.com/rgeo/rgeo) and [RGeo-ActiveRecord](https://github.com/rgeo/rgeo-activerecord) to represent geometric and geographic data as Ruby objects and easily interface them with the adapter. The following is a brief introduction to RGeo and tips to help setup your spatial application. More documentation about RGeo can be found in the [YARD Docs](https://rubydoc.info/github/rgeo/rgeo) and [wiki](https://github.com/rgeo/rgeo/wiki).

data/activerecord-cockroachdb-adapter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "activerecord-cockroachdb-adapter"
-  spec.version       = "5.2.2"
+  spec.version       = "6.0.0-beta.5"
   spec.licenses      = ["Apache-2.0"]
   spec.authors       = ["Cockroach Labs"]
   spec.email         = ["cockroach-db@googlegroups.com"]
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.description   = "Allows the use of CockroachDB as a backend for ActiveRecord and Rails apps."
   spec.homepage      = "https://github.com/cockroachdb/activerecord-cockroachdb-adapter"
 
-  spec.add_dependency "activerecord", "~> 5.2"
+  spec.add_dependency "activerecord", "~> 6.0.3"
   spec.add_dependency "pg", ">= 0.20"
   spec.add_dependency "rgeo-activerecord", "~> 7.0.0"
 

data/lib/active_record/connection_adapters/cockroachdb/column.rb

@@ -4,19 +4,18 @@ module ActiveRecord
       module PostgreSQLColumnMonkeyPatch
         # most functions taken from activerecord-postgis-adapter spatial_column
         # https://github.com/rgeo/activerecord-postgis-adapter/blob/master/lib/active_record/connection_adapters/postgis/spatial_column.rb
-        def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil,
-                       default_function = nil, collation = nil, comment = nil, cast_type = nil, opts = nil)
+        def initialize(name, default, sql_type_metadata = nil, null = true,
+                       default_function = nil, collation: nil, comment: nil,
+                       serial: nil, spatial: nil)
           @sql_type_metadata = sql_type_metadata
           @geographic = !!(sql_type_metadata.sql_type =~ /geography\(/i)
 
-          @cast_type = cast_type
-          @geographic = !!(sql_type_metadata.sql_type =~ /geography\(/i)
-          if opts
+          if spatial
             # This case comes from an entry in the geometry_columns table
-            set_geometric_type_from_name(opts[:type])
-            @srid = opts[:srid].to_i
-            @has_z = !!opts[:has_z]
-            @has_m = !!opts[:has_m]
+            set_geometric_type_from_name(spatial[:type])
+            @srid = spatial[:srid].to_i
+            @has_z = !!spatial[:has_z]
+            @has_m = !!spatial[:has_m]
           elsif @geographic
             # Geographic type information is embedded in the SQL type
             @srid = 4326
@@ -29,14 +28,13 @@ module ActiveRecord
             # @geometric_type = geo_type_from_sql_type(sql_type)
             build_from_sql_type(sql_type_metadata.sql_type)
           end
-          super(name, default, sql_type_metadata, null, table_name, default_function, collation, comment: comment)
-          if spatial?
-            if @srid
-              @limit = { srid: @srid, type: to_type_name(geometric_type) }
-              @limit[:has_z] = true if @has_z
-              @limit[:has_m] = true if @has_m
-              @limit[:geographic] = true if @geographic
-            end
+          super(name, default, sql_type_metadata, null, default_function,
+            collation: collation, comment: comment, serial: serial)
+          if spatial? && @srid
+            @limit = { srid: @srid, type: to_type_name(geometric_type) }
+            @limit[:has_z] = true if @has_z
+            @limit[:has_m] = true if @has_m
+            @limit[:geographic] = true if @geographic
           end
         end
 

data/lib/active_record/connection_adapters/cockroachdb/oid/type_map_initializer.rb

@@ -0,0 +1,26 @@
+module ActiveRecord
+  module ConnectionAdapters
+    module CockroachDB
+      module OID
+        module TypeMapInitializer
+          # override
+          # Replaces the query with a faster version that doesn't rely on the
+          # use of 'array_in(cstring,oid,integer)'::regprocedure.
+          def query_conditions_for_initial_load
+            known_type_names = @store.keys.map { |n| "'#{n}'" }
+            known_type_types = %w('r' 'e' 'd')
+            <<~SQL % [known_type_names.join(", "), known_type_types.join(", ")]
+              WHERE
+                t.typname IN (%s)
+                OR t.typtype IN (%s)
+                OR (t.typarray = 0 AND t.typcategory='A')
+                OR t.typelem != 0
+            SQL
+          end
+        end
+
+        PostgreSQL::OID::TypeMapInitializer.prepend(TypeMapInitializer)
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/cockroachdb/quoting.rb

@@ -20,12 +20,11 @@ module ActiveRecord
         # but when creating objects, using RGeo features is more convenient than
         # converting to WKB, so this does it automatically.
         def _quote(value)
-          case value
-          when Numeric
+          if value.is_a?(Numeric)
             "'#{quote_string(value.to_s)}'"
-          when RGeo::Feature::Geometry
+          elsif RGeo::Feature::Geometry.check_type(value)
             "'#{RGeo::WKRep::WKBGenerator.new(hex_format: true, type_format: :ewkb, emit_ewkb_srid: true).generate(value)}'"
-          when RGeo::Cartesian::BoundingBox
+          elsif value.is_a?(RGeo::Cartesian::BoundingBox)
             "'#{value.min_x},#{value.min_y},#{value.max_x},#{value.max_y}'::box"
           else
             super

data/lib/active_record/connection_adapters/cockroachdb/schema_statements.rb

@@ -39,41 +39,32 @@ module ActiveRecord
           nil
         end
 
-        def columns(table_name)
-          # Limit, precision, and scale are all handled by the superclass.
-          column_definitions(table_name).map do |column_name, type, default, notnull, oid, fmod, collation, comment|
-            oid = oid.to_i
-            fmod = fmod.to_i
-            type_metadata = fetch_type_metadata(column_name, type, oid, fmod)
-            cast_type = get_oid_type(oid.to_i, fmod.to_i, column_name, type)
-            default_value = extract_value_from_default(default)
-
-            default_function = extract_default_function(default_value, default)
-            new_column(table_name, column_name, default_value, cast_type, type_metadata, !notnull,
-                       default_function, collation, comment)
-          end
-        end
+        # override
+        # https://github.com/rails/rails/blob/6-0-stable/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb#L624
+        def new_column_from_field(table_name, field)
+          column_name, type, default, notnull, oid, fmod, collation, comment = field
+          type_metadata = fetch_type_metadata(column_name, type, oid.to_i, fmod.to_i)
+          default_value = extract_value_from_default(default)
+          default_function = extract_default_function(default_value, default)
 
-        def new_column(table_name, column_name, default, cast_type, sql_type_metadata = nil,
-                        null = true, default_function = nil, collation = nil, comment = nil)
-          # JDBC gets true/false in Rails 4, where other platforms get 't'/'f' strings.
-          if null.is_a?(String)
-            null = (null == "t")
-          end
+          serial =
+            if (match = default_function&.match(/\Anextval\('"?(?<sequence_name>.+_(?<suffix>seq\d*))"?'::regclass\)\z/))
+              sequence_name_from_parts(table_name, column_name, match[:suffix]) == match[:sequence_name]
+            end
 
-          column_info = spatial_column_info(table_name).get(column_name, sql_type_metadata.sql_type)
+          # {:dimension=>2, :has_m=>false, :has_z=>false, :name=>"latlon", :srid=>0, :type=>"GEOMETRY"}
+          spatial = spatial_column_info(table_name).get(column_name, type_metadata.sql_type)
 
-          PostgreSQLColumn.new(
+          PostgreSQL::Column.new(
             column_name,
-            default,
-            sql_type_metadata,
-            null,
-            table_name,
+            default_value,
+            type_metadata,
+            !notnull,
             default_function,
-            collation,
-            comment,
-            cast_type,
-            column_info
+            collation: collation,
+            comment: comment.presence,
+            serial: serial,
+            spatial: spatial
           )
         end
 
@@ -103,11 +94,36 @@ module ActiveRecord
             end
           # The call to super might have appeneded [] already.
           if array && type != :primary_key && !sql.end_with?("[]")
-            sql = "#{sql}[]" 
+            sql = "#{sql}[]"
           end
           sql
         end
 
+        # This overrides the method from PostegreSQL adapter
+        # Resets the sequence of a table's primary key to the maximum value.
+        def reset_pk_sequence!(table, pk = nil, sequence = nil)
+          unless pk && sequence
+            default_pk, default_sequence = pk_and_sequence_for(table)
+
+            pk ||= default_pk
+            sequence ||= default_sequence
+          end
+
+          if @logger && pk && !sequence
+            @logger.warn "#{table} has primary key #{pk} with no default sequence."
+          end
+
+          if pk && sequence
+            quoted_sequence = quote_table_name(sequence)
+            max_pk = query_value("SELECT MAX(#{quote_column_name pk}) FROM #{quote_table_name(table)}", "SCHEMA")
+            if max_pk.nil?
+              minvalue = query_value("SELECT seqmin FROM pg_sequence WHERE seqrelid = #{quote(quoted_sequence)}::regclass", "SCHEMA")
+            end
+
+            query_value("SELECT setval(#{quote(quoted_sequence)}, #{max_pk ? max_pk : minvalue}, #{max_pk ? true : false})", "SCHEMA")
+          end
+        end
+
         # override
         def native_database_types
           # Add spatial types
@@ -127,7 +143,7 @@ module ActiveRecord
 
         # override
         def create_table_definition(*args, **kwargs)
-          CockroachDB::TableDefinition.new(*args, **kwargs)
+          CockroachDB::TableDefinition.new(self, *args, **kwargs)
         end
 
         # memoize hash of column infos for tables

data/lib/active_record/connection_adapters/cockroachdb/transaction_manager.rb

@@ -8,16 +8,22 @@ module ActiveRecord
         # transactions that fail due to serialization errors. Failed
         # transactions will be retried until they pass or the max retry limit is
         # exceeded.
-        def within_new_transaction(options = {})
-          attempts = options.fetch(:attempts, 0)
+        def within_new_transaction(isolation: nil, joinable: true, attempts: 0)
           super
-        rescue ActiveRecord::SerializationFailure => error
+        rescue ActiveRecord::StatementInvalid => error
+          raise unless retryable? error
           raise if attempts >= @connection.max_transaction_retries
 
           attempts += 1
           sleep_seconds = (2 ** attempts + rand) / 10
           sleep(sleep_seconds)
-          within_new_transaction(options.merge(attempts: attempts)) { yield }
+          within_new_transaction(isolation: isolation, joinable: joinable, attempts: attempts) { yield }
+        end
+
+        def retryable?(error)
+          return true if error.is_a? ActiveRecord::SerializationFailure
+          return retryable? error.cause if error.cause
+          false
         end
       end
     end

data/lib/active_record/connection_adapters/cockroachdb_adapter.rb

@@ -13,6 +13,7 @@ require "active_record/connection_adapters/cockroachdb/attribute_methods"
 require "active_record/connection_adapters/cockroachdb/column"
 require "active_record/connection_adapters/cockroachdb/spatial_column_info"
 require "active_record/connection_adapters/cockroachdb/setup"
+require "active_record/connection_adapters/cockroachdb/oid/type_map_initializer"
 require "active_record/connection_adapters/cockroachdb/oid/spatial"
 require "active_record/connection_adapters/cockroachdb/arel_tosql"
 
@@ -36,10 +37,14 @@ module ActiveRecord
       valid_conn_param_keys = PG::Connection.conndefaults_hash.keys + [:sslmode, :application_name]
       conn_params.slice!(*valid_conn_param_keys)
 
-      # The postgres drivers don't allow the creation of an unconnected
-      # PG::Connection object, so just pass a nil connection object for the
-      # time being.
-      ConnectionAdapters::CockroachDBAdapter.new(nil, logger, conn_params, config)
+      conn = PG.connect(conn_params)
+      ConnectionAdapters::CockroachDBAdapter.new(conn, logger, conn_params, config)
+    rescue ::PG::Error, ActiveRecord::ActiveRecordError  => error
+      if error.message.include?("does not exist")
+        raise ActiveRecord::NoDatabaseError
+      else
+        raise
+      end
     end
   end
 end
@@ -75,9 +80,11 @@ module ActiveRecord
       # override
       # This method makes a sql query to gather information about columns
       # in a table. It returns an array of arrays (one for each col) and
-      # is mapped to columns in the SchemaStatements#columns method.
+      # passes each to the SchemaStatements#new_column_from_field method
+      # as the field parameter. This data is then used to format the column
+      # objects for the model and sent to the OID for data casting.
       #
-      # The issue with the default method is that the sql_type field is
+      # The issue with the default method is that the sql_type field is 
       # retrieved with the `format_type` function, but this is implemented
       # differently in CockroachDB than PostGIS, so geometry/geography
       # types are missing information which makes parsing them impossible.
@@ -90,7 +97,7 @@ module ActiveRecord
       #
       # The solution is to make the default query with super, then
       # iterate through the columns and if it is a spatial type,
-      # access the proper column_type with the information_schema.columns
+      # access the proper column_type with the information_schema.columns 
       # table.
       #
       # @see: https://github.com/rails/rails/blob/8695b028261bdd244e254993255c6641bdbc17a5/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L829
@@ -171,11 +178,6 @@ module ActiveRecord
         false
       end
 
-      def supports_ranges?
-        # See cockroachdb/cockroach#17022
-        false
-      end
-
       def supports_materialized_views?
         false
       end
@@ -216,6 +218,10 @@ module ActiveRecord
         @crdb_version >= 202
       end
 
+      def supports_partitioned_indexes?
+        false
+      end
+
       # This is hardcoded to 63 (as previously was in ActiveRecord 5.0) to aid in
       # migration from PostgreSQL to CockroachDB. In practice, this limitation
       # is arbitrary since CockroachDB supports index name lengths and table alias
@@ -249,7 +255,13 @@ module ActiveRecord
         end
         @crdb_version = version_num
       end
-        
+
+      def self.database_exists?(config)
+        !!ActiveRecord::Base.cockroachdb_connection(config)
+      rescue ActiveRecord::NoDatabaseError
+        false
+      end
+
       private
 
         def initialize_type_map(m = type_map)
@@ -391,6 +403,100 @@ module ActiveRecord
           return "{}"
         end
 
+        # override
+        # This method loads info about data types from the database to
+        # populate the TypeMap.
+        #
+        # Currently, querying from the pg_type catalog can be slow due to geo-partitioning
+        # so this modified query uses AS OF SYSTEM TIME '-10s' to read historical data.
+        def load_additional_types(oids = nil)
+          if @config[:use_follower_reads_for_type_introspection]
+            initializer = OID::TypeMapInitializer.new(type_map)
+
+            query = <<~SQL
+              SELECT t.oid, t.typname, t.typelem, t.typdelim, t.typinput, r.rngsubtype, t.typtype, t.typbasetype
+              FROM pg_type as t
+              LEFT JOIN pg_range as r ON oid = rngtypid AS OF SYSTEM TIME '-10s'
+            SQL
+
+            if oids
+              query += "WHERE t.oid IN (%s)" % oids.join(", ")
+            else
+              query += initializer.query_conditions_for_initial_load
+            end
+
+            execute_and_clear(query, "SCHEMA", []) do |records|
+              initializer.run(records)
+            end
+          else
+            super
+          end
+        rescue ActiveRecord::StatementInvalid => e
+          raise e unless e.cause.is_a? PG::InvalidCatalogName
+          # use original if database is younger than 10s
+          super
+        end
+
+        # override
+        # This method maps data types to their proper decoder.
+        #
+        # Currently, querying from the pg_type catalog can be slow due to geo-partitioning
+        # so this modified query uses AS OF SYSTEM TIME '-10s' to read historical data.
+        def add_pg_decoders
+          if @config[:use_follower_reads_for_type_introspection]
+            @default_timezone = nil
+            @timestamp_decoder = nil
+
+            coders_by_name = {
+              "int2" => PG::TextDecoder::Integer,
+              "int4" => PG::TextDecoder::Integer,
+              "int8" => PG::TextDecoder::Integer,
+              "oid" => PG::TextDecoder::Integer,
+              "float4" => PG::TextDecoder::Float,
+              "float8" => PG::TextDecoder::Float,
+              "numeric" => PG::TextDecoder::Numeric,
+              "bool" => PG::TextDecoder::Boolean,
+              "timestamp" => PG::TextDecoder::TimestampUtc,
+              "timestamptz" => PG::TextDecoder::TimestampWithTimeZone,
+            }
+
+            known_coder_types = coders_by_name.keys.map { |n| quote(n) }
+            query = <<~SQL % known_coder_types.join(", ")
+              SELECT t.oid, t.typname
+              FROM pg_type as t AS OF SYSTEM TIME '-10s'
+              WHERE t.typname IN (%s)
+            SQL
+
+            coders = execute_and_clear(query, "SCHEMA", []) do |result|
+              result
+                .map { |row| construct_coder(row, coders_by_name[row["typname"]]) }
+                .compact
+            end
+
+            map = PG::TypeMapByOid.new
+            coders.each { |coder| map.add_coder(coder) }
+            @connection.type_map_for_results = map
+
+            @type_map_for_results = PG::TypeMapByOid.new
+            @type_map_for_results.default_type_map = map
+            @type_map_for_results.add_coder(PG::TextDecoder::Bytea.new(oid: 17, name: "bytea"))
+
+            # extract timestamp decoder for use in update_typemap_for_default_timezone
+            @timestamp_decoder = coders.find { |coder| coder.name == "timestamp" }
+            update_typemap_for_default_timezone
+          else
+            super
+          end
+        rescue ActiveRecord::StatementInvalid => e
+          raise e unless e.cause.is_a? PG::InvalidCatalogName
+          # use original if database is younger than 10s
+          super
+        end
+
+        def arel_visitor
+          Arel::Visitors::CockroachDB.new(self)
+        end
+
       # end private
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-cockroachdb-adapter
 version: !ruby/object:Gem::Version
-  version: 5.2.2
+  version: 6.0.0.pre.beta.5
 platform: ruby
 authors:
 - Cockroach Labs
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-13 00:00:00.000000000 Z
+date: 2021-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: 6.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: 6.0.3
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -62,6 +62,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".gitmodules"
+- CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE
@@ -82,6 +83,7 @@ files:
 - lib/active_record/connection_adapters/cockroachdb/database_statements.rb
 - lib/active_record/connection_adapters/cockroachdb/database_tasks.rb
 - lib/active_record/connection_adapters/cockroachdb/oid/spatial.rb
+- lib/active_record/connection_adapters/cockroachdb/oid/type_map_initializer.rb
 - lib/active_record/connection_adapters/cockroachdb/quoting.rb
 - lib/active_record/connection_adapters/cockroachdb/referential_integrity.rb
 - lib/active_record/connection_adapters/cockroachdb/schema_statements.rb
@@ -97,7 +99,7 @@ licenses:
 - Apache-2.0
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -108,12 +110,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: CockroachDB adapter for ActiveRecord.
 test_files: []