activerecord-spanner-adapter 0.3.0 → 1.0.0

data/lib/active_record/connection_adapters/spanner/type_metadata.rb

@@ -0,0 +1,37 @@
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Spanner
+      class TypeMetadata < DelegateClass(SqlTypeMetadata)
+        undef to_yaml if method_defined? :to_yaml
+
+        attr_reader :ordinal_position
+
+        def initialize type_metadata, ordinal_position: nil
+          super type_metadata
+          @ordinal_position = ordinal_position
+        end
+
+        def == other
+          other.is_a?(TypeMetadata) &&
+            __getobj__ == other.__getobj__ &&
+            ordinal_position == other.ordinal_position
+        end
+        alias eql? ==
+
+        def hash
+          TypeMetadata.hash ^
+            __getobj__.hash ^
+            ordinal_position.hash
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/spanner_adapter.rb

@@ -1,126 +1,233 @@
-require 'google/cloud/spanner'
-
-require 'active_record/connection_adapters/abstract_adapter'
-require 'active_record/connection_adapters/spanner/client'
-require 'active_record/connection_adapters/spanner/database_statements'
-require 'active_record/connection_adapters/spanner/schema_creation'
-require 'active_record/connection_adapters/spanner/schema_statements'
-require 'active_record/connection_adapters/spanner/quoting'
-require 'grpc'
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+require "securerandom"
+require "google/cloud/spanner"
+require "spanner_client_ext"
+require "active_record/connection_adapters/abstract_adapter"
+require "active_record/connection_adapters/spanner/database_statements"
+require "active_record/connection_adapters/spanner/schema_statements"
+require "active_record/connection_adapters/spanner/schema_cache"
+require "active_record/connection_adapters/spanner/schema_definitions"
+require "active_record/connection_adapters/spanner/type_metadata"
+require "active_record/connection_adapters/spanner/quoting"
+require "active_record/type/spanner/array"
+require "active_record/type/spanner/bytes"
+require "active_record/type/spanner/spanner_active_record_converter"
+require "active_record/type/spanner/time"
+require "arel/visitors/spanner"
+require "activerecord_spanner_adapter/base"
+require "activerecord_spanner_adapter/connection"
+require "activerecord_spanner_adapter/errors"
+require "activerecord_spanner_adapter/information_schema"
+require "activerecord_spanner_adapter/primary_key"
+require "activerecord_spanner_adapter/transaction"
 
 module ActiveRecord
-  module ConnectionHandling
-    def spanner_connection(config)
-      ConnectionAdapters::SpannerAdapter.new(nil, logger, config)
+  module ConnectionHandling # :nodoc:
+    def spanner_connection config
+      connection = ActiveRecordSpannerAdapter::Connection.new config
+      connection.connect!
+      ConnectionAdapters::SpannerAdapter.new connection, logger, nil, config
+    rescue Google::Cloud::Error => error
+      if error.instance_of? Google::Cloud::NotFoundError
+        raise ActiveRecord::NoDatabaseError
+      end
+      raise error
     end
   end
 
   module ConnectionAdapters
-    # A Google Cloud Spanner adapter
-    #
-    # Options:
-    # - project
+    module AbstractPool
+      def get_schema_cache connection
+        @schema_cache ||= SpannerSchemaCache.new connection
+        @schema_cache.connection = connection
+        @schema_cache
+      end
+    end
+
     class SpannerAdapter < AbstractAdapter
-      ADAPTER_NAME = 'Spanner'.freeze
-      CLIENT_PARAMS = [:project, :keyfile, :scope, :timeout, :client_config].freeze
-      ADAPTER_OPTS = (CLIENT_PARAMS + [:instance, :database]).freeze
+      ADAPTER_NAME = "spanner".freeze
+      NATIVE_DATABASE_TYPES = {
+        primary_key:  "INT64",
+        parent_key:   "INT64",
+        string:       { name: "STRING", limit: "MAX" },
+        text:         { name: "STRING", limit: "MAX" },
+        integer:      { name: "INT64" },
+        bigint:       { name: "INT64" },
+        float:        { name: "FLOAT64" },
+        decimal:      { name: "NUMERIC" },
+        numeric:      { name: "NUMERIC" },
+        datetime:     { name: "TIMESTAMP" },
+        time:         { name: "TIMESTAMP" },
+        date:         { name: "DATE" },
+        binary:       { name: "BYTES", limit: "MAX" },
+        boolean:      { name: "BOOL" },
+        json:         { name: "JSON" }
+      }.freeze
 
-      include Spanner::SchemaStatements
-      include Spanner::DatabaseStatements
       include Spanner::Quoting
+      include Spanner::DatabaseStatements
+      include Spanner::SchemaStatements
 
-      def initialize(connection, logger, config)
-        conn_params = config.symbolize_keys.slice(*ADAPTER_OPTS)
-        connect(conn_params)
-        super(connection, logger, config)
+      def initialize connection, logger, connection_options, config
+        super connection, logger, config
+        @connection_options = connection_options
       end
 
-      def schema_creation # :nodoc:
-        Spanner::SchemaCreation.new self
+      def max_identifier_length
+        128
       end
 
-      def arel_visitor
-        QueryVisitor.new(self)
+      def native_database_types
+        NATIVE_DATABASE_TYPES
       end
 
-      def active?
-        [
-          ::GRPC::Core::ConnectivityStates::IDLE,
-          ::GRPC::Core::ConnectivityStates::READY,
-        ].include?(@conn.service.channel.connectivity_state)
+      # Database
+
+      def self.database_exists? config
+        connection = ActiveRecordSpannerAdapter::Connection.new config
+        connection.connect!
+        true
+      rescue ActiveRecord::NoDatabaseError
+        false
       end
 
-      def connect(params)
-        client_params = params.slice(*CLIENT_PARAMS)
-        @conn = Google::Cloud::Spanner.new(**client_params)
-        @instance_id = params[:instance]
-        @database_id = params[:database]
+      # Connection management
+
+      def active?
+        @connection.active?
       end
 
       def disconnect!
         super
-        release_client!
+        @connection.disconnect!
+      end
+
+      def reset!
+        super
+        @connection.reset!
+      end
+      alias reconnect! reset!
+
+      # Spanner Connection API
+      delegate :ddl_batch, :ddl_batch?, :start_batch_ddl, :abort_batch, :run_batch, to: :@connection
+
+      def current_spanner_transaction
+        @connection.current_transaction
       end
 
-      def prefetch_primary_key?(table_name = nil)
+      # Supported features
+
+      def supports_bulk_alter?
         true
       end
 
-      def next_sequence_value(table_name = nil)
-        require 'securerandom'
-        SecureRandom.uuid
+      def supports_common_table_expressions?
+        true
       end
 
-      private
-      attr_reader :conn
+      def supports_explain?
+        false
+      end
 
-      def initialize_type_map(m) # :nodoc:
-        register_class_with_limit m, %r(STRING)i, Type::String
-        register_class_with_limit m, %r(BYTES)i, Type::Binary
-        m.register_type %r[STRING(MAX)]i, Type::Text.new(limit: 10 * 1024**2)
-        m.register_type %r[BYTES(MAX)]i, Type::Binary.new(limit: 10 * 1024**2)
-        m.register_type %r[BOOL]i, Type::Boolean.new
-        m.register_type %r[INT64]i, Type::Integer.new(limit: 8)
-        m.register_type %r[FLOAT64]i, Type::Float.new(limit: 53)
-        m.register_type %r[DATE]i, Type::Date.new
-        m.register_type %r[TIMESTAMP]i, Type::DateTime.new
-        # TODO(yugui) Support array and struct
+      def supports_foreign_keys?
+        true
+      end
+
+      def supports_index_sort_order?
+        true
+      end
+
+      def supports_insert_on_conflict?
+        true
+      end
+      alias supports_insert_on_duplicate_skip? supports_insert_on_conflict?
+      alias supports_insert_on_duplicate_update? supports_insert_on_conflict?
+      alias supports_insert_conflict_target? supports_insert_on_conflict?
+
+      def supports_insert_returning?
+        true
+      end
+
+      def supports_multi_insert?
+        true
       end
 
-      def instance
-        @instance ||= conn.instance(@instance_id)
-        raise ActiveRecord::NoDatabaseError unless @instance
+      def supports_optimizer_hints?
+        true
+      end
 
-        @instance
+      def supports_primary_key?
+        true
       end
 
-      def database
-        return @db if @db
+      def prefetch_primary_key? _table_name = nil
+        true
+      end
 
-        @db = instance.database(@database_id)
-        raise ActiveRecord::NoDatabaseError unless @db
-        raise ActiveRecord::ConnectionNotEstablished,
-          "database #{@db.database_path} is not ready" unless @db.ready?
+      # Generate next sequence number for primary key
+      def next_sequence_value _sequence_name
+        SecureRandom.uuid.gsub("-", "").hex & 0x7FFFFFFFFFFFFFFF
+      end
 
-        @db
+      def arel_visitor
+        Arel::Visitors::Spanner.new self
       end
 
-      def raw_client
-        @raw_client ||= conn.client(@instance_id, @database_id)
+      private
+
+      def initialize_type_map m = type_map
+        m.register_type "BOOL", Type::Boolean.new
+        register_class_with_limit(
+          m, %r{^BYTES}i, ActiveRecord::Type::Spanner::Bytes
+        )
+        m.register_type "DATE", Type::Date.new
+        m.register_type "FLOAT64", Type::Float.new
+        m.register_type "NUMERIC", Type::Decimal.new
+        m.register_type "INT64", Type::Integer.new(limit: 8)
+        register_class_with_limit m, %r{^STRING}i, Type::String
+        m.register_type "TIMESTAMP", ActiveRecord::Type::Spanner::Time.new
+        m.register_type "JSON", ActiveRecord::Type::Json.new
+
+        register_array_types m
       end
 
-      def client
-        @client ||= ConnectionAdapters::Spanner::InitialPhaseClient.new(raw_client)
+      def register_array_types m
+        m.register_type %r{^ARRAY<BOOL>}i, Type::Spanner::Array.new(Type::Boolean.new)
+        m.register_type %r{^ARRAY<BYTES\((MAX|d+)\)>}i, Type::Spanner::Array.new(Type::Binary.new)
+        m.register_type %r{^ARRAY<DATE>}i, Type::Spanner::Array.new(Type::Date.new)
+        m.register_type %r{^ARRAY<FLOAT64>}i, Type::Spanner::Array.new(Type::Float.new)
+        m.register_type %r{^ARRAY<NUMERIC>}i, Type::Spanner::Array.new(Type::Decimal.new)
+        m.register_type %r{^ARRAY<INT64>}i, Type::Spanner::Array.new(Type::Integer.new(limit: 8))
+        m.register_type %r{^ARRAY<STRING\((MAX|d+)\)>}i, Type::Spanner::Array.new(Type::String.new)
+        m.register_type %r{^ARRAY<TIMESTAMP>}i, Type::Spanner::Array.new(ActiveRecord::Type::Spanner::Time.new)
+        m.register_type %r{^ARRAY<JSON>}i, Type::Spanner::Array.new(ActiveRecord::Type::Json.new)
       end
 
-      def release_client!
-        @client&.close
-        @client = nil
+      def extract_limit sql_type
+        value = /\((.*)\)/.match sql_type
+        return unless value
+
+        value[1] == "MAX" ? "MAX" : value[1].to_i
       end
 
-      #def reset_transaction
-      #  # @transaction_manager = Spanner::TransactionManager.new(self, client)
-      #end
+      def translate_exception exception, message:, sql:, binds:
+        if exception.is_a? Google::Cloud::FailedPreconditionError
+          case exception.message
+          when /.*does not specify a non-null value for these NOT NULL columns.*/,
+               /.*must not be NULL.*/
+            NotNullViolation.new message, sql: sql, binds: binds
+          else
+            super
+          end
+        else
+          super
+        end
+      end
     end
   end
 end

data/lib/active_record/tasks/spanner_database_tasks.rb

@@ -0,0 +1,74 @@
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+require "active_record/connection_adapters/spanner_adapter"
+
+module ActiveRecord
+  module Tasks
+    class SpannerDatabaseTasks
+      def initialize config
+        config = config.symbolize_keys
+        @connection = ActiveRecordSpannerAdapter::Connection.new config
+      end
+
+      def create
+        @connection.create_database
+      rescue Google::Cloud::Error => error
+        if error.instance_of? Google::Cloud::AlreadyExistsError
+          raise ActiveRecord::DatabaseAlreadyExists
+        end
+
+        raise error
+      end
+
+      def drop
+        @connection.database.drop
+      end
+
+      def purge
+        drop
+        create
+      end
+
+      def charset
+        nil
+      end
+
+      def collation
+        nil
+      end
+
+      def structure_dump filename, _extra_flags
+        file = File.open filename, "w"
+        ignore_tables = ActiveRecord::SchemaDumper.ignore_tables
+
+        if ignore_tables.any?
+          index_regx = /^CREATE(.*)INDEX(.*)ON (#{ignore_tables.join "|"})\(/
+          table_regx = /^CREATE TABLE (#{ignore_tables.join "|"})/
+        end
+
+        @connection.database.ddl(force: true).each do |statement|
+          next if ignore_tables.any? &&
+                  (table_regx =~ statement || index_regx =~ statement)
+          file.write statement
+          file.write "\n"
+        end
+      ensure
+        file.close
+      end
+
+      def structure_load filename, _extra_flags
+        statements = File.read(filename).split(/(?=^CREATE)/)
+        @connection.execute_ddl statements
+      end
+    end
+
+    DatabaseTasks.register_task(
+      /spanner/,
+      "ActiveRecord::Tasks::SpannerDatabaseTasks"
+    )
+  end
+end

data/lib/active_record/type/spanner/array.rb

@@ -0,0 +1,32 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+module ActiveRecord
+  module Type
+    module Spanner
+      class Array < Type::Value
+        attr_reader :element_type
+        delegate :type, :user_input_in_time_zone, :limit, :precision, :scale, to: :element_type
+
+        def initialize element_type
+          @element_type = element_type
+        end
+
+        def serialize value
+          return super if value.nil?
+          return super unless @element_type.is_a? Type::Decimal
+          return super unless value.respond_to? :map
+
+          # Convert a decimal (NUMERIC) array to a String array to prevent it from being encoded as a FLOAT64 array.
+          value.map do |v|
+            next if v.nil?
+            v.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/type/spanner/bytes.rb

@@ -0,0 +1,26 @@
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Type
+    module Spanner
+      class Bytes < ActiveRecord::Type::Binary
+        def serialize value
+          return super value if value.nil?
+
+          if value.respond_to?(:read) && value.respond_to?(:rewind)
+            value.rewind
+            value = value.read
+          end
+
+          Base64.strict_encode64 value.force_encoding("ASCII-8BIT")
+        end
+      end
+    end
+  end
+end

data/lib/active_record/type/spanner/spanner_active_record_converter.rb

@@ -0,0 +1,33 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Type
+    module Spanner
+      class SpannerActiveRecordConverter
+        ##
+        # Converts an ActiveModel::Type to a Spanner type code.
+        def self.convert_active_model_type_to_spanner type # rubocop:disable Metrics/CyclomaticComplexity
+          case type
+          when NilClass then nil
+          when ActiveModel::Type::Integer, ActiveModel::Type::BigInteger then :INT64
+          when ActiveModel::Type::Boolean then :BOOL
+          when ActiveModel::Type::String, ActiveModel::Type::ImmutableString then :STRING
+          when ActiveModel::Type::Binary, ActiveRecord::Type::Spanner::Bytes then :BYTES
+          when ActiveModel::Type::Float then :FLOAT64
+          when ActiveModel::Type::Decimal then :NUMERIC
+          when ActiveModel::Type::DateTime, ActiveModel::Type::Time, ActiveRecord::Type::Spanner::Time then :TIMESTAMP
+          when ActiveModel::Type::Date then :DATE
+          when ActiveRecord::Type::Json then :JSON
+          when ActiveRecord::Type::Spanner::Array then [convert_active_model_type_to_spanner(type.element_type)]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/type/spanner/time.rb

@@ -0,0 +1,37 @@
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Type
+    module Spanner
+      class Time < ActiveRecord::Type::Time
+        def serialize value, *options
+          return "PENDING_COMMIT_TIMESTAMP()" if value == :commit_timestamp && options.length && options[0] == :dml
+          return "spanner.commit_timestamp()" if value == :commit_timestamp && options.length && options[0] == :mutation
+          val = super value
+          val.acts_like?(:time) ? val.utc.rfc3339(9) : val
+        end
+
+        def user_input_in_time_zone value
+          return value.in_time_zone if value.is_a? ::Time
+          super value
+        end
+
+        private
+
+        def cast_value value
+          if value.is_a? ::String
+            value = value.empty? ? nil : ::Time.parse(value)
+          end
+
+          value
+        end
+      end
+    end
+  end
+end

data/lib/activerecord-spanner-adapter.rb

@@ -0,0 +1,23 @@
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+require "activerecord_spanner_adapter/version"
+
+if defined?(Rails)
+  module ActiveRecord
+    module ConnectionAdapters
+      class SpannerRailtie < ::Rails::Railtie
+        rake_tasks do
+          require "active_record/tasks/spanner_database_tasks"
+        end
+
+        ActiveSupport.on_load :active_record do
+          require "active_record/connection_adapters/spanner_adapter"
+        end
+      end
+    end
+  end
+end