activerecord-spanner-adapter 0.3.0 → 1.0.0

data/acceptance/test_helper.rb

@@ -0,0 +1,261 @@
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+gem "minitest"
+require "minitest/autorun"
+require "minitest/focus"
+require "minitest/rg"
+require "active_support"
+require "google/cloud/spanner"
+require "active_record"
+require "active_support/testing/stream"
+require "activerecord-spanner-adapter"
+require "active_record/connection_adapters/spanner_adapter"
+require "securerandom"
+
+# rubocop:disable Style/GlobalVars
+
+$spanner_test_database = "ar-test-#{SecureRandom.hex 4}"
+
+def connector_config
+  {
+    "adapter" => "spanner",
+    "emulator_host" => ENV["SPANNER_EMULATOR_HOST"],
+    "project" => ENV["SPANNER_TEST_PROJECT"],
+    "instance" => ENV["SPANNER_TEST_INSTANCE"],
+    "credentials" => ENV["SPANNER_TEST_KEYFILE"],
+    "database" => $spanner_test_database
+  }
+end
+
+def spanner
+  $spanner ||= Google::Cloud::Spanner.new(
+    project_id: ENV["SPANNER_TEST_PROJECT"],
+    credentials: ENV["SPANNER_TEST_KEYFILE"]
+  )
+end
+
+def spanner_instance
+  unless spanner.instance ENV["SPANNER_TEST_INSTANCE"]
+    config = ENV["SPANNER_EMULATOR_HOST"] ? "emulator-config" : "regional-us-central1"
+    puts "Creating test instance #{ENV["SPANNER_TEST_INSTANCE"]} with config #{config}"
+    job = spanner.create_instance ENV["SPANNER_TEST_INSTANCE"],
+                                  name:   "ActiveRecord Test Instance",
+                                  config: config,
+                                  nodes:  1
+    job.wait_until_done!
+    $owned_test_instance = true
+  end
+  $spanner_instance ||= spanner.instance ENV["SPANNER_TEST_INSTANCE"]
+end
+
+def create_test_database
+  job = spanner_instance.create_database $spanner_test_database
+  job.wait_until_done!
+  if job.error?
+    raise "Error in creating database. Error code#{job.error.message}"
+  end
+
+  puts "'#{$spanner_test_database}' test db created."
+
+  puts "Loading test schema..."
+  ActiveRecord::Base.establish_connection connector_config
+  require_relative "schema/schema"
+end
+
+def drop_test_database
+  ActiveRecord::Base.connection_pool.disconnect!
+  spanner_instance&.delete if $owned_test_instance
+  spanner_instance.database($spanner_test_database)&.drop unless $owned_test_instance
+
+  puts "Test instance #{spanner_instance} deleted" if $owned_test_instance
+  puts "#{$spanner_test_database} database deleted" unless $owned_test_instance
+end
+
+def current_adapter? *names
+  names.include? :SpannerAdapter
+end
+
+def load_test_schema
+  ActiveRecord::Base.establish_connection connector_config
+
+  require_relative "schema/schema"
+end
+
+module SpannerAdapter
+  class TestCase < ActiveSupport::TestCase
+    def assert_column(model, column_name, msg = nil)
+      assert has_column?(model, column_name), msg
+    end
+
+    def assert_no_column(model, column_name, msg = nil)
+      assert_not has_column?(model, column_name), msg
+    end
+
+    def has_column?(model, column_name)
+      model.reset_column_information
+      model.column_names.include?(column_name.to_s)
+    end
+
+    def capture_sql
+      ActiveRecord::Base.connection.materialize_transactions
+      SQLCounter.clear_log
+      yield
+      SQLCounter.log.dup
+    end
+
+    def assert_queries(num = 1, options = {})
+      ignore_none = options.fetch(:ignore_none) { num == :any }
+      ActiveRecord::Base.connection.materialize_transactions
+      SQLCounter.clear_log
+      x = yield
+      the_log = ignore_none ? SQLCounter.log_all : SQLCounter.log
+      if num == :any
+        assert_operator the_log.size, :>=, 1, "1 or more queries expected, but none were executed."
+      else
+        mesg = "#{the_log.size} instead of #{num} queries were executed.#{the_log.size == 0 ? '' : "\nQueries:\n#{the_log.join("\n")}"}"
+        assert_equal num, the_log.size, mesg
+      end
+      x
+    end
+
+    def assert_no_queries(options = {}, &block)
+      options.reverse_merge! ignore_none: true
+      assert_queries(0, options, &block)
+    end
+  end
+
+  module Migration
+    module TestHelper
+      attr_accessor :connection
+
+      CONNECTION_METHODS = %w[
+        add_column remove_column rename_column add_index change_column
+        rename_table column_exists? index_exists?
+        add_reference add_belongs_to remove_reference remove_references
+        remove_belongs_to change_column_default
+      ].freeze
+
+      class TestModel < ActiveRecord::Base
+        self.table_name = :test_models
+      end
+
+      def setup
+        ActiveRecord::Base.establish_connection connector_config
+        @connection = ActiveRecord::Base.connection
+
+        unless @skip_test_table_create
+          connection.create_table :test_models do |t|
+            t.timestamps null: true
+          end
+
+          TestModel.reset_column_information
+        end
+        super
+      end
+
+      def skip_test_table_create!
+        @skip_test_table_create = true
+      end
+
+      def teardown
+        TestModel.reset_table_name
+
+        unless @skip_test_table_create
+          connection.drop_table :test_models, if_exists: true
+        end
+
+        super
+      end
+
+      def generate_id
+        connection.next_sequence_value nil
+      end
+
+      delegate *CONNECTION_METHODS, to: :connection
+    end
+  end
+
+  module Types
+    module TestHelper
+      attr_accessor :connection
+
+      class TestTypeModel < ActiveRecord::Base
+        self.table_name = :test_types
+      end
+
+      def setup
+        super
+
+        ActiveRecord::Base.establish_connection connector_config
+        @connection = ActiveRecord::Base.connection
+
+        return if connection.table_exists? :test_types
+
+        connection.create_table :test_types do |t|
+          t.string :name, limit: 255
+          t.string :description
+          t.text :bio
+          t.integer :length
+          t.float :weight
+          t.numeric :price
+          t.boolean :active
+          t.binary :file
+          t.binary :data, limit: 255
+          t.date :start_date
+          t.datetime :start_datetime
+          t.time :start_time
+          t.json :details unless ENV["SPANNER_EMULATOR_HOST"]
+        end
+      end
+
+      def teardown
+        super
+        TestTypeModel.delete_all
+      end
+    end
+  end
+
+  module Associations
+    module TestHelper
+      def setup
+        ActiveRecord::Base.establish_connection connector_config
+        @connection = ActiveRecord::Base.connection
+      end
+    end
+  end
+
+  class SQLCounter
+    class << self
+      attr_accessor :ignored_sql, :log, :log_all
+
+      def clear_log
+        self.log = []
+        self.log_all = []
+      end
+    end
+
+    clear_log
+
+    def call(name, start, finish, message_id, values)
+      return if values[:cached]
+
+      sql = values[:sql]
+      self.class.log_all << sql
+      self.class.log << sql unless ["SCHEMA", "TRANSACTION"].include? values[:name]
+    end
+  end
+
+  ActiveSupport::Notifications.subscribe("sql.active_record", SQLCounter.new)
+end
+
+Minitest.after_run do
+  drop_test_database
+end
+
+create_test_database
+
+# rubocop:enable Style/GlobalVars

data/activerecord-spanner-adapter.gemspec

@@ -1,27 +1,42 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'activerecord-spanner-adapter/version'
+lib = File.expand_path "lib", __dir__
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include? lib
+require "activerecord_spanner_adapter/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "activerecord-spanner-adapter"
   spec.version       = ActiveRecordSpannerAdapter::VERSION
-  spec.authors       = ["Yuki Yugui Sonoda"]
-  spec.email         = ["yuki.sonoda@supership.jp"]
+  spec.authors       = ["Google LLC"]
+  spec.email         = ["cloud-spanner-developers@googlegroups.com"]
 
-  spec.summary       = %q{Adapts Google Cloud Spanner to ActiveRecord}
-  spec.description   = %q{Connection Adapter of Google Cloud Spanner to ActiveRecord O/R mapper library}
-  spec.homepage      = "https://github.com/supership-jp/activerecord-spanner-adapter"
+  spec.summary       = %q{Rails ActiveRecord connector for Google Spanner Database}
+  spec.description   = %q{Rails ActiveRecord connector for Google Spanner Database}
+  spec.homepage      = "https://github.com/googleapis/ruby-spanner-activerecord"
+  spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir File.expand_path(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match %r{^(test|spec|features)/} }
   end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename f }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'activerecord', "~> 5.0"
-  spec.add_dependency 'google-cloud-spanner', "~> 0.23"
-  spec.add_dependency 'google-gax', '~> 0.8'
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.6.0"
+  spec.required_ruby_version = ">= 2.5"
+
+  spec.add_dependency "google-cloud-spanner", "~> 2.10"
+  spec.add_runtime_dependency "activerecord", "~> 6.1.4"
+
+  spec.add_development_dependency "autotest-suffix", "~> 1.1"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "google-style", "~> 1.24.0"
+  spec.add_development_dependency "minitest", "~> 5.10"
+  spec.add_development_dependency "minitest-autotest", "~> 1.0"
+  spec.add_development_dependency "minitest-focus", "~> 1.1"
+  spec.add_development_dependency "minitest-rg", "~> 5.2"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "redcarpet", "~> 3.0"
+  spec.add_development_dependency "simplecov", "~> 0.9"
+  spec.add_development_dependency "yard", "~> 0.9"
+  spec.add_development_dependency "yard-doctest", "~> 0.1.13"
 end

data/benchmarks/README.md

@@ -0,0 +1,17 @@
+# Benchmarks
+
+Benchmarks the Spanner ActiveRecord adapter using a small set of standardized use cases. The benchmarks consists of two
+separate runs:
+* Execute each use case separately and measure the time needed for each use case.
+* Batch all use cases together and execute multiple batches in parallel, measuring the time needed to finish all
+  batches. The number of batches varies between 1 and 400. The session pool is configured to contain at most 400
+  connections.
+
+Change the configuration in the file `config/database.yml` before running the benchmarks. The instance and database in
+the configuration must exist. The tables that are needed will automatically be created by the benchmark script.
+
+Run the benchmark with the command
+
+```bash
+bundle exec rake run
+```

data/benchmarks/Rakefile

@@ -0,0 +1,14 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+require_relative "config/environment"
+require "sinatra/activerecord/rake"
+
+desc "Runs benchmarks against a Cloud Spanner database."
+task :run do |_t, _args|
+  sh "rake db:migrate"
+  sh "ruby application.rb"
+end

data/benchmarks/application.rb

@@ -0,0 +1,308 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+require "benchmark"
+require "io/console"
+require "securerandom"
+require_relative "config/environment"
+require_relative "models/singer"
+require_relative "models/album"
+
+class Application
+  def self.run # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    ActiveRecord::Base.logger.level = Logger::WARN
+    config = ActiveRecord::Base.connection_config
+    spanner = Google::Cloud::Spanner.new project: config[:project], credentials: config[:credentials]
+    spanner_client = spanner.client config[:instance], config[:database], pool: { max: config[:pool], fail: false }
+
+    [nil, spanner_client].each do |client| # rubocop:disable Metrics/BlockLength
+      puts ""
+      puts ""
+      puts "Benchmarks for #{client ? 'Spanner client' : 'ActiveRecord'}"
+      Album.delete_all
+      Singer.delete_all
+
+      # Seed the database with 10,000 random singers.
+      # Having a relatively large number of records in the database prevents the parallel test cases from using the same
+      # records, which would cause many of the transactions to be aborted and retried all the time.
+      singer = nil
+      10.times do
+        singer = create_singers 1000, client
+      end
+
+      execute_individual_benchmarks singer, client
+
+      Benchmark.bm 75 do |bm|
+        [1, 5, 10, 25, 50, 100, 200, 400].each do |parallel_benchmarks|
+          bm.report "Total execution time (#{parallel_benchmarks}):" do
+            threads = []
+            parallel_benchmarks.times do
+              threads << Thread.new do
+                benchmark_select_one_singer singer, client
+                benchmark_select_and_update_using_mutation 1, client
+                benchmark_select_and_update_using_dml 1, client
+                benchmark_create_and_reload client
+                benchmark_create_albums_using_mutations 1, client
+                benchmark_create_albums_using_dml 1, client
+                benchmark_select_100_singers client
+                benchmark_select_100_singers_in_read_only_transaction client
+                benchmark_select_100_singers_in_read_write_transaction client
+              end
+            end
+            threads.each(&:join)
+          end
+        end
+      end
+    end
+
+    spanner_client.close
+    ActiveRecord::Base.connection_pool.disconnect
+
+    puts ""
+    puts "Press any key to end the application"
+    STDIN.getch
+  end
+
+  def self.execute_individual_benchmarks singer, client
+    puts ""
+    Benchmark.bm 75 do |bm| # rubocop:disable Metrics/BlockLength
+      bm.report "Select one row:" do
+        benchmark_select_one_singer singer, client
+      end
+
+      bm.report "Save one row with fetch after:" do
+        benchmark_create_and_reload client
+      end
+
+      bm.report "Select and update 1 row in transaction using mutation:" do
+        benchmark_select_and_update_using_mutation 1, client
+      end
+
+      bm.report "Select and update 1 row in transaction using DML:" do
+        benchmark_select_and_update_using_dml 1, client
+      end
+
+      bm.report "Select and update 25 rows in transaction using mutation:" do
+        benchmark_select_and_update_using_mutation 25, client
+      end
+
+      bm.report "Select and update 25 rows in transaction using DML:" do
+        benchmark_select_and_update_using_dml 25, client
+      end
+
+      bm.report "Create 100 albums using mutations:" do
+        benchmark_create_albums_using_mutations 100, client
+      end
+
+      bm.report "Create 100 albums using DML:" do
+        benchmark_create_albums_using_dml 100, client
+      end
+
+      bm.report "Select and iterate over 100 singers:" do
+        benchmark_select_100_singers client
+      end
+
+      bm.report "Select and iterate over 100 singers in a read-only transaction:" do
+        benchmark_select_100_singers_in_read_only_transaction client
+      end
+
+      bm.report "Select and iterate over 100 singers in a read/write transaction:" do
+        benchmark_select_100_singers_in_read_write_transaction client
+      end
+    end
+  end
+
+  def self.benchmark_select_one_singer singer, client
+    if client
+      sql = "SELECT * FROM Singers WHERE id=@id"
+      params = { id: singer[:id] }
+      param_types = { id: :INT64 }
+      client.execute(sql, params: params, types: param_types).rows.each do |row|
+        return row
+      end
+    else
+      Singer.find singer.id
+    end
+  end
+
+  def self.benchmark_create_and_reload client
+    singer = create_singers 1, client
+    if client
+      sql = "SELECT * FROM Singers WHERE id=@id"
+      params = { id: singer[:id] }
+      param_types = { id: :INT64 }
+      client.execute(sql, params: params, types: param_types).rows.each do |row|
+        return row
+      end
+    else
+      singer.reload
+    end
+  end
+
+  def self.benchmark_select_and_update_using_mutation count, client
+    benchmark_select_and_update count, :buffered_mutations, client
+  end
+
+  def self.benchmark_select_and_update_using_dml count, client
+    benchmark_select_and_update count, :serializable, client
+  end
+
+  def self.benchmark_select_and_update count, isolation, client
+    # Select some random singers OUTSIDE of a transaction to prevent the random select to cause transactions to abort.
+    sql = "SELECT id FROM singers TABLESAMPLE RESERVOIR (#{count} ROWS)"
+    rows = Singer.connection.select_all(sql).to_a
+    if client
+      client.transaction do |transaction|
+        rows.each do |row|
+          singer = transaction.execute("SELECT * FROM singers WHERE id=@id",
+                                       params: { id: row["id"] },
+                                       types:  { id: :INT64 }).rows.first
+          if isolation == :buffered_mutations
+            transaction.update "singers", id: singer[:id], last_name: SecureRandom.uuid
+          else
+            params      = { id: singer[:id], last_name: SecureRandom.uuid }
+            param_types = { id: :INT64, last_name: :STRING }
+            transaction.execute_update "UPDATE singers SET last_name=@last_name WHERE id=@id",
+                                       params: params,
+                                       types: param_types
+          end
+        end
+      end
+    else
+      Singer.transaction isolation: isolation do
+        rows.each do |row|
+          singer = Singer.find row["id"]
+          singer.update last_name: SecureRandom.uuid
+        end
+      end
+    end
+  end
+
+  def self.benchmark_create_albums_using_mutations count, client
+    create_albums count, :buffered_mutations, client
+  end
+
+  def self.benchmark_create_albums_using_dml count, client
+    create_albums count, :serializable, client
+  end
+
+  def self.benchmark_select_100_singers client
+    sql = "SELECT * FROM singers TABLESAMPLE RESERVOIR (100 ROWS)"
+    count = 0
+    if client
+      client.execute(sql).rows.each do |_row|
+        count += 1
+      end
+    else
+      Singer.find_by_sql(sql).each do
+        count += 1
+      end
+    end
+  end
+
+  def self.benchmark_select_100_singers_in_read_only_transaction client
+    sql = "SELECT * FROM singers TABLESAMPLE RESERVOIR (100 ROWS)"
+    count = 0
+    if client
+      client.snapshot do |snapshot|
+        snapshot.execute(sql).rows.each do |_row|
+          count += 1
+        end
+      end
+    else
+      Singer.transaction isolation: :read_only do
+        Singer.find_by_sql(sql).each do
+          count += 1
+        end
+      end
+    end
+  end
+
+  def self.benchmark_select_100_singers_in_read_write_transaction client
+    sql = "SELECT * FROM singers TABLESAMPLE RESERVOIR (100 ROWS)"
+    count = 0
+    if client
+      client.transaction do |transaction|
+        transaction.execute(sql).rows.each do |_row|
+          count += 1
+        end
+      end
+    else
+      Singer.transaction do
+        Singer.find_by_sql(sql).each do
+          count += 1
+        end
+      end
+    end
+  end
+
+  def self.create_singers count, client
+    first_names = %w[Pete Alice John Ethel Trudy Naomi Wendy Ruben Thomas Elly Cora Elise April Libby Alexandra Shania]
+    last_names = %w[Wendelson Allison Peterson Johnson Henderson Ericsson Aronson Tennet Courtou Mcdonald Berry Ramirez]
+
+    last_singer = nil
+    if client
+      client.commit do |c|
+        singers = []
+        count.times do
+          last_singer = { id: SecureRandom.uuid.gsub("-", "").hex & 0x7FFFFFFFFFFFFFFF,
+                          first_name: first_names.sample, last_name: last_names.sample,
+                          birth_date: Date.new(rand(1920..2005), rand(1..12), rand(1..28)),
+                          picture: StringIO.new(SecureRandom.uuid) }
+          singers << last_singer
+        end
+        c.insert "singers", singers
+      end
+    else
+      Singer.transaction isolation: :buffered_mutations do
+        count.times do
+          last_singer = Singer.create first_name: first_names.sample, last_name: last_names.sample,
+                                      birth_date: Date.new(rand(1920..2005), rand(1..12), rand(1..28)),
+                                      picture: StringIO.new("some-picture-#{SecureRandom.uuid}")
+        end
+      end
+    end
+    last_singer
+  end
+
+  def self.create_albums count, isolation, client
+    sql = "SELECT * FROM singers TABLESAMPLE RESERVOIR (1 ROWS)"
+    singer = Singer.find_by_sql(sql).first
+    if client
+      if isolation == :buffered_mutations
+        client.commit do |c|
+          albums = []
+          count.times do
+            albums << { id: SecureRandom.uuid.gsub("-", "").hex & 0x7FFFFFFFFFFFFFFF,
+                        singer_id: singer.id, title: "Some random title",
+                        release_date: Date.new(2021, 7, 1) }
+          end
+          c.insert "albums", albums
+        end
+      else
+        client.transaction do |transaction|
+          sql = "INSERT INTO albums (id, singer_id, title, release_date) VALUES (@id, @singer, @title, @release_date)"
+          transaction.batch_update do |b|
+            count.times do
+              params = { id: SecureRandom.uuid.gsub("-", "").hex & 0x7FFFFFFFFFFFFFFF, singer: singer.id,
+                         title: "Some random title", release_date: Date.new(2021, 7, 1) }
+              param_types = { id: :INT64, singer: :INT64, title: :STRING, release_date: :DATE }
+              b.batch_update sql, params: params, types: param_types
+            end
+          end
+        end
+      end
+    else
+      Album.transaction isolation: isolation do
+        count.times do
+          Album.create singer: singer, title: "Some random title", release_date: Date.new(2021, 7, 1)
+        end
+      end
+    end
+  end
+end
+
+Application.run

data/benchmarks/config/database.yml

@@ -0,0 +1,8 @@
+development:
+  adapter: spanner
+  project: <benchmark-project>
+  instance: <benchmark-instance>
+  database: <benchmark-database>
+  credentials: <benchmark-credentials>
+  pool: 400
+  checkout_timeout: 60000

data/benchmarks/config/environment.rb

@@ -0,0 +1,12 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+require "active_record"
+require "bundler"
+
+Dir["../lib/*.rb"].each { |file| require file }
+
+Bundler.require