octoball 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '0669e5023c27ce0b1ec2e5116331e8b9f6e5fa405e9f9ac87ea198ecc10023e4'
+  data.tar.gz: 0b222767e80de3ad53e689d0ffda6e7fe87b4ce551e3bcd2eacc6beb79b98abe
+SHA512:
+  metadata.gz: 9282d81935c2c736cd9d9abec4b9a5a42b595aabb64326bd77a4f74027fb12e04855bf8f7197f9abc83f24488eac3e9a64e4689ab6a275da7764999a252f812c
+  data.tar.gz: 0c00b0f6ee82aaa84278b0343b8a264f999e34eabe7d7fe96c99d6845bd56496745c5e39479a34033b8dd01fd87ab036859faeaf510e2d2a54b34ef7288f9df1

data/.circleci/config.yml

@@ -0,0 +1,46 @@
+version: 2.1
+
+executors:
+  ruby:
+    parameters:
+      version:
+        type: string
+    docker:
+    - image: circleci/ruby:<< parameters.version >>
+    - image: circleci/mysql:5
+    environment:
+    - RAILS_ENV: test
+    - MYSQL_HOST: 127.0.0.1
+
+jobs:
+  rspec:
+    parameters:
+      version:
+        type: string
+    executor:
+      name: ruby
+      version: << parameters.version >>
+    steps:
+    - checkout
+    - run:
+        name: bundle install
+        command: |
+          gem update bundler
+          bundle config --local path vendor/bundle
+          bundle install --jobs=4 --retry=3
+    - run:
+        name: Setup databases
+        command: bundle exec rake db:prepare
+    - run:
+        name: Run tests
+        command: bundle exec rspec
+
+workflows:
+  version: 2
+  rspecs:
+    jobs:
+    - rspec:
+        matrix:
+          parameters:
+            version:
+            - "2.7"

data/.gitignore

@@ -0,0 +1,7 @@
+/.bundle
+/database.log
+.byebug_history
+/vendor
+/pkg
+*.lock
+*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,107 @@
+# Octoball - Octopus-like sharding helper library for ActiveRecord 6.1+
+
+<img src="https://user-images.githubusercontent.com/26372128/98494380-3711be00-2280-11eb-8805-6f9e47aeee21.jpg" align="left" width=120>
+
+Octoball provides [Octopus](https://github.com/thiagopradi/octopus)-like database sharding helpers for ActiveRecord 6.1+.
+This will make it easier to upgrade Rails to 6.1+ for applications using [Octopus gem](https://github.com/thiagopradi/octopus) for database sharding with Rails 4.x/5.x.
+
+Currently, its implementation is focusing on horizontal database sharding. However, by customizing shard key mapping, it can be applied to replication use case too.
+<br clear="both">
+
+## Scope of this gem
+
+### What is included in Octoball
+- Octopus-like shard swithcing by `using` class method, e.g.:
+  ```ruby
+  Octoball.using(:shard1) { User.find_by_name("Alice") }
+  User.using(:shard1).first
+  ```
+- Each model instance knows which shard it came from so shard will be switched automatically:
+  ```ruby
+  user1 = User.using(:shard1).find_by_name("Bob")
+  user2 = User.using(:shard2).find_by_name("Charlie")
+  user1.age += 1
+  user2.age += 1
+  user1.save!  #  Save the user1 in the correct shard `:shard1`
+  user2.save!  #  Save the user2 in the correct shard `:shard2`
+  ```
+- Relations such as `has_many` are also resolved from the model instance's shard:
+  ```ruby
+  user = User.using(:shard1).find_by_name("Alice")
+  user.blogs.where(title: "blog")  # user's blogs are fetched from `:shard1`
+  ```
+
+### What is NOT included in Octoball
+- Connection handling and configuration -- managed by the native `ActiveRecord::Base.connects_to` methods introduced in ActiveRecord 6.1.
+  - You need to migrate from Octopus' `config/shards.yml` to [Rails native multiple DB configuration using `config/database.yml`](https://edgeguides.rubyonrails.org/active_record_multiple_databases.html). Please refer the [Setup](#Setup) section for more details.
+- Migration -- done by ActiveRecord 6.1+ natively.
+  - Instead of `using` method in Octopus, you can specify the `migrations_paths` parameter in the `config/database.yml` file.
+- Replication handling -- done by ActiveRecord's `role`
+  - round-robin connection scheduler is currently omitted.
+
+## Setup
+
+```
+gem "octoball"
+```
+
+Until first release:
+```
+# Bundle edge Rails
+gem 'rails', github: 'rails/rails'
+gem 'octoball', git: 'git@github.com:aktsk/octoball'
+```
+
+Define the database connections in `config/database.yml`, e.g.:
+```
+default: &default
+  adapter: mysql2
+  pool: 5
+  username: root
+  host: localhost
+  timeout: 5000
+  connnect_timeout: 5000
+
+development:
+  master:
+    <<: *default
+    database: db_primary
+  shard1_connection:
+    <<: *default
+    database: db_shard1
+```
+And define shards and corresponding connections in abstract ActiveRecord model class, e.g.:
+```ruby
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+
+  connects_to shards: {
+    master: { writing: :master },
+    shard1: { writing: :shard1_connection },
+  }
+end
+
+class User < ApplicationRecord
+  ...
+end
+```
+
+Optionally, to use the `:master` shard as a default connection like Octopus, add the following script to `config/initializers/default_shard.rb`:
+```
+ActiveRecord::Base.default_shard = :master
+```
+
+
+## Development of Octoball
+Octoball has rspec tests delived from subsets of Octopus' rspec.
+
+To run the rspec tests, follow these steps:
+```
+RAILS_ENV=test bundle exec rake db:prepare
+RAILS_ENV=test bundle exec rake spec
+```
+
+## License
+Octoball is released under the MIT license.
+
+Original Octopus' copyright: Copyright (c) Thiago Pradi

data/Rakefile

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new
+RuboCop::RakeTask.new
+
+namespace :db do
+  mysql_spec = {
+    adapter:  'mysql2',
+    host:     (ENV['MYSQL_HOST'] || 'localhost'),
+    username: (ENV['MYSQL_USER'] || 'root'),
+    encoding: 'utf8mb4',
+  }
+
+  desc 'Build the databases for tests'
+  task :build_databases do
+    require 'active_record'
+
+    # Connect to MYSQL
+    ActiveRecord::Base.establish_connection(mysql_spec)
+    (1..5).map do |i|
+      # drop the old database (if it exists)
+      ActiveRecord::Base.connection.drop_database("octoball_shard_#{i}")
+      # create new database
+      ActiveRecord::Base.connection.create_database("octoball_shard_#{i}", charset: 'utf8mb4')
+    end
+  end
+
+  desc 'Create tables on tests databases'
+  task :create_tables do
+    ActiveRecord::Base.configurations = {
+      "test" => {
+        shard1: mysql_spec.merge(database: 'octoball_shard_1'),
+        shard2: mysql_spec.merge(database: 'octoball_shard_2'),
+        shard3: mysql_spec.merge(database: 'octoball_shard_3'),
+        shard4: mysql_spec.merge(database: 'octoball_shard_4'),
+        shard5: mysql_spec.merge(database: 'octoball_shard_5'),
+      }
+    }
+    require './spec/models/application_record'
+    ActiveRecord::Base.configurations.configs_for(env_name: "test").each do |config|
+      ActiveRecord::Base.establish_connection(config)
+      schema_migration = ActiveRecord::Base.connection.schema_migration
+      ActiveRecord::MigrationContext.new("spec/migration", schema_migration)
+        .migrate(config.database == 'octoball_shard_5' ? 2 : 1)
+    end
+  end
+
+  desc 'Prepare the test databases'
+  task prepare: [:build_databases, :create_tables]
+end

data/lib/octoball.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require 'octoball/version'
+require 'octoball/relation_proxy'
+require 'octoball/connection_adapters'
+require 'octoball/connection_handling'
+require 'octoball/current_shard_tracker'
+require 'octoball/association'
+require 'octoball/association_shard_check'
+require 'octoball/persistence'
+require 'octoball/log_subscriber'
+
+class Octoball
+  def self.using(shard, &block)
+    ActiveRecord::Base.connected_to(role: current_role, shard: shard&.to_sym, &block)
+  end
+
+  def self.current_role
+    ActiveRecord::Base.current_role || ActiveRecord::Base.writing_role
+  end
+
+  module UsingShard
+    def using(shard)
+      Octoball::RelationProxy.new(all, shard&.to_sym)
+    end
+  end
+
+  ::ActiveRecord::Base.singleton_class.prepend(UsingShard)
+end

data/lib/octoball/association.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class Octoball
+  module RelationCurrentShard
+    attr_accessor :current_shard
+  end
+
+  module ShardedCollectionAssociation
+    [:reader, :writer, :ids_reader, :ids_writer, :create, :create!,
+     :build, :include?, :load_target, :reload, :size, :select].each do |method|
+      class_eval <<-"END", __FILE__, __LINE__ + 1
+        def #{method}(*args, &block)
+          shard = owner.current_shard
+          return super if !shard || shard == ActiveRecord::Base.current_shard
+          ActiveRecord::Base.connected_to(shard: shard, role: Octoball.current_role) do
+            ret = super
+            return ret unless ret.is_a?(::ActiveRecord::Relation) || ret.is_a?(::ActiveRecord::QueryMethods::WhereChain)
+            RelationProxy.new(ret, shard)
+          end
+        end
+        ruby2_keywords(:#{method}) if respond_to?(:ruby2_keywords, true)
+      END
+    end
+  end
+
+  module ShardedCollectionProxy
+    [:any?, :build, :count, :create, :create!, :concat, :delete, :delete_all,
+     :destroy, :destroy_all, :empty?, :find, :first, :include?, :last, :length,
+     :many?, :pluck, :replace, :select, :size, :sum, :to_a, :uniq].each do |method|
+      class_eval <<-"END", __FILE__, __LINE__ + 1
+        def #{method}(*args, &block)
+          return super if !@association.owner.current_shard || @association.owner.current_shard == ActiveRecord::Base.current_shard
+          ActiveRecord::Base.connected_to(shard: @association.owner.current_shard, role: Octoball.current_role) do
+            super
+          end
+        end
+        ruby2_keywords(:#{method}) if respond_to?(:ruby2_keywords, true)
+      END
+    end
+  end
+
+  module ShardedSingularAssociation
+    [:reader, :writer, :create, :create!, :build].each do |method|
+      class_eval <<-"END", __FILE__, __LINE__ + 1
+        def #{method}(*args, &block)
+          return super if !owner.current_shard || owner.current_shard == ActiveRecord::Base.current_shard
+          ActiveRecord::Base.connected_to(shard: owner.current_shard, role: Octoball.current_role) do
+            super
+          end
+        end
+        ruby2_keywords(:#{method}) if respond_to?(:ruby2_keywords, true)
+      END
+    end
+  end
+
+  ::ActiveRecord::Relation.prepend(RelationCurrentShard)
+  ::ActiveRecord::QueryMethods::WhereChain.prepend(RelationCurrentShard)
+  ::ActiveRecord::Associations::CollectionAssociation.prepend(ShardedCollectionAssociation)
+  ::ActiveRecord::Associations::CollectionProxy.prepend(ShardedCollectionProxy)
+  ::ActiveRecord::Associations::SingularAssociation.prepend(ShardedSingularAssociation)
+end

data/lib/octoball/association_shard_check.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class Octoball
+  class MismatchedShards < StandardError
+    attr_reader :record, :current_shard
+
+    def initialize(record, current_shard)
+      @record = record
+      @current_shard = current_shard
+    end
+
+    def message
+      "Association shard mismatch: record shard is \"#{record.current_shard}\" but current shard is \"#{current_shard}\""
+    end
+  end
+
+  module AssociationShardCheck
+    def association_shard_check(record)
+      fail MismatchedShards.new(record, current_shard) if record.current_shard != current_shard
+    end
+  end
+
+  module AssociationShardChecker
+    def has_many(name, scope = nil, **options, &extension)
+      assign_octoball_check_opts(options)
+      super
+    end
+
+    def has_and_belongs_to_many(association_id, scope = nil, **options, &extension)
+      assign_octoball_check_opts(options)
+      super
+    end
+
+    private
+
+    def assign_octoball_check_opts(options)
+      options[:before_add] = [:association_shard_check, options[:before_add]].compact.flatten
+      options[:before_remove] = [:association_shard_check, options[:before_remove]].compact.flatten
+    end
+  end
+
+  ::ActiveRecord::Base.prepend(AssociationShardCheck)
+  ::ActiveRecord::Base.singleton_class.prepend(AssociationShardChecker)
+end

data/lib/octoball/connection_adapters.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Octoball
+  module ConnectionHandlerSetCurrentShard
+    def retrieve_connection(spec_name, role: ActiveRecord::Base.current_role, shard: ActiveRecord::Base.current_shard)
+      conn = super
+      conn.current_shard = shard
+      conn
+    end
+  end
+
+  module ConnectionHasCurrentShard
+    attr_accessor :current_shard
+  end
+
+  ::ActiveRecord::ConnectionAdapters::ConnectionHandler.prepend(ConnectionHandlerSetCurrentShard)
+  ::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ConnectionHasCurrentShard)
+end

data/lib/octoball/connection_handling.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Octoball
+  module ConnectionHandlingAvoidAutoLoadProxy
+    private
+
+    def swap_connection_handler(handler, &blk)
+      old_handler, ActiveRecord::Base.connection_handler = ActiveRecord::Base.connection_handler, handler
+      return_value = yield
+      return_value.load if !return_value.respond_to?(:ar_relation) && return_value.is_a?(ActiveRecord::Relation)
+      return_value
+    ensure
+      ActiveRecord::Base.connection_handler = old_handler
+    end
+  end
+
+  ::ActiveRecord::Base.extend(ConnectionHandlingAvoidAutoLoadProxy)
+end

data/lib/octoball/current_shard_tracker.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class Octoball
+  module CurrentShardTracker
+    attr_reader :current_shard
+
+    def becomes(klass)
+      became = super
+      became.instance_variable_set(:@current_shard, current_shard)
+      became
+    end
+
+    def ==(other)
+      super && current_shard == other.current_shard
+    end
+
+    module ClassMethods
+      private
+
+      def instantiate_instance_of(klass, attributes, column_types = {}, &block)
+        result = super
+        result.instance_variable_set(:@current_shard, connection.current_shard)
+        result
+      end
+    end
+  end
+
+  ::ActiveRecord::Base.prepend(CurrentShardTracker)
+  ::ActiveRecord::Base.singleton_class.prepend(CurrentShardTracker::ClassMethods)
+end

data/lib/octoball/log_subscriber.rb

@@ -0,0 +1,21 @@
+# Implementation courtesy of db-charmer.
+class Octoball
+  module LogSubscriber
+    attr_accessor :current_shard
+
+    def sql(event)
+      shard = event.payload[:connection]&.current_shard
+      self.current_shard = shard == ActiveRecord::Base.default_shard ? nil : shard
+      super
+    end
+
+    private
+
+    def debug(progname = nil, &block)
+      conn = current_shard ? color("[Shard: #{current_shard}]", ActiveSupport::LogSubscriber::GREEN, true) : ''
+      super(conn + progname.to_s, &block)
+    end
+  end
+end
+
+ActiveRecord::LogSubscriber.prepend(Octoball::LogSubscriber)