activerecord-shard_for 0.1.0

data/gemfiles/rails_edge.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", :github => "rails/rails"
+gem "arel", :github => "rails/arel"
+
+gemspec :path => "../"

data/gemfiles/rails_edge.gemfile.lock

@@ -0,0 +1,193 @@
+GIT
+  remote: git://github.com/rails/arel.git
+  revision: 09827f361d4bc73e6ff157d9feb74a465e4e4cdd
+  specs:
+    arel (7.1.1)
+
+GIT
+  remote: git://github.com/rails/rails.git
+  revision: 815b730b1b79158511f9f4c8465c476b9fe9b7e0
+  specs:
+    activemodel (5.1.0.alpha)
+      activesupport (= 5.1.0.alpha)
+    activerecord (5.1.0.alpha)
+      activemodel (= 5.1.0.alpha)
+      activesupport (= 5.1.0.alpha)
+      arel (~> 7.0)
+    activesupport (5.1.0.alpha)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+
+PATH
+  remote: ../
+  specs:
+    activerecord-shard_for (0.1.0)
+      activerecord (>= 4.1.0)
+      activesupport (>= 4.1.0)
+      expeditor (>= 0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    abstract_type (0.0.7)
+    adamantium (0.2.0)
+      ice_nine (~> 0.11.0)
+      memoizable (~> 0.4.0)
+    appraisal (2.1.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    ast (2.3.0)
+    binding_of_caller (0.7.2)
+      debug_inspector (>= 0.0.1)
+    coderay (1.1.1)
+    concord (0.1.5)
+      adamantium (~> 0.2.0)
+      equalizer (~> 0.0.9)
+    concurrent-ruby (1.0.2)
+    concurrent-ruby-ext (1.0.2)
+      concurrent-ruby (~> 1.0.2)
+    debug_inspector (0.0.2)
+    diff-lcs (1.2.5)
+    equalizer (0.0.11)
+    expeditor (0.4.0)
+      concurrent-ruby (~> 1.0.0)
+      concurrent-ruby-ext (~> 1.0.0)
+      retryable (> 1.0)
+    ffi (1.9.14)
+    formatador (0.2.5)
+    guard (2.14.0)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (~> 1.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    guard-rubocop (1.2.0)
+      guard (~> 2.0)
+      rubocop (~> 0.20)
+    i18n (0.7.0)
+    ice_nine (0.11.2)
+    interception (0.5)
+    listen (3.1.5)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+      ruby_dep (~> 1.2)
+    lumberjack (1.0.10)
+    memoizable (0.4.2)
+      thread_safe (~> 0.3, >= 0.3.1)
+    method_source (0.8.2)
+    minitest (5.9.0)
+    nenv (0.3.0)
+    notiffany (0.1.1)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    parser (2.3.1.2)
+      ast (~> 2.2)
+    powerpack (0.1.1)
+    proc_to_ast (0.1.0)
+      coderay
+      parser
+      unparser
+    procto (0.0.3)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-doc (0.9.0)
+      pry (~> 0.9)
+      yard (~> 0.8)
+    pry-inline (1.0.2)
+      pry (~> 0.10.0)
+      unicode (~> 0.4.4)
+    pry-rescue (1.4.4)
+      interception (>= 0.5)
+      pry
+    pry-stack_explorer (0.4.9.2)
+      binding_of_caller (>= 0.7)
+      pry (>= 0.9.11)
+    rainbow (2.1.0)
+    rake (10.5.0)
+    rb-fsevent (0.9.7)
+    rb-inotify (0.9.7)
+      ffi (>= 0.5.0)
+    retryable (2.0.4)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.2)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-parameterized (0.3.0)
+      binding_of_caller
+      parser
+      proc_to_ast
+      rspec (>= 2.13, < 4)
+      unparser
+    rspec-support (3.5.0)
+    rubocop (0.42.0)
+      parser (>= 2.3.1.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.8.1)
+    ruby_dep (1.3.1)
+    shellany (0.0.1)
+    slop (3.6.0)
+    sqlite3 (1.3.11)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unicode (0.4.4.2)
+    unicode-display_width (1.1.0)
+    unparser (0.2.5)
+      abstract_type (~> 0.0.7)
+      adamantium (~> 0.2.0)
+      concord (~> 0.1.5)
+      diff-lcs (~> 1.2.5)
+      equalizer (~> 0.0.9)
+      parser (~> 2.3.0)
+      procto (~> 0.0.2)
+    yard (0.9.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord!
+  activerecord-shard_for!
+  appraisal
+  arel!
+  bundler (~> 1.11)
+  guard-rspec
+  guard-rubocop
+  pry
+  pry-doc
+  pry-inline
+  pry-rescue
+  pry-stack_explorer
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rspec-parameterized
+  rubocop
+  sqlite3
+
+BUNDLED WITH
+   1.12.5

data/lib/activerecord/shard_for.rb

@@ -0,0 +1,32 @@
+require 'active_record'
+require 'expeditor'
+require 'activerecord/shard_for/version'
+require 'activerecord/shard_for/config'
+require 'activerecord/shard_for/cluster_config'
+require 'activerecord/shard_for/model'
+require 'activerecord/shard_for/errors'
+require 'activerecord/shard_for/cluster_router'
+require 'activerecord/shard_for/hash_modulo_router'
+require 'activerecord/shard_for/database_tasks'
+require 'activerecord/shard_for/shard_repogitory'
+require 'activerecord/shard_for/all_shards_in_parallel'
+require 'activerecord/shard_for/replication_mapping'
+require 'activerecord/railtie' if defined?(Rails5)
+
+module ActiveRecord
+  module ShardFor
+    class << self
+      # @return [Activerecord::ShardFor::Config]
+      def config
+        @config ||= Config.new
+      end
+
+      # @yield [Activerecord::ShardFor::Config]
+      def configure(&block)
+        config.instance_eval(&block)
+      end
+    end
+  end
+end
+
+ActiveRecord::ShardFor.config.register_cluster_router(:hash_modulo, ActiveRecord::ShardFor::HashModuloRouter)

data/lib/activerecord/shard_for/all_shards_in_parallel.rb

@@ -0,0 +1,43 @@
+module ActiveRecord
+  # Support parallel execution with each shard and deal with AR connection
+  # management in parallel execution.
+  module ShardFor
+    class AllShardsInParallel
+      # @param [Array<Class>] An array of shard model class
+      def initialize(shards)
+        @shards = shards
+      end
+
+      # @yield [Class] A shard model class
+      # @return [Array] A result
+      # @example
+      #   User.all_shards_in_parallel.map(&:count).reduce(&:+)
+      def map
+        return [] unless block_given?
+
+        commands = @shards.map do |m|
+          Expeditor::Command.new { m.connection_pool.with_connection { yield m } }
+        end
+        commands.each(&:start)
+        commands.map(&:get)
+      end
+
+      # @yield [Class] A shard model class
+      # @return [Array] A result
+      # @example
+      #   User.all_shards_in_parallel.flat_map {|m| m.where(age: 1) }
+      def flat_map(&block)
+        map(&block).flatten
+      end
+
+      # @yield [Class] A shard model class
+      # @return [ActiveRecord::ShardFor::AllShardsInParallel]
+      # @example
+      #   User.all_shards_in_parallel.each {|m| puts m.count }
+      def each(&block)
+        map(&block) if block_given?
+        self
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/cluster_config.rb

@@ -0,0 +1,32 @@
+module ActiveRecord
+  module ShardFor
+    class ClusterConfig
+      attr_reader :name, :connection_registry
+
+      # @param [Symbol] name
+      def initialize(name)
+        @name = name
+        @connection_registry = {}
+      end
+
+      # @param [Object] key sharding key object for connection
+      # @param [Symbol] connection_name
+      # @raise [RuntimeError] when duplicate entry of  key
+      def register(key, connection_name)
+        raise RuntimeError.new, "#{key} is registered" if connection_registry.key?(key)
+        connection_registry[key] = connection_name
+      end
+
+      # @return [Array<Symbol>] An array of connection name
+      def connections
+        connection_registry.values
+      end
+
+      # @param [Object] key
+      # @return [Symbol] registered connection name
+      def fetch(key)
+        connection_registry.fetch(key)
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/config.rb

@@ -0,0 +1,45 @@
+module ActiveRecord
+  module ShardFor
+    class Config
+      attr_reader :cluster_configs, :routers
+
+      def initialize
+        @cluster_configs = {}
+        @routers = {}
+      end
+
+      # Define config for specific cluster.
+      # See README.md for example.
+      # @param [String] cluster_name
+      # @yield [ActiveRecord::ShardFor::ClusterConfig]
+      # @return [ActiveRecord::ShardFor::ClusterConfig]
+      # raise [RuntimeError] when this cluster config is invalid.
+      def define_cluster(cluster_name, &block)
+        cluster_config = ClusterConfig.new(cluster_name)
+        cluster_config.instance_eval(&block)
+        cluster_configs[cluster_name] = cluster_config
+      end
+
+      # @param [Symbol] cluster_name
+      # @return [ActiveRecord::ShardFor::ClusterConfig]
+      # @raise [KeyError] when not registered key given
+      def fetch_cluster_config(cluster_name)
+        cluster_configs.fetch(cluster_name)
+      end
+
+      # Register router for ActiveRecord::ShardFor
+      # See README.md for example.
+      # @param [Symbol] router_name
+      # @router_class [Class] router_class
+      def register_cluster_router(router_name, router_class)
+        routers[router_name] = router_class
+      end
+
+      # @param [Symbol] router_name
+      # @return [Class] registered class by [#register_router]
+      def fetch_cluster_router(router_name)
+        routers[router_name]
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/connection_router.rb

@@ -0,0 +1,33 @@
+module ActiveRecord
+  module ShardFor
+    # @abstract Subclass and override [#route] to inplement
+    class ConnectionRouter
+      attr_reader :cluster_config
+
+      # @param [ActiveRecord::ShardFor::ClusterConfig]
+      def initialize(cluster_config)
+        @cluster_config = cluster_config
+      end
+
+      # Fetch shard by sharding key
+      # @param [Object] key routing key
+      def fetch_connection_name(key)
+        cluster_config.fetch route(key)
+      end
+
+      # Decide routing for shard.
+      # Override this method in subclass.
+      # @param [Object] key sharding key
+      def route(_key)
+        raise NotImplementedError.new, 'Please impement this method'
+      end
+
+      private
+
+      # @return [Integer] count of registered connection
+      def connection_count
+        cluster_config.connections.count
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/database_tasks.rb

@@ -0,0 +1,185 @@
+module ActiveRecord
+  module ShardFor
+    module DatabaseTasks
+      module_function
+
+      # @return [Boolean]
+      def ar5?
+        ActiveRecord::VERSION::MAJOR == 5
+      end
+
+      # @return [Boolean]
+      def ar4?
+        ActiveRecord::VERSION::MAJOR == 4
+      end
+
+      # @return [Boolean]
+      def ar42?
+        ar4? && ActiveRecord::VERSION::MINOR == 2
+      end
+
+      # @return [Boolean]
+      def ar41?
+        ar4? && ActiveRecord::VERSION::MINOR == 1
+      end
+
+      # @return [Boolean]
+      def ar417_above?
+        ar41? && ActiveRecord::VERSION::TINY > 7
+      end
+
+      # Show information of database sharding config.
+      def info
+        puts 'All clusters registered to ActiveRecord::ShardFor'
+        puts
+        clusters.each do |cluster|
+          puts "= Cluster: #{cluster.name} ="
+          cluster.connections.each do |name|
+            puts "- #{name}"
+          end
+          puts
+        end
+      end
+
+      # @private
+      # @param [String] task_name
+      # @return [Rake::Task]
+      def to_rake_task(task_name)
+        Rake::Task[task_name]
+      end
+
+      # @private
+      # @return [Array<Symbol>]
+      def cluster_names
+        ActiveRecord::ShardFor.config.cluster_configs.keys
+      end
+
+      # @private
+      # @return [Array<ActiveRecord::ShardFor::ClusterConfig>]
+      def clusters
+        ActiveRecord::ShardFor.config.cluster_configs.values
+      end
+
+      # @private
+      # @return [ActiveRecord::ShardFor::ClusterConfig]
+      # @raise [KeyError]
+      def fetch_cluster_config(cluster_name)
+        ActiveRecord::ShardFor.config.fetch_cluster_config(cluster_name)
+      end
+
+      # For mock-ablity
+      # @private
+      def exit_with_error
+        exit 1
+      end
+
+      module TasksForMultipleClusters
+        # @param [String] task_name
+        def invoke_task_for_all_clusters(task_name)
+          cluster_names.each do |cluster_name|
+            invoke_task(task_name, cluster_name)
+          end
+        end
+
+        # @private
+        # @param [String] name
+        # @param [Symbol] cluster_name
+        def invoke_task(name, cluster_name)
+          task_name = "activerecord:shard_for:#{name}"
+          to_rake_task(task_name).invoke(cluster_name.to_s)
+          to_rake_task(task_name).reenable
+        end
+      end
+      extend TasksForMultipleClusters
+
+      # Organize cluster config and handle error for invalid args, call single
+      # cluster task with each single connection config.
+      module TaskOrganizerForSingleClusterTask
+        # @param [Hash{Symbol => String}] args
+        def create_all_databases(args)
+          exec_task_for_all_databases('create', args)
+        end
+
+        # @param [Hash{Symbol => String}] args
+        def drop_all_databases(args)
+          exec_task_for_all_databases('drop', args)
+        end
+
+        # @param [Hash{Symbol => String}] args
+        def load_schema_all_databases(args)
+          exec_task_for_all_databases('load_schema', args)
+        end
+
+        private
+
+        # @param [String] task_name
+        # @param [Hash{Symbol => String}] args
+        def exec_task_for_all_databases(task_name, args)
+          cluster_name = cluster_name_or_error(task_name, args)
+          cluster = cluster_or_error(cluster_name)
+          cluster.connections.each do |connection_name|
+            __send__(task_name, connection_name.to_s)
+          end
+        end
+
+        # @param [String] name A task name
+        # @param [Hash{Symbol => String}] args
+        # @return [String]
+        def cluster_name_or_error(name, args)
+          cluster_name = args[:cluster_name]
+          return cluster_name if cluster_name
+
+          $stderr.puts <<-MSG
+Missing cluster_name. Find cluster_name via `rake activerecord:shard_for:info` then call `rake "activerecord:shard_for:#{name}[$cluster_name]"`.
+          MSG
+          exit_with_error
+        end
+
+        # @param [String] cluster_name
+        # @return [ActiveRecord::ShardFor::ClusterConfig]
+        def cluster_or_error(cluster_name)
+          fetch_cluster_config(cluster_name.to_sym)
+        rescue KeyError
+          $stderr.puts %(!cluster name "#{cluster_name}" not found.!)
+          exit_with_error
+        end
+      end
+      extend TaskOrganizerForSingleClusterTask
+
+      # Create, drop, load_schema for single connection config.
+      module TasksForSingleConnection
+        # @param [String] connection_name
+        def create(connection_name)
+          configuration = ActiveRecord::Base.configurations[connection_name]
+          ActiveRecord::Tasks::DatabaseTasks.create(configuration)
+          # Re-configure using configuration with database
+          ActiveRecord::Base.establish_connection(configuration)
+        end
+
+        # @param [String] connection_name
+        def drop(connection_name)
+          configuration = ActiveRecord::Base.configurations[connection_name]
+          ActiveRecord::Tasks::DatabaseTasks.drop(configuration)
+        end
+
+        # @param [String] connection_name
+        def load_schema(connection_name)
+          configuration = ActiveRecord::Base.configurations[connection_name]
+
+          case
+          when ar5?
+            ActiveRecord::Tasks::DatabaseTasks.load_schema(configuration, :ruby)
+          when ar42? || ar417_above?
+            ActiveRecord::Tasks::DatabaseTasks.load_schema_for(configuration, :ruby)
+          when ar41?
+            ActiveRecord::Base.establish_connection(configuration)
+            ActiveRecord::Tasks::DatabaseTasks.load_schema(:ruby)
+          else
+            raise "This version of ActiveRecord is not supported: v#{ActiveRecord::VERSION::STRING}"
+          end
+        end
+      end
+      extend TasksForSingleConnection
+    end
+  end
+end