activerecord-shard_for 0.1.0

data/lib/activerecord/shard_for/errors.rb

@@ -0,0 +1,14 @@
+module ActiveRecord
+  module ShardFor
+    class Error < ::StandardError
+    end
+
+    # Raised when try to put new record without distkey attribute.
+    class MissingDistkeyAttribute < Error
+    end
+
+    # Inherit from AR::RecordNotFound to enable to handle as AR's one.
+    class RecordNotFound < ActiveRecord::RecordNotFound
+    end
+  end
+end

data/lib/activerecord/shard_for/hash_modulo_router.rb

@@ -0,0 +1,18 @@
+require 'zlib'
+
+module ActiveRecord
+  module ShardFor
+    class HashModuloRouter < ConnectionRouter
+      # @param [String] key sharding key
+      def route(key)
+        hash(key) % connection_count
+      end
+
+      private
+
+      def hash(v)
+        Zlib.crc32(v.to_s)
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/model.rb

@@ -0,0 +1,134 @@
+require 'active_support/concern'
+
+module ActiveRecord
+  module ShardFor
+    module Model
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :cluster_router, instance_writer: false
+        class_attribute :shard_repository, instance_writer: false
+        class_attribute :replication_mapping, instance_writer: false
+        class_attribute :distkey, instance_writer: false
+      end
+
+      module ClassMethods
+        # The cluster config must be defined before `use_cluster`
+        # @param [Symbol] name A cluster name which is set by ActiveRecord::ShardFor.configure
+        def use_cluster(name, router_name)
+          cluster_config = ActiveRecord::ShardFor.config.fetch_cluster_config(name)
+          cluster_router_class = ActiveRecord::ShardFor.config.fetch_cluster_router(router_name)
+          self.cluster_router = cluster_router_class.new(cluster_config)
+          self.shard_repository = ActiveRecord::ShardFor::ShardRepogitory.new(cluster_config, self)
+          self.abstract_class = true
+        end
+
+        # Returns a generated model class of included model class which has proper
+        # connection config for the shard for given key.
+        # @param [String] key A value of distkey
+        # @return [Class] A generated model class for given distkey value
+        def shard_for(key)
+          connection_name = cluster_router.fetch_connection_name(key)
+          shard_repository.fetch(connection_name)
+        end
+
+        # Create new record with given attributes in proper shard for given key.
+        # When distkey value is empty, raises ActiveRecord::ShardFor::MissingDistkeyAttribute
+        # error.
+        # @param [Hash] attributes
+        # @return [ActiveRecord::Base] A shard model instance
+        # @raise [ActiveRecord::ShardFor::MissingDistkeyAttribute]
+        def put!(attributes)
+          raise '`distkey` is not defined. Use `def_distkey`.' unless distkey
+          key = attributes[distkey]
+          raise ActiveRecord::ShardFor::MissingDistkeyAttribute unless key || attributes[distkey.to_s]
+
+          @before_put_callback.call(attributes) if @before_put_callback
+
+          shard_for(key).create!(attributes)
+        end
+
+        # Register hook to assign auto-generated distkey or something.
+        # Sometimes you want to generates distkey value before validation. Since
+        # activerecord-shard_for generates sub class of your models, AR's callback is not
+        # useless for this usecase, so activerecord-shard_for offers its own callback method.
+        # @example
+        #   class User
+        #     include ActiveRecord::ShardFor::Model
+        #     use_cluster :user
+        #     def_distkey :name
+        #     before_put do |attributes|
+        #       attributes[:name] = generate_name unless attributes[:name]
+        #     end
+        #   end
+        def before_put(&block)
+          @before_put_callback = block
+        end
+
+        # Returns nil when not found. Except that, is same as `.get!`.
+        # @param [String] key
+        # @return [ActiveRecord::Base, nil] A shard model instance
+        def get(key)
+          shard_for(key).find_by(distkey => key)
+        end
+
+        # `.get!` raises ActiveRecord::ShardFor::RecordNotFound which is child class of
+        # `ActiveRecord::RecordNotFound` so you can rescue that exception as same
+        # as AR's RecordNotFound.
+        # @param [String] key
+        # @return [ActiveRecord::Base] A shard model instance
+        # @raise [ActiveRecord::ShardFor::RecordNotFound]
+        def get!(key)
+          model = get(key)
+          return model if model
+
+          raise ActiveRecord::ShardFor::RecordNotFound
+        end
+
+        # Distkey is a column. activerecord-shard_for gave to cluster_router that value
+        # and cluster_router determine which shard to store.
+        # @param [Symbol] column
+        def def_distkey(column)
+          self.distkey = column.to_sym
+        end
+
+        # Returns all generated shard model class. Useful to query to all shards.
+        # @return [Array<Class>] An array of shard models
+        # @example
+        #   User.all_shards.flat_map {|m| m.find_by(name: 'alice') }.compact
+        def all_shards
+          shard_repository.all
+        end
+
+        # @return [ActiveRecord::ShardFor::AllShardsInParallel]
+        # @example
+        #   User.all_shards_in_parallel.map {|m| m.where.find_by(name: 'Alice') }.compact
+        def all_shards_in_parallel
+          AllShardsInParallel.new(all_shards)
+        end
+        alias_method :parallel, :all_shards_in_parallel
+
+        # @param [Hash{Symbol => Symbol}] mapping A pairs of role name and
+        #   AR model class name.
+        def replicates_with(mapping)
+          self.replication_mapping = ActiveRecord::ShardFor::ReplicationMapping.new(mapping)
+        end
+
+        # See example definitions in `spec/models.rb`.
+        # @param [Symbol] A role name of target cluster.
+        # @return [Class, Object] if block given then yielded result else
+        #   target shard model.
+        # @example
+        #   UserReadonly.all_shards.each do |m|
+        #     target_ids = m.where(age: 1).pluck(:id)
+        #     m.switch(:master) do |master|
+        #       master.where(id: target_ids).delete_all
+        #     end
+        #   end
+        def switch(role_name, &block)
+          replication_mapping.switch(self, role_name, &block)
+        end
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/railtie.rb

@@ -0,0 +1,10 @@
+module ActiveRecord
+  module ShardFor
+    # Railtie of activerecord-shard_for
+    class Railtie < ::Rails::Railtie
+      rake_tasks do
+        load File.expand_path('../../tasks/activerecord_shard_for.rake', __FILE__)
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/replication_mapping.rb

@@ -0,0 +1,37 @@
+module ActiveRecord
+  module ShardFor
+    class ReplicationMapping
+      def initialize(mapping)
+        @mapping = mapping
+        @lock = Mutex.new
+      end
+
+      # @param [Class] A shard model having connection to specific shard
+      # @param [Symbol] A role name of target cluster.
+      # @return [Class, Object] if block given then yielded result else
+      #   target shard model.
+      def switch(from, role_name)
+        @lock.synchronize { constantize! unless constantized? }
+
+        model = @mapping.fetch(role_name)
+        target_shard_model = model.shard_repository.fetch_by_key(from.assigned_key)
+
+        if block_given?
+          target_shard_model.connection_pool.with_connection { yield target_shard_model }
+        else
+          target_shard_model
+        end
+      end
+
+      private
+
+      def constantize!
+        @mapping = Hash[@mapping.map { |k, name| [k, name.to_s.constantize] }]
+      end
+
+      def constantized?
+        @mapping.values.first.is_a? Class
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/shard_repogitory.rb

@@ -0,0 +1,69 @@
+module ActiveRecord
+  module ShardFor
+    class ShardRepogitory
+      attr_reader :base_class
+
+      # @param [ClusterConfig] cluster_config
+      # @param [Class] base_class A AR Model
+      def initialize(cluster_config, base_class)
+        @base_class = base_class
+
+        shards = cluster_config.connection_registry.map do |key, connection_name|
+          [connection_name, generate_model_for_shard(connection_name, key)]
+        end
+
+        @shards = Hash[shards]
+      end
+
+      # @param [Symbol] connection_name
+      # @return [Class] A model class for this shard
+      def fetch(connection_name)
+        @shards.fetch(connection_name)
+      end
+
+      # @param [Object] key sharding key object for connection
+      # @return [Class, nil] A AR model class.
+      def fetch_by_key(key)
+        @shards.find { |_, model| model.assigned_key == key }[1]
+      end
+
+      # @return [Array<Class>]
+      def all
+        @shards.values
+      end
+
+      private
+
+      # @param [Symbol] connection_name
+      # @param [Range] slot_range
+      # @return [Class] A sub class of given AR model.
+      #   A sub class has connection setting for specific shard.
+      def generate_model_for_shard(connection_name, key)
+        class_name = generate_class_name(connection_name)
+
+        model = Class.new(base_class) do
+          self.table_name = base_class.table_name
+          class << self
+            attr_reader :assigned_key
+          end
+          @assigned_key = key
+
+          module_eval <<-RUBY, __FILE__, __LINE__ + 1
+            def self.name
+              "#{base_class.name}::#{class_name}"
+            end
+          RUBY
+        end
+
+        model.class_eval { establish_connection(connection_name) }
+        model
+      end
+
+      # @param [Symbol] connection_name
+      # @return [String]
+      def generate_class_name(connection_name)
+        "ShardFor#{connection_name.to_s.tr('-', '_').classify}"
+      end
+    end
+  end
+end

data/lib/activerecord/shard_for/version.rb

@@ -0,0 +1,5 @@
+module ActiveRecord
+  module ShardFor
+    VERSION = '0.1.0'.freeze
+  end
+end

data/lib/activerecord/tasks/activerecord_shard_for.rake

@@ -0,0 +1,42 @@
+namespace :activerecord do
+  namespace :shard_for do
+    desc 'Show all defined clusters and their detail'
+    task info: %i(environment) do
+      ActiveRecord::ShardFor::DatabaseTasks.info
+    end
+
+    desc 'Setup all databases in all clusters'
+    task setup: %i(create_all load_schema_all) do
+    end
+
+    desc 'Create all databases in all clusters'
+    task :create_all => :environment do
+      ActiveRecord::ShardFor::DatabaseTasks.invoke_task_for_all_clusters('create')
+    end
+
+    desc 'Drop all databases in all clusters'
+    task :drop_all => :environment do
+      ActiveRecord::ShardFor::DatabaseTasks.invoke_task_for_all_clusters('drop')
+    end
+
+    desc 'Load schema to all databases in all clusters'
+    task :load_schema_all => :environment do
+      ActiveRecord::ShardFor::DatabaseTasks.invoke_task_for_all_clusters('load_schema')
+    end
+
+    desc 'Create all databases in specific cluster'
+    task :create, %i(cluster_name) => %i(environment) do |_, args|
+      ActiveRecord::ShardFor::DatabaseTasks.create_all_databases(args)
+    end
+
+    desc 'Drop all databases in specific cluster'
+    task :drop, %i(cluster_name) => %i(environment) do |_, args|
+      ActiveRecord::ShardFor::DatabaseTasks.drop_all_databases(args)
+    end
+
+    desc 'Load schema to all databases in specific cluster'
+    task :load_schema, %i(cluster_name) => %i(environment) do |_, args|
+      ActiveRecord::ShardFor::DatabaseTasks.load_schema_all_databases(args)
+    end
+  end
+end

metadata

@@ -0,0 +1,323 @@
+--- !ruby/object:Gem::Specification
+name: activerecord-shard_for
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- yuemori
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: expeditor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-inline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-rescue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-parameterized
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Database sharding library for ActiveRecord
+email:
+- yuemori@aiming-inc.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Appraisals
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- activerecord-shard_for.gemspec
+- bin/console
+- bin/setup
+- gemfiles/.bundle/config
+- gemfiles/ar_4.1.0.gemfile
+- gemfiles/ar_4.1.0.gemfile.lock
+- gemfiles/ar_4.1.7.gemfile
+- gemfiles/ar_4.1.7.gemfile.lock
+- gemfiles/ar_4.1.8.gemfile
+- gemfiles/ar_4.1.8.gemfile.lock
+- gemfiles/ar_4.2.gemfile
+- gemfiles/ar_4.2.gemfile.lock
+- gemfiles/ar_5.0.gemfile
+- gemfiles/ar_5.0.gemfile.lock
+- gemfiles/rails_edge.gemfile
+- gemfiles/rails_edge.gemfile.lock
+- lib/activerecord/shard_for.rb
+- lib/activerecord/shard_for/all_shards_in_parallel.rb
+- lib/activerecord/shard_for/cluster_config.rb
+- lib/activerecord/shard_for/config.rb
+- lib/activerecord/shard_for/connection_router.rb
+- lib/activerecord/shard_for/database_tasks.rb
+- lib/activerecord/shard_for/errors.rb
+- lib/activerecord/shard_for/hash_modulo_router.rb
+- lib/activerecord/shard_for/model.rb
+- lib/activerecord/shard_for/railtie.rb
+- lib/activerecord/shard_for/replication_mapping.rb
+- lib/activerecord/shard_for/shard_repogitory.rb
+- lib/activerecord/shard_for/version.rb
+- lib/activerecord/tasks/activerecord_shard_for.rake
+homepage: https://github.com/yuemori/activerecord-shard_for
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Database sharding library for ActiveRecord
+test_files: []