ar-octopus-master 0.9.2.master → 0.9.2.1

data/TODO.txt

@@ -0,0 +1,7 @@
+- Find the best way to move data along shards.
+- Improve documentation.
+- Automatic configuration file for shards.
+- Support specifics cases of sharding/replication.
+- Basic algorithm to do load balancing between slaves server (handling servers down)
+- Support replication + sharding
+- Wiki about validation

data/ar-octopus.gemspec

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'octopus/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'ar-octopus-master'
+  s.version     = Octopus::VERSION
+  s.authors     = ['Thiago Pradi', 'Mike Perham', 'Gabriel Sobrinho']
+  s.email       = ['tchandy@gmail.com', 'mperham@gmail.com', 'gabriel.sobrinho@gmail.com']
+  s.homepage    = 'https://github.com/tchandy/octopus'
+  s.summary     = 'Easy Database Sharding for ActiveRecord'
+  s.description = 'This gem allows you to use sharded databases with ActiveRecord. This also provides a interface for replication and for running migrations with multiples shards.'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.post_install_message = "Important: If you are upgrading from < Octopus 0.5.0 you need to run:\n" \
+                           "$ rake octopus:copy_schema_versions\n\n" \
+                           'Octopus now stores schema version information in each shard and migrations will not ' \
+                           'work properly unless this task is invoked.'
+
+  s.required_ruby_version = '>= 2.1.0'
+
+  s.add_dependency 'activerecord', '>= 4.0.0', "< 5.2.0"
+  s.add_dependency 'activesupport', '>= 4.0.0', "< 5.2.0"
+
+  s.add_development_dependency 'appraisal', '>= 0.3.8'
+  s.add_development_dependency 'mysql2', '>= 0.4', "< 0.5"
+  s.add_development_dependency 'pg', '~> 0.18'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec', '>= 3'
+  s.add_development_dependency 'rubocop'
+  s.add_development_dependency 'sqlite3', '>= 1.3.4'
+  s.add_development_dependency 'pry-byebug'
+
+  s.license = 'MIT'
+end

data/gemfiles/rails4.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 4.0.0"
+
+gemspec :path => "../"

data/gemfiles/rails41.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 4.1.0"
+
+gemspec :path => "../"

data/gemfiles/rails42.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 4.2.0"
+
+gemspec :path => "../"

data/gemfiles/rails5.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 5.0.0"
+
+gemspec :path => "../"

data/gemfiles/rails51.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 5.1.0"
+
+gemspec :path => "../"

data/lib/ar-octopus.rb

@@ -0,0 +1 @@
+require 'octopus'

data/lib/octopus.rb

@@ -0,0 +1,205 @@
+require 'active_record'
+require 'active_support/version'
+require 'active_support/core_ext/class'
+
+require 'yaml'
+require 'erb'
+
+module Octopus
+  def self.env
+    @env ||= 'octopus'
+  end
+
+  def self.rails_env
+    @rails_env ||= defined?(::Rails.env) ? Rails.env.to_s : 'shards'
+  end
+
+  def self.config
+    @config ||= begin
+      file_name = File.join(Octopus.directory, 'config/shards.yml').to_s
+
+      if File.exist?(file_name) || File.symlink?(file_name)
+        config ||= HashWithIndifferentAccess.new(YAML.load(ERB.new(File.read(file_name)).result))[Octopus.env]
+      else
+        config ||= HashWithIndifferentAccess.new
+      end
+
+      config
+    end
+  end
+
+  def self.load_balancer=(balancer)
+    @load_balancer = balancer
+  end
+
+  def self.load_balancer
+    @load_balancer ||= Octopus::LoadBalancing::RoundRobin
+  end
+
+  def self.master_shard
+    ((config && config[:master_shard]) || :master).to_sym
+  end
+
+  # Public: Whether or not Octopus is configured and should hook into the
+  # current environment. Checks the environments config option for the Rails
+  # environment by default.
+  #
+  # Returns a boolean
+  def self.enabled?
+    if defined?(::Rails.env)
+      Octopus.environments.include?(Rails.env.to_s)
+    else
+      # TODO: This doens't feel right but !Octopus.config.blank? is breaking a
+      #       test. Also, Octopus.config is always returning a hash.
+      Octopus.config
+    end
+  end
+
+  # Returns the Rails.root_to_s when you are using rails
+  # Running the current directory in a generic Ruby process
+  def self.directory
+    @directory ||= defined?(::Rails.root) ? Rails.root.to_s : Dir.pwd
+  end
+
+  # This is the default way to do Octopus Setup
+  # Available variables:
+  # :enviroments => the enviroments that octopus will run. default: 'production'
+  def self.setup
+    yield self
+  end
+
+  def self.environments=(environments)
+    @environments = environments.map(&:to_s)
+  end
+
+  def self.environments
+    @environments ||= config['environments'] || ['production']
+  end
+
+  def self.robust_environments=(environments)
+    @robust_environments = environments.map(&:to_s)
+  end
+
+  # Environments in which to swallow failures from a single shard
+  # when iterating through all.
+  def self.robust_environments
+    @robust_environments ||= config['robust_environments'] || ['production']
+  end
+
+  def self.robust_environment?
+    robust_environments.include? rails_env
+  end
+
+  def self.rails4?
+    ActiveRecord::VERSION::MAJOR == 4
+  end
+
+  def self.rails40?
+    rails4? && ActiveRecord::VERSION::MINOR == 0
+  end
+
+  def self.rails41_only?
+    rails4? && ActiveRecord::VERSION::MINOR == 1
+  end
+
+  def self.rails41?
+    rails4? && ActiveRecord::VERSION::MINOR >= 1
+  end
+
+  def self.rails42?
+    rails4? && ActiveRecord::VERSION::MINOR == 2
+  end
+
+  def self.rails50?
+    ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR == 0
+  end
+
+  def self.rails51?
+    ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR == 1
+  end
+
+  def self.atleast_rails51?
+    ActiveRecord::VERSION::MAJOR > 5 || (ActiveRecord::VERSION::MAJOR == 5 && ActiveRecord::VERSION::MINOR >= 1)
+  end
+
+  attr_writer :logger
+
+  def self.logger
+    if defined?(Rails.logger)
+      @logger ||= Rails.logger
+    else
+      @logger ||= Logger.new($stderr)
+    end
+  end
+
+  def self.shards=(shards)
+    config[rails_env] = HashWithIndifferentAccess.new(shards)
+    ActiveRecord::Base.connection.initialize_shards(@config)
+  end
+
+  def self.using(shard, &block)
+    conn = ActiveRecord::Base.connection
+
+    if conn.is_a?(Octopus::Proxy)
+      conn.run_queries_on_shard(shard, &block)
+    else
+      yield
+    end
+  end
+
+  def self.using_group(group, &block)
+    conn = ActiveRecord::Base.connection
+
+    if conn.is_a?(Octopus::Proxy)
+      conn.send_queries_to_group(group, &block)
+    else
+      yield
+    end
+  end
+
+  def self.using_all(&block)
+    conn = ActiveRecord::Base.connection
+
+    if conn.is_a?(Octopus::Proxy)
+      conn.send_queries_to_all_shards(&block)
+    else
+      yield
+    end
+  end
+
+  def self.fully_replicated(&_block)
+    old_fully_replicated = Thread.current[Octopus::ProxyConfig::FULLY_REPLICATED_KEY]
+    Thread.current[Octopus::ProxyConfig::FULLY_REPLICATED_KEY] = true
+    yield
+  ensure
+    Thread.current[Octopus::ProxyConfig::FULLY_REPLICATED_KEY] = old_fully_replicated
+  end
+end
+
+require 'octopus/exception'
+
+require 'octopus/shard_tracking'
+require 'octopus/shard_tracking/attribute'
+require 'octopus/shard_tracking/dynamic'
+
+require 'octopus/model'
+require 'octopus/result_patch'
+require 'octopus/migration'
+require 'octopus/association'
+require 'octopus/collection_association'
+require 'octopus/has_and_belongs_to_many_association' unless Octopus.rails41? || Octopus.rails50? || Octopus.rails51?
+require 'octopus/association_shard_tracking'
+require 'octopus/persistence'
+require 'octopus/log_subscriber'
+require 'octopus/abstract_adapter'
+require 'octopus/singular_association'
+require 'octopus/finder_methods'
+require 'octopus/query_cache_for_shards' unless Octopus.rails4?
+
+require 'octopus/railtie' if defined?(::Rails::Railtie)
+
+require 'octopus/proxy_config'
+require 'octopus/proxy'
+require 'octopus/collection_proxy'
+require 'octopus/relation_proxy'
+require 'octopus/scope_proxy'

data/lib/octopus/abstract_adapter.rb

@@ -0,0 +1,33 @@
+# Implementation courtesy of db-charmer.
+module Octopus
+  module AbstractAdapter
+    module OctopusShard
+      class InstrumenterDecorator < ActiveSupport::ProxyObject
+        def initialize(adapter, instrumenter)
+          @adapter = adapter
+          @instrumenter = instrumenter
+        end
+
+        def instrument(name, payload = {}, &block)
+          payload[:octopus_shard] ||= @adapter.octopus_shard
+          @instrumenter.instrument(name, payload, &block)
+        end
+
+        def method_missing(meth, *args, &block)
+          @instrumenter.send(meth, *args, &block)
+        end
+      end
+
+      def octopus_shard
+        @config[:octopus_shard]
+      end
+
+      def initialize(*args)
+        super
+        @instrumenter = InstrumenterDecorator.new(self, @instrumenter)
+      end
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:prepend, Octopus::AbstractAdapter::OctopusShard)

data/lib/octopus/association.rb

@@ -0,0 +1,14 @@
+module Octopus
+  module Association
+    def self.included(base)
+      base.send(:include, Octopus::ShardTracking::Dynamic)
+      base.sharded_methods :target_scope
+    end
+
+    def current_shard
+      owner.current_shard
+    end
+  end
+end
+
+ActiveRecord::Associations::Association.send(:include, Octopus::Association)

data/lib/octopus/association_shard_tracking.rb

@@ -0,0 +1,74 @@
+module Octopus
+  module AssociationShardTracking
+    class MismatchedShards < StandardError
+      attr_reader :record, :current_shard
+
+      def initialize(record, current_shard)
+        @record = record
+        @current_shard = current_shard
+      end
+
+      def message
+        [
+          "Association Error: Records are from different shards",
+          "Record: #{record.inspect}",
+          "Current Shard: #{current_shard.inspect}",
+          "Current Record Shard: #{record.current_shard.inspect}",
+        ].join(" ")
+      end
+    end
+
+    def self.extended(base)
+      base.send(:include, InstanceMethods)
+    end
+
+    module InstanceMethods
+      def connection_on_association=(record)
+        return unless ::Octopus.enabled?
+        return if !self.class.connection.respond_to?(:current_shard) || !self.respond_to?(:current_shard)
+
+        if !record.current_shard.nil? && !current_shard.nil? && record.current_shard.to_s != current_shard.to_s
+          raise MismatchedShards.new(record, current_shard)
+        end
+
+        record.current_shard = self.class.connection.current_shard = current_shard if should_set_current_shard?
+      end
+    end
+
+    def has_many(association_id, scope = nil, options = {}, &extension)
+      if options == {} && scope.is_a?(Hash)
+        default_octopus_opts(scope)
+      else
+        default_octopus_opts(options)
+      end
+      super
+    end
+
+    def has_and_belongs_to_many(association_id, scope = nil, options = {}, &extension)
+      assign_octopus_opts(scope, options)
+      super
+    end
+
+    def default_octopus_opts(options)
+      options[:before_add] = [ :connection_on_association=, options[:before_add] ].compact.flatten
+      options[:before_remove] = [ :connection_on_association=, options[:before_remove] ].compact.flatten
+    end
+
+    def assign_octopus_opts(scope, options)
+      if options == {} && scope.is_a?(Hash)
+        default_octopus_opts(scope)
+      else
+        default_octopus_opts(options)
+      end
+    end
+
+    if Octopus.atleast_rails51?
+      def has_and_belongs_to_many(association_id, scope = nil, **options, &extension)
+        assign_octopus_opts(scope, options)
+        super
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.extend(Octopus::AssociationShardTracking)

data/lib/octopus/collection_association.rb

@@ -0,0 +1,17 @@
+module Octopus
+  module CollectionAssociation
+    def self.included(base)
+      if Octopus.rails51?
+        base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
+                             :build, :include?,
+                             :load_target, :reload, :size, :select
+      else
+        base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
+                             :build, :any?, :count, :empty?, :first, :include?, :last, :length,
+                             :load_target, :many?, :reload, :size, :select, :uniq
+      end
+    end
+  end
+end
+
+ActiveRecord::Associations::CollectionAssociation.send(:include, Octopus::CollectionAssociation)

data/lib/octopus/collection_proxy.rb

@@ -0,0 +1,16 @@
+module Octopus
+  module CollectionProxy
+    def self.included(base)
+      base.send(:include, Octopus::ShardTracking::Dynamic)
+      base.sharded_methods :any?, :build, :count, :create, :create!, :concat, :delete, :delete_all,
+                           :destroy, :destroy_all, :empty?, :find, :first, :include?, :last, :length,
+                           :many?, :pluck, :replace, :select, :size, :sum, :to_a, :uniq
+    end
+
+    def current_shard
+      @association.owner.current_shard
+    end
+  end
+end
+
+ActiveRecord::Associations::CollectionProxy.send(:include, Octopus::CollectionProxy)