misha-ar-octopus 0.8.5

data/TODO.txt

@@ -0,0 +1,7 @@
+- Find the best way to move data along shards.
+- Improve documentation.
+- Automatic configuration file for shards.
+- Support specifics cases of sharding/replication.
+- Basic algorithm to do load balancing between slaves server (handling servers down)
+- Support replication + sharding
+- Wiki about validation

data/ar-octopus.gemspec

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'octopus/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'misha-ar-octopus'
+  s.version     = Octopus::VERSION
+  s.authors     = ['Thiago Pradi', 'Mike Perham', 'Gabriel Sobrinho']
+  s.email       = ['tchandy@gmail.com', 'mperham@gmail.com', 'gabriel.sobrinho@gmail.com']
+  s.homepage    = 'https://github.com/tchandy/octopus'
+  s.summary     = 'Easy Database Sharding for ActiveRecord'
+  s.description = 'This gem allows you to use sharded databases with ActiveRecord. This also provides a interface for replication and for running migrations with multiples shards.'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.post_install_message = "Important: If you are upgrading from < Octopus 0.5.0 you need to run:\n" \
+                           "$ rake octopus:copy_schema_versions\n\n" \
+                           'Octopus now stores schema version information in each shard and migrations will not ' \
+                           'work properly unless this task is invoked.'
+
+  s.add_dependency 'activerecord', '>= 3.2.0'
+  s.add_dependency 'activesupport', '>= 3.2.0'
+
+  s.add_development_dependency 'appraisal', '>= 0.3.8'
+  s.add_development_dependency 'mysql2', '~> 0.3.18'
+  s.add_development_dependency 'pg', '>= 0.11.0'
+  s.add_development_dependency 'rake', '>= 0.8.7'
+  s.add_development_dependency 'rspec', '>= 3'
+  s.add_development_dependency 'rubocop'
+  s.add_development_dependency 'sqlite3', '>= 1.3.4'
+
+  if RUBY_VERSION < '2.0.0'
+    s.add_development_dependency 'pry-debugger'
+  else
+    s.add_development_dependency 'pry-byebug'
+  end
+
+  s.license = 'MIT'
+end

data/gemfiles/rails32.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 3.2.0"
+
+gemspec :path => "../"

data/gemfiles/rails4.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 4.0.0"
+
+gemspec :path => "../"

data/gemfiles/rails41.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 4.1.0"
+
+gemspec :path => "../"

data/gemfiles/rails42.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "~> 4.2.0"
+
+gemspec :path => "../"

data/init.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'rails', 'init')

data/lib/ar-octopus.rb

@@ -0,0 +1 @@
+require 'octopus'

data/lib/octopus.rb

@@ -0,0 +1,181 @@
+require 'active_record'
+require 'active_support/version'
+require 'active_support/core_ext/class'
+
+require 'yaml'
+require 'erb'
+
+module Octopus
+  def self.env
+    @env ||= 'octopus'
+  end
+
+  def self.rails_env
+    @rails_env ||= defined?(::Rails.env) ? Rails.env.to_s : 'shards'
+  end
+
+  def self.config
+    @config ||= begin
+      file_name = File.join(Octopus.directory, 'config/shards.yml').to_s
+
+      if File.exist?(file_name) || File.symlink?(file_name)
+        config ||= HashWithIndifferentAccess.new(YAML.load(ERB.new(File.read(file_name)).result))[Octopus.env]
+      else
+        config ||= HashWithIndifferentAccess.new
+      end
+
+      config
+    end
+  end
+
+  def self.load_balancer=(balancer)
+    @load_balancer = balancer
+  end
+
+  def self.load_balancer
+    @load_balancer ||= Octopus::LoadBalancing::RoundRobin
+  end
+
+  def self.master_shard
+    ((config && config[:master_shard]) || :master).to_sym
+  end
+
+  # Public: Whether or not Octopus is configured and should hook into the
+  # current environment. Checks the environments config option for the Rails
+  # environment by default.
+  #
+  # Returns a boolean
+  def self.enabled?
+    if defined?(::Rails.env)
+      Octopus.environments.include?(Rails.env.to_s)
+    else
+      # TODO: This doens't feel right but !Octopus.config.blank? is breaking a
+      #       test. Also, Octopus.config is always returning a hash.
+      Octopus.config
+    end
+  end
+
+  # Returns the Rails.root_to_s when you are using rails
+  # Running the current directory in a generic Ruby process
+  def self.directory
+    @directory ||= defined?(::Rails.root) ? Rails.root.to_s : Dir.pwd
+  end
+
+  # This is the default way to do Octopus Setup
+  # Available variables:
+  # :enviroments => the enviroments that octopus will run. default: 'production'
+  def self.setup
+    yield self
+  end
+
+  def self.environments=(environments)
+    @environments = environments.map(&:to_s)
+  end
+
+  def self.environments
+    @environments ||= config['environments'] || ['production']
+  end
+
+  def self.robust_environments=(environments)
+    @robust_environments = environments.map(&:to_s)
+  end
+
+  # Environments in which to swallow failures from a single shard
+  # when iterating through all.
+  def self.robust_environments
+    @robust_environments ||= config['robust_environments'] || ['production']
+  end
+
+  def self.robust_environment?
+    robust_environments.include? rails_env
+  end
+
+  def self.rails3?
+    ActiveRecord::VERSION::MAJOR <= 3
+  end
+
+  def self.rails4?
+    ActiveRecord::VERSION::MAJOR >= 4
+  end
+
+  def self.rails41?
+    rails4? && ActiveRecord::VERSION::MINOR >= 1
+  end
+
+  attr_writer :logger
+
+  def self.logger
+    if defined?(Rails.logger)
+      @logger ||= Rails.logger
+    else
+      @logger ||= Logger.new($stderr)
+    end
+  end
+
+  def self.shards=(shards)
+    config[rails_env] = HashWithIndifferentAccess.new(shards)
+    ActiveRecord::Base.connection.initialize_shards(@config)
+  end
+
+  def self.using(shard, options = {}, &block)
+    conn = ActiveRecord::Base.connection
+
+    if conn.is_a?(Octopus::Proxy)
+      conn.run_queries_on_shard(shard, options, &block)
+    else
+      yield
+    end
+  end
+
+  def self.using_group(group, &block)
+    conn = ActiveRecord::Base.connection
+
+    if conn.is_a?(Octopus::Proxy)
+      conn.send_queries_to_group(group, &block)
+    else
+      yield
+    end
+  end
+
+  def self.using_all(&block)
+    conn = ActiveRecord::Base.connection
+
+    if conn.is_a?(Octopus::Proxy)
+      conn.send_queries_to_all_shards(&block)
+    else
+      yield
+    end
+  end
+
+  def self.fully_replicated(&_block)
+    old_fully_replicated = Thread.current[Octopus::Proxy::FULLY_REPLICATED_KEY]
+    Thread.current[Octopus::Proxy::FULLY_REPLICATED_KEY] = true
+    yield
+  ensure
+    Thread.current[Octopus::Proxy::FULLY_REPLICATED_KEY] = old_fully_replicated
+  end
+end
+
+require 'octopus/exception'
+
+require 'octopus/shard_tracking'
+require 'octopus/shard_tracking/attribute'
+require 'octopus/shard_tracking/dynamic'
+
+require 'octopus/model'
+require 'octopus/migration'
+require 'octopus/association'
+require 'octopus/collection_association'
+require 'octopus/has_and_belongs_to_many_association' unless Octopus.rails41?
+require 'octopus/association_shard_tracking'
+require 'octopus/persistence'
+require 'octopus/log_subscriber'
+require 'octopus/abstract_adapter'
+require 'octopus/singular_association'
+
+require 'octopus/railtie' if defined?(::Rails::Railtie)
+
+require 'octopus/proxy'
+require 'octopus/collection_proxy'
+require 'octopus/relation_proxy'
+require 'octopus/scope_proxy'

data/lib/octopus/abstract_adapter.rb

@@ -0,0 +1,35 @@
+# Implementation courtesy of db-charmer.
+module Octopus
+  module AbstractAdapter
+    module OctopusShard
+      parent = Octopus.rails3? ? ActiveSupport::BasicObject : ActiveSupport::ProxyObject
+
+      class InstrumenterDecorator < parent
+        def initialize(adapter, instrumenter)
+          @adapter = adapter
+          @instrumenter = instrumenter
+        end
+
+        def instrument(name, payload = {}, &block)
+          payload[:octopus_shard] ||= @adapter.octopus_shard
+          @instrumenter.instrument(name, payload, &block)
+        end
+
+        def method_missing(meth, *args, &block)
+          @instrumenter.send(meth, *args, &block)
+        end
+      end
+
+      def octopus_shard
+        @config[:octopus_shard]
+      end
+
+      def initialize(*args)
+        super
+        @instrumenter = InstrumenterDecorator.new(self, @instrumenter)
+      end
+    end
+  end
+end
+
+ActiveRecord::ConnectionAdapters::AbstractAdapter.send(:prepend, Octopus::AbstractAdapter::OctopusShard)

data/lib/octopus/association.rb

@@ -0,0 +1,13 @@
+module Octopus
+  module Association
+    def self.included(base)
+      base.send(:include, Octopus::ShardTracking::Dynamic)
+    end
+
+    def current_shard
+      owner.current_shard
+    end
+  end
+end
+
+ActiveRecord::Associations::Association.send(:include, Octopus::Association)

data/lib/octopus/association_shard_tracking.rb

@@ -0,0 +1,114 @@
+module Octopus
+  module AssociationShardTracking
+    def self.extended(base)
+      base.send(:include, InstanceMethods)
+    end
+
+    module QueryOnCurrentShard
+      METHODS = %w(
+        all
+        average
+        count
+        empty?
+        exists?
+        find
+        find_by_sql
+        first
+        last
+        maximum
+        minimum
+        pluck
+        scoping
+        size
+        sum
+        to_a
+      )
+
+      METHODS.each do |m|
+        if Octopus.rails4?
+          define_method m.to_sym do |*args, &block|
+            if defined?(@association) && @association
+              @association.owner.run_on_shard { super(*args, &block) }
+            else
+              super(*args, &block)
+            end
+          end
+        else
+          define_method m.to_sym do |*args, &block|
+            if respond_to?(:proxy_association) && proxy_association
+              proxy_association.owner.run_on_shard { super(*args, &block) }
+            else
+              super(*args, &block)
+            end
+          end
+        end
+      end
+    end
+
+    module InstanceMethods
+      def connection_on_association=(record)
+        return unless ::Octopus.enabled?
+        return if !self.class.connection.respond_to?(:current_shard) || !self.respond_to?(:current_shard)
+        if !record.current_shard.nil? && !current_shard.nil? && record.current_shard != current_shard
+          fail 'Association Error: Records are from different shards'
+        end
+
+        record.current_shard = self.class.connection.current_shard = current_shard if should_set_current_shard?
+      end
+    end
+
+    if Octopus.rails4?
+      def has_many(association_id, scope = nil, options = {}, &extension)
+        if options == {} && scope.is_a?(Hash)
+          default_octopus_opts(scope)
+        else
+          default_octopus_opts(options)
+        end
+        super
+      end
+    else
+      def has_many(association_id, options = {}, &extension)
+        default_octopus_opts(options)
+        super
+      end
+    end
+
+    if Octopus.rails4?
+      def has_and_belongs_to_many(association_id, scope = nil, options = {}, &extension)
+        if options == {} && scope.is_a?(Hash)
+          default_octopus_opts(scope)
+        else
+          default_octopus_opts(options)
+        end
+        super
+      end
+    else
+      def has_and_belongs_to_many(association_id, options = {}, &extension)
+        default_octopus_opts(options)
+        super
+      end
+    end
+
+    def default_octopus_opts(options)
+      if options[:before_add].is_a?(Array)
+        options[:before_add] << :connection_on_association=
+      elsif options[:before_add].is_a?(Symbol)
+        options[:before_add] = [:connection_on_association=, options[:before_add]]
+      else
+        options[:before_add] = :connection_on_association=
+      end
+
+      if options[:before_remove].is_a?(Array)
+        options[:before_remove] << :connection_on_association=
+      elsif options[:before_remove].is_a?(Symbol)
+        options[:before_remove] = [:connection_on_association=, options[:before_remove]]
+      else
+        options[:before_remove] = :connection_on_association=
+      end
+
+      options[:extend] = [Octopus::AssociationShardTracking::QueryOnCurrentShard, options[:extend]].flatten.compact
+    end
+  end
+end
+
+ActiveRecord::Base.extend(Octopus::AssociationShardTracking)

data/lib/octopus/collection_association.rb

@@ -0,0 +1,11 @@
+module Octopus
+  module CollectionAssociation
+    def self.included(base)
+      base.sharded_methods :reader, :writer, :ids_reader, :ids_writer, :create, :create!,
+                           :build, :any?, :count, :empty?, :first, :include?, :last, :length,
+                           :load_target, :many?, :reload, :size, :select, :uniq
+    end
+  end
+end
+
+ActiveRecord::Associations::CollectionAssociation.send(:include, Octopus::CollectionAssociation)