rails-sharding 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cbec123e1537bac8bf7f3b6d2e711cfefa156f41
-  data.tar.gz: 4fae6ab76f8b687746436585a74d618c5f4a2308
+  metadata.gz: 58f0c90d4f03a5f3aa3d4ff363cdac165c948f8f
+  data.tar.gz: d927face94cf2c001534c1353b322294ab1984b8
 SHA512:
-  metadata.gz: f8488d56f2e5eb38df46f138d4ad7ac28bc12008ee08383f984268fd3201629bb3eebeb54c96120bddf846e2e1f39f171e38fc4ab52457bbcdbcafa805b0e72c
-  data.tar.gz: 95a78672f724efa645c911d0d824b57be1dcf43d96257be54aab15e92ba544b81ce370e3a9131ee5abdd27f4605a8a49a5e45dd66dd00f4e4f58b3c7dfc48f28
+  metadata.gz: aa192b584a7ef8e58be570798c8d35ae04f17134791ed325a272f32dc5c32d3d847e60d06894da35093736d7c29f52bba32414b4d27b6d5107e2390e86c43221
+  data.tar.gz: cd329c1e3473f0f8a11f93bfc15bdd5278ebb041b8fccf6d5e2b7b496fa229919914b4080e4d97913e61ad090c6b67dfdf2a9c22e729cace4129e4f63e92bf95

data/README.md

@@ -4,8 +4,9 @@
 [![Code Climate](https://codeclimate.com/github/hsgubert/rails-sharding/badges/gpa.svg)](https://codeclimate.com/github/hsgubert/rails-sharding)
 [![Test Coverage](https://codeclimate.com/github/hsgubert/rails-sharding/badges/coverage.svg)](https://codeclimate.com/github/hsgubert/rails-sharding/coverage)
 [![Gem Version](https://badge.fury.io/rb/rails-sharding.svg)](https://badge.fury.io/rb/rails-sharding)
+[![Dependency Status](https://gemnasium.com/badges/github.com/hsgubert/rails-sharding.svg)](https://gemnasium.com/github.com/hsgubert/rails-sharding)
 
-Simple and robust sharding for Rails, including Migrations and ActiveRecord extensions
+Simple and robust sharding gem for Rails, including Migrations and ActiveRecord extensions
 
 This gems allows you to easily create extra databases to your rails application, and freely allocate ActiveRecord instances to any of the databases. It also provides rake tasks and migrations to help you manage the schema by shard groups.
 
@@ -27,8 +28,10 @@ You can also use the block syntax, where all your queries inside will be directe
 You can also pick and choose which models will be shardable, so that all the models that are not shardable will still be retrieved from the master database, even if inside a using_shard block.
 
 ## Compatibility
+Gem version 1.x.x -> compatible with Rails 5.0
+
 Gem version 0.1.1 -> compatible with Rails 4.2
-Gem version 1.0.0 -> compatible with Rails 5.0
+
 
 ## Installation
 
@@ -141,6 +144,29 @@ rake shards:migrate SHARD_GROUP=shard_group_1
 rake shards:migrate SHARD_GROUP=shard_group_1 SHARD=shard1
 ```
 
+## Gem Options
+Running the `rails g rails_sharding:scaffold` will create an initializer at `config/initializers/rails-sharding.rb`. You can pass additional configurations on this initializer to control the gem behavior. You can see below all available options and their default values:
+```ruby
+# config/initializers/rails-sharding.rb
+
+Rails::Sharding.setup do |config|
+  # If true one connection will be established per shard (in every shard group) on startup.
+  # If false the user must call Shards::ConnectionHandler.establish_connection(shard_group, shard_name) manually at least once before using each shard.
+  config.establish_all_connections_on_setup = true
+
+  # If true the method #using_shard will be mixed in ActiveRecord scopes. Put this to false if you don't want the gem to modify ActiveRecord
+  config.extend_active_record_scope = true,
+
+  # Specifies where to find the definition of the shards configurations
+  config.shards_config_file = 'config/shards.yml',
+
+  # Specifies where to find the migrations for each shard group
+  config.shards_migrations_dir = 'db/shards_migrations',
+
+  # Specifies where to find the schemas for each shard group
+  config.shards_schemas_dir = 'db/shards_schemas'
+end
+```
 
 ## Development and Contributing
 

data/Rakefile

@@ -6,7 +6,8 @@ RSpec::Core::RakeTask.new(:spec)
 task :default => :spec
 
 # defines an environment task so we can run rake tasks from lib/tasks/rails-sharding.rake.
-# They require the :environment task, which is defined by rails, but we don need anything
+# The tasks on rails-sharding.rake depend on the :environment task, which is usuallu defined
+# by rails. In our case, we just stub it so the rake tasks run
 task :environment do
   # do nothing
 end
@@ -16,12 +17,7 @@ namespace :db do
 
     desc 'Loads gem test environment and rake tasks from gem'
     task :load_env do
-      require 'rspec'
-
-      # requires spec helper but ensures no test coverage is reported to codeclimate
-      ENV['CODECLIMATE_REPO_TOKEN'] = nil
-      require './spec/spec_helper'
-      
+      require './spec/load_gem_test_env'
       load 'lib/tasks/rails-sharding.rake'
     end
 

data/lib/generators/templates/shards.yml.example

@@ -1,7 +1,7 @@
 default: &default
   adapter: mysql2
   encoding: utf8
-  reconnect: false
+  reconnect: true
   pool: 5
   username: ___
   password: ___

data/lib/rails/sharding/config.rb

@@ -13,6 +13,16 @@ module Rails::Sharding
       # this to false if you don't want the gem to modify ActiveRecord
       extend_active_record_scope: true,
 
+      # If true the query logs of ActiveRecord will be tagged with the corresponding
+      # shard you're querying
+      add_shard_tag_to_query_logs: true,
+
+      # If true a warning will be printed everytime a using_shard block ends without
+      # the shard connection being retrieved at least once inside the block. This warning
+      # is helpful to remember the developer to include ShardableModel module on the
+      # sharded models, otherwise they will always connect to the master database.
+      no_connection_retrieved_warning: true,
+
       # Specifies where to find the definition of the shards configurations
       shards_config_file: 'config/shards.yml',
 
@@ -27,6 +37,5 @@ module Rails::Sharding
       self.cattr_accessor config_name
       self.send(config_name.to_s + '=', default_value)
     end
-
   end
 end

data/lib/rails/sharding/connection_handler.rb

@@ -49,7 +49,14 @@ module Rails::Sharding
     end
 
     def self.retrieve_connection(shard_group, shard_name)
-      connection_handler.retrieve_connection(connection_name(shard_group, shard_name))
+      connection_name = connection_name(shard_group, shard_name)
+      connection = connection_handler.retrieve_connection(connection_name)
+
+      if connection && Config.add_shard_tag_to_query_logs
+        add_shard_tag_to_connection_log(connection, connection_name)
+      else
+        connection
+      end
     end
 
     def self.connected?(shard_group, shard_name)
@@ -57,7 +64,13 @@ module Rails::Sharding
     end
 
     def self.with_connection(shard_group, shard_name, &block)
-    	connection_pool(shard_group, shard_name).with_connection(&block)
+    	connection_pool(shard_group, shard_name).with_connection do |connection|
+        if connection && Config.add_shard_tag_to_query_logs
+          connection_name = connection_name(shard_group, shard_name)
+          add_shard_tag_to_connection_log(connection, connection_name)
+        end
+        block.call(connection)
+      end
     end
 
     def self.remove_connection(shard_group, shard_name)
@@ -79,5 +92,31 @@ module Rails::Sharding
     def self.connection_name(shard_group, shard_name)
       shard_group.to_s + ':' + shard_name.to_s
     end
+
+    # Adds a shard tag to the log of all queries executed through this connection
+    def self.add_shard_tag_to_connection_log(connection, shard_tag)
+      # avoids modifing connection twice
+      if connection.respond_to? :shard_tag
+        connection.shard_tag = shard_tag
+        return connection
+      end
+
+      # creates #shard_tag attribute in connection
+      connection.singleton_class.send(:attr_accessor, :shard_tag)
+      connection.shard_tag = shard_tag
+
+      # create an alias #original_execute, as a copy of the #execute for this connection
+      connection.singleton_class.send(:alias_method, :original_execute, :execute)
+
+      # defines a new #execute that adds a tag to the log
+      class << connection
+        def execute(sql, name=nil)
+          name = (name.to_s + " (#{shard_tag})").strip
+          self.original_execute(sql, name)
+        end
+      end
+
+      connection
+    end
   end
 end

data/lib/rails/sharding/core.rb

@@ -10,28 +10,29 @@ module Rails::Sharding
 
     # Opens a block where all queries will be directed to the selected shard
     def self.using_shard(shard_group, shard_name)
-      raise 'Cannot nest using_shard blocks' if ShardThreadRegistry.connecting_to_shard?
-
-      ShardThreadRegistry.current_shard_group = shard_group
-      ShardThreadRegistry.current_shard_name = shard_name
-      ShardThreadRegistry.shard_connection_used = false
+      ShardThreadRegistry.push_current_shard(shard_group, shard_name)
       yield
     ensure
+      shard_group, shard_name, connection_used = ShardThreadRegistry.pop_current_shard
+
       # shows warning to user
-      if !ShardThreadRegistry.shard_connection_used
-        puts "Warning: no connection to shard '#{ShardThreadRegistry.current_shard_group}:#{ShardThreadRegistry.current_shard_name}' was made inside the using_shard block. Make sure you don't forget to include Rails::Sharding::ShardableModel to the models you want to be sharded"
+      if Config.no_connection_retrieved_warning && !connection_used
+        puts "Warning: no connection to shard '#{shard_group}:#{shard_name}' was retrieved inside the using_shard block. Make sure you don't forget to include Rails::Sharding::ShardableModel to the models you want to be sharded. Disable this warning with Rails::Sharding::Config.no_connection_retrieved_warning = false."
       end
 
       # Releases connections in case user left some connection in the reserved state
       # (by calling retrieve_connection instead of with_connection). Also, using
       # normal activerecord queries leaves a connection in the reserved state
-      ConnectionHandler.connection_pool(*ShardThreadRegistry.current_shard_group_and_name).release_connection
-      ShardThreadRegistry.connect_back_to_master!
+      # Obs: don't do this with a master database connection
+      ConnectionHandler.connection_pool(shard_group, shard_name).release_connection if shard_group && shard_name
     end
 
     def self.configurations(environment=Rails.env)
       @@db_configs ||= YAML.load_file(Config.shards_config_file)
-      @@db_configs[environment]
+      environment_config = @@db_configs[environment]
+      return environment_config if environment_config
+
+      raise Errors::ConfigNotFoundError, 'Found no shard configurations for enviroment "' + environment + '" in ' + Config.shards_config_file.to_s + ' file was not found'
     rescue Errno::ENOENT
       raise Errors::ConfigNotFoundError, Config.shards_config_file.to_s + ' file was not found'
     end

data/lib/rails/sharding/shard_thread_registry.rb

@@ -1,13 +1,21 @@
 
 module Rails::Sharding
   class ShardThreadRegistry
-    # creates two thread-specific variables to store the current shard of connection
-    thread_mattr_accessor :_current_shard_group
-    thread_mattr_accessor :_current_shard_name
+    # Creates two thread-specific stacks to store the shard of connection
+    # The top of the stack indicates the current connection
+    # This allows us to have nested blocks of #using_shard and keep track of the
+    # connections as we open/close those blocks
+    thread_mattr_accessor :_shard_group_stack
+    thread_mattr_accessor :_shard_name_stack
 
-    # auxiliary variable used to check if shard connectio was used inside an
-    # using_shard block (so we can print an alert if not)
-    thread_mattr_accessor :shard_connection_used
+    # auxiliary stack that keeps track of wether each shard connection was used
+    # inside its respective using_shard block (so we can print an alert if not)
+    thread_mattr_accessor :_shard_connection_used_stack
+
+    # accessors that initialize stacks if necessary
+    def self.shard_group_stack; self._shard_group_stack ||= [] end;
+    def self.shard_name_stack; self._shard_name_stack ||= [] end;
+    def self.shard_connection_used_stack; self._shard_connection_used_stack ||= [] end;
 
     def self.connecting_to_master?
       current_shard_group.nil? || current_shard_name.nil?
@@ -17,30 +25,48 @@ module Rails::Sharding
       !connecting_to_master?
     end
 
+    # Clears the connection stack and goes back to connecting to master
     def self.connect_back_to_master!
-      self.current_shard_group = nil
-      self.current_shard_name = nil
-      self.shard_connection_used = false
+      shard_group_stack.clear
+      shard_name_stack.clear
+      shard_connection_used_stack.clear
     end
 
     # Returns the current shard group (for the current Thread)
     def self.current_shard_group
-      _current_shard_group
-    end
-
-    # Sets the current shard group (for the current Thread)
-    def self.current_shard_group=(group)
-      self._current_shard_group = group.blank? ? nil : group.to_sym
+      shard_group_stack.last
     end
 
     # Returns the current shard name (for the current Thread)
     def self.current_shard_name
-      _current_shard_name
+      shard_name_stack.last
+    end
+
+    def self.current_connection_used?
+      shard_connection_used_stack.last
+    end
+
+    # adds shard connection to the stack
+    def self.push_current_shard(group, name)
+      # this line supresses the unused connection warning when there are nested
+      # using_shard blocks. We suppress the warning because we view nested using_shard
+      # blocks as a override
+      notify_connection_retrieved
+
+      shard_group_stack.push(group.blank? ? nil : group.to_sym)
+      shard_name_stack.push(name.blank? ? nil : name.to_sym)
+      shard_connection_used_stack.push(false)
+    end
+
+    # notifies the current connection was used (wee keep track of this to warn
+    # the user in case the connection is not used)
+    def self.notify_connection_retrieved
+      shard_connection_used_stack[-1] = true if shard_connection_used_stack.present?
     end
 
-    # Sets the current shard name (for the current Thread)
-    def self.current_shard_name=(name)
-      self._current_shard_name = name.blank? ? nil : name.to_sym
+    # removes shard connection to the stack
+    def self.pop_current_shard
+      [shard_group_stack.pop, shard_name_stack.pop, shard_connection_used_stack.pop]
     end
 
     def self.current_shard_group_and_name

data/lib/rails/sharding/shardable_model.rb

@@ -32,20 +32,22 @@ module Rails::Sharding
 
       # @overrides ActiveRecord::ConnectionHandling#connection_pool
       def sharded_connection_pool
+        ShardThreadRegistry.notify_connection_retrieved
+
         if ShardThreadRegistry.connecting_to_master?
           return original_connection_pool
         else
-          ShardThreadRegistry.shard_connection_used = true # records that shard connection was used at least once
           return ConnectionHandler.connection_pool(*ShardThreadRegistry.current_shard_group_and_name)
         end
       end
 
       # @overrides ActiveRecord::ConnectionHandling#retrieve_connection
       def sharded_retrieve_connection
+        ShardThreadRegistry.notify_connection_retrieved
+
         if ShardThreadRegistry.connecting_to_master?
           return original_retrieve_connection
         else
-          ShardThreadRegistry.shard_connection_used = true # records that shard connection was used at least once
           return ConnectionHandler.retrieve_connection(*ShardThreadRegistry.current_shard_group_and_name)
         end
       end

data/lib/rails/sharding/version.rb

@@ -1,5 +1,5 @@
 module Rails
   module Sharding
-    VERSION = "1.0.0"
+    VERSION = "1.0.1"
   end
 end

data/rails-sharding.gemspec

@@ -26,9 +26,9 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'rails', '~> 5.0'
 
   spec.add_development_dependency "bundler", "~> 1.12"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 11.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "byebug", '~> 0'
+  spec.add_development_dependency "byebug", '~> 9'
   spec.add_development_dependency "mysql2", '~> 0'
   spec.add_development_dependency "codeclimate-test-reporter", '~> 0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails-sharding
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Henrique Gubert
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-10 00:00:00.000000000 Z
+date: 2016-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '11.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '11.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '9'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement