activerecord_autoreplica 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.document +5 -0
- data/.gitlab-ci.yml +28 -0
- data/Gemfile +19 -0
- data/LICENSE.txt +21 -0
- data/README.md +90 -0
- data/Rakefile +35 -0
- data/activerecord_autoreplica.gemspec +67 -0
- data/gemfiles/Gemfile.rails-3.2.x +12 -0
- data/gemfiles/Gemfile.rails-4.1.x +12 -0
- data/lib/activerecord_autoreplica.rb +150 -0
- data/spec/activerecord_autoreplica_spec.rb +179 -0
- data/spec/helper.rb +12 -0
- metadata +141 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 4842ab6d5f5cbf12c929f25b2933aca4eb6c034e
|
4
|
+
data.tar.gz: 94caf1b3480ea4ca6170571dc2d264e1d85f1e2d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e199dcaaa96886e1aeb7d10f022b840280abea1fd00f0ce4d114c6db332e12a81c19e71e43cf1af5cd7327c81df44d626f7fe4e99925653a0e51f98f98e89375
|
7
|
+
data.tar.gz: 475170a5a2e715a729d9da07c5a022866234df582e1caaa0930c318ea9b2c7b31e3ceb722d352b9eaec4735acdfe7eeb561175ad7c7e6b5579f18ce88cdca61b
|
data/.document
ADDED
data/.gitlab-ci.yml
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# This file is generated by GitLab CI
|
2
|
+
rake-ar4:
|
3
|
+
script:
|
4
|
+
- "apt-get update && apt-get install -y libsqlite3-dev"
|
5
|
+
- git submodule update --init
|
6
|
+
- ls -la
|
7
|
+
- gem install bundler
|
8
|
+
- apt-get install libsqlite3-dev -y
|
9
|
+
- bundle config --global jobs 4
|
10
|
+
- bundle config --global path /cache/gems
|
11
|
+
- bundle config build.nokogiri "--use-system-libraries --with-xml2-include=/usr/include/libxml2"
|
12
|
+
- BUNDLE_GEMFILE=gemfiles/Gemfile.rails-4.1.x bundle install
|
13
|
+
- BUNDLE_GEMFILE=gemfiles/Gemfile.rails-4.1.x bundle exec rake
|
14
|
+
tags:
|
15
|
+
- ruby
|
16
|
+
rake-ar3:
|
17
|
+
script:
|
18
|
+
- "apt-get update && apt-get install -y libsqlite3-dev"
|
19
|
+
- git submodule update --init
|
20
|
+
- ls -la
|
21
|
+
- gem install bundler
|
22
|
+
- bundle config --global jobs 4
|
23
|
+
- bundle config --global path /cache/gems
|
24
|
+
- bundle config build.nokogiri "--use-system-libraries --with-xml2-include=/usr/include/libxml2"
|
25
|
+
- BUNDLE_GEMFILE=gemfiles/Gemfile.rails-3.2.x bundle install
|
26
|
+
- BUNDLE_GEMFILE=gemfiles/Gemfile.rails-3.2.x bundle exec rake
|
27
|
+
tags:
|
28
|
+
- ruby
|
data/Gemfile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
# We test both with AR 4 and AR3, see gemfiles/ for more.
|
4
|
+
# To run specs against specific versions of dependencies, use
|
5
|
+
#
|
6
|
+
# $ BUNDLE_GEMFILE=gemfiles/Gemfile.rails-4.1.x bundle exec rake
|
7
|
+
#
|
8
|
+
# etc.
|
9
|
+
gem 'activerecord', "> 3.0"
|
10
|
+
|
11
|
+
# Add dependencies to develop your gem here.
|
12
|
+
# Include everything needed to run rake, tests, features, etc.
|
13
|
+
group :development do
|
14
|
+
gem 'sqlite3'
|
15
|
+
gem "rspec", "~> 2.4"
|
16
|
+
gem "rdoc", "~> 3.12"
|
17
|
+
gem "bundler", "~> 1.0"
|
18
|
+
gem "jeweler"
|
19
|
+
end
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2014 Julik Tarkhanov for WeTransfer
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
21
|
+
|
data/README.md
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
# activerecord_autoreplica
|
2
|
+
|
3
|
+
An automatic ActiveRecord connection multiplexer, and is a reimplementation of / greatly inspired by
|
4
|
+
the [makara gem](https://github.com/taskrabbit/makara)
|
5
|
+
|
6
|
+
Automatically redirects your `SELECT` queries to a different database connection. Can be mighty
|
7
|
+
useful when you have a read replica defined and you want to make use of it for reporting or separate
|
8
|
+
tasks.
|
9
|
+
|
10
|
+
Does not require you to change the format of your `database.yml` or whatnot. Does not support replica weighting,
|
11
|
+
randomization, middleware contexts and other things that I do not need - it is very imperative and can be read
|
12
|
+
and changed in one sitting.
|
13
|
+
|
14
|
+
The only dependency is ActiveRecord itself.
|
15
|
+
|
16
|
+
### Usage
|
17
|
+
|
18
|
+
The API consists of _one_ method. Pass it a complete ActiveRecord connection specificaton hash, and
|
19
|
+
everything within the block is going to use the read slave connections when performing standard
|
20
|
+
ActiveRecord `SELECT` queries (not the hand-written ones).
|
21
|
+
|
22
|
+
AutoReplica.using_read_replica_at(:adapter => 'mysql2', :datbase => 'read_replica', ...) do
|
23
|
+
customer = Customer.find(3) # Will SELECT from the replica database at the connection spec passed to the block
|
24
|
+
customer.register_complaint! # Will UPDATE to the master database connection
|
25
|
+
end
|
26
|
+
|
27
|
+
Connection strings (URLs) are also supported, just like in ActiveRecord itself:
|
28
|
+
|
29
|
+
AutoReplica.using_read_replica_at('sqlite3:/replica_db_145.sqlite3') do
|
30
|
+
...
|
31
|
+
end
|
32
|
+
|
33
|
+
To use in Rails controller context (for all actions of this controller):
|
34
|
+
|
35
|
+
class SlowDataReportsController < ApplicationController
|
36
|
+
around_filter ->{
|
37
|
+
AutoReplica.using_read_replica_at(...) { yield }
|
38
|
+
}
|
39
|
+
...
|
40
|
+
end
|
41
|
+
|
42
|
+
To use in Rack middleware context:
|
43
|
+
|
44
|
+
# Make sure to mount this downstream from ActiveRecord::ConnectionManagement in Rails
|
45
|
+
def call(env)
|
46
|
+
AutoReplica.using_read_replica_at(...) { @app.call(env) }
|
47
|
+
end
|
48
|
+
|
49
|
+
Currently there are no runtime switches - once you are _in_ the block the `SELECT` queries composed via Arel will
|
50
|
+
all go to the read replica, no exception. If you do not want that - just exit the block.
|
51
|
+
|
52
|
+
The library does not make any assumptions about whether your data is up to date on the read slave versus master, so
|
53
|
+
act accordingly.
|
54
|
+
|
55
|
+
The `using_read_replica_at` block will allocate a `ConnectionPool` like the standard `ActiveRecord` connection
|
56
|
+
manager does, and the pool is going to be closed and torn down at the end of the block. Since it only uses the basic
|
57
|
+
ActiveRecord facilities (including mutexes) it should be threadsafe (but _not_ thread-local since the connection
|
58
|
+
handler in ActiveRecord isn't).
|
59
|
+
|
60
|
+
### Running the specs
|
61
|
+
|
62
|
+
You will only need sqlite3 and it's gem. Separate database files are going to be created for the master and replica and those
|
63
|
+
files are going to be erased on test completion.
|
64
|
+
|
65
|
+
There are Gemfiles for testing against various versions of ActiveRecord. To run those, use:
|
66
|
+
|
67
|
+
$ BUNDLE_GEMFILE=gemfiles/Gemfile.rails-4.1.x bundle exec rake
|
68
|
+
$ BUNDLE_GEMFILE=gemfiles/Gemfile.rails-3.2.x bundle exec rake
|
69
|
+
|
70
|
+
The library contains a couple of switches that allow it to function in both Rails versions.
|
71
|
+
|
72
|
+
### Versioning
|
73
|
+
|
74
|
+
The gem version is specified in the Rakefile.
|
75
|
+
|
76
|
+
### Contributing to activerecord_autoreplica
|
77
|
+
|
78
|
+
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
79
|
+
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
|
80
|
+
* Fork the project.
|
81
|
+
* Start a feature/bugfix branch.
|
82
|
+
* Commit and push until you are happy with your contribution.
|
83
|
+
* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
|
84
|
+
* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
|
85
|
+
|
86
|
+
### Copyright
|
87
|
+
|
88
|
+
Copyright (c) 2014 Julik Tarkhanov for WeTransfer. See LICENSE.txt for
|
89
|
+
further details.
|
90
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
begin
|
6
|
+
Bundler.setup(:default, :development)
|
7
|
+
rescue Bundler::BundlerError => e
|
8
|
+
$stderr.puts e.message
|
9
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
10
|
+
exit e.status_code
|
11
|
+
end
|
12
|
+
require 'rake'
|
13
|
+
|
14
|
+
require 'jeweler'
|
15
|
+
Jeweler::Tasks.new do |gem|
|
16
|
+
gem.version = '1.1.0'
|
17
|
+
# gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
|
18
|
+
gem.name = "activerecord_autoreplica"
|
19
|
+
gem.homepage = "http://github.com/WeTransfer/activerecord_autoreplica"
|
20
|
+
gem.license = "MIT"
|
21
|
+
gem.summary = %Q{ Palatable-size read replica adapter for ActiveRecord }
|
22
|
+
gem.description = %Q{ Redirect all SELECT queries to a separate connection within a block }
|
23
|
+
gem.email = "me@julik.nl"
|
24
|
+
gem.authors = ["Julik Tarkhanov"]
|
25
|
+
# dependencies defined in Gemfile
|
26
|
+
end
|
27
|
+
|
28
|
+
Jeweler::RubygemsDotOrgTasks.new
|
29
|
+
|
30
|
+
require 'rspec/core/rake_task'
|
31
|
+
RSpec::Core::RakeTask.new(:spec) do |t|
|
32
|
+
t.rspec_opts = ["-c", "-f progress", "-r ./spec/helper.rb"]
|
33
|
+
t.pattern = 'spec/**/*_spec.rb'
|
34
|
+
end
|
35
|
+
task :default => :spec
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
# stub: activerecord_autoreplica 1.1.0 ruby lib
|
6
|
+
|
7
|
+
Gem::Specification.new do |s|
|
8
|
+
s.name = "activerecord_autoreplica"
|
9
|
+
s.version = "1.1.0"
|
10
|
+
|
11
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
|
+
s.require_paths = ["lib"]
|
13
|
+
s.authors = ["Julik Tarkhanov"]
|
14
|
+
s.date = "2015-10-19"
|
15
|
+
s.description = " Redirect all SELECT queries to a separate connection within a block "
|
16
|
+
s.email = "me@julik.nl"
|
17
|
+
s.extra_rdoc_files = [
|
18
|
+
"LICENSE.txt",
|
19
|
+
"README.md"
|
20
|
+
]
|
21
|
+
s.files = [
|
22
|
+
".document",
|
23
|
+
".gitlab-ci.yml",
|
24
|
+
"Gemfile",
|
25
|
+
"LICENSE.txt",
|
26
|
+
"README.md",
|
27
|
+
"Rakefile",
|
28
|
+
"activerecord_autoreplica.gemspec",
|
29
|
+
"gemfiles/Gemfile.rails-3.2.x",
|
30
|
+
"gemfiles/Gemfile.rails-4.1.x",
|
31
|
+
"lib/activerecord_autoreplica.rb",
|
32
|
+
"spec/activerecord_autoreplica_spec.rb",
|
33
|
+
"spec/helper.rb"
|
34
|
+
]
|
35
|
+
s.homepage = "http://github.com/WeTransfer/activerecord_autoreplica"
|
36
|
+
s.licenses = ["MIT"]
|
37
|
+
s.rubygems_version = "2.2.2"
|
38
|
+
s.summary = "Palatable-size read replica adapter for ActiveRecord"
|
39
|
+
|
40
|
+
if s.respond_to? :specification_version then
|
41
|
+
s.specification_version = 4
|
42
|
+
|
43
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
44
|
+
s.add_runtime_dependency(%q<activerecord>, ["> 3.0"])
|
45
|
+
s.add_development_dependency(%q<sqlite3>, [">= 0"])
|
46
|
+
s.add_development_dependency(%q<rspec>, ["~> 2.4"])
|
47
|
+
s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
|
48
|
+
s.add_development_dependency(%q<bundler>, ["~> 1.0"])
|
49
|
+
s.add_development_dependency(%q<jeweler>, [">= 0"])
|
50
|
+
else
|
51
|
+
s.add_dependency(%q<activerecord>, ["> 3.0"])
|
52
|
+
s.add_dependency(%q<sqlite3>, [">= 0"])
|
53
|
+
s.add_dependency(%q<rspec>, ["~> 2.4"])
|
54
|
+
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
55
|
+
s.add_dependency(%q<bundler>, ["~> 1.0"])
|
56
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
57
|
+
end
|
58
|
+
else
|
59
|
+
s.add_dependency(%q<activerecord>, ["> 3.0"])
|
60
|
+
s.add_dependency(%q<sqlite3>, [">= 0"])
|
61
|
+
s.add_dependency(%q<rspec>, ["~> 2.4"])
|
62
|
+
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
63
|
+
s.add_dependency(%q<bundler>, ["~> 1.0"])
|
64
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
gem 'activerecord', "~> 3"
|
3
|
+
|
4
|
+
# Add dependencies to develop your gem here.
|
5
|
+
# Include everything needed to run rake, tests, features, etc.
|
6
|
+
group :development do
|
7
|
+
gem 'sqlite3'
|
8
|
+
gem "rspec", "~> 2.4"
|
9
|
+
gem "rdoc", "~> 3.12"
|
10
|
+
gem "bundler", "~> 1.0"
|
11
|
+
gem "jeweler", "1.8.4" # The last without Nokogiri
|
12
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
gem 'activerecord', "~> 4"
|
3
|
+
|
4
|
+
# Add dependencies to develop your gem here.
|
5
|
+
# Include everything needed to run rake, tests, features, etc.
|
6
|
+
group :development do
|
7
|
+
gem 'sqlite3'
|
8
|
+
gem "rspec", "~> 2.4"
|
9
|
+
gem "rdoc", "~> 3.12"
|
10
|
+
gem "bundler", "~> 1.0"
|
11
|
+
gem "jeweler", "1.8.4" # The last without Nokogiri
|
12
|
+
end
|
@@ -0,0 +1,150 @@
|
|
1
|
+
# The idea is this: we can have ActiveRecord::Base.connection return an Adapter object (that executes the
|
2
|
+
# actual SQL queries) and inside that adapter we can redirect all SELECT queries to the read replica instead of
|
3
|
+
# the main database. This is however slightly more involved than it looks, because if we keep calling
|
4
|
+
# establish_connection willy-nilly we are going to disconnect from the master DB, connect to the read DB, disconnect
|
5
|
+
# again and so on.
|
6
|
+
#
|
7
|
+
# The right approach for this is to maintain a separate connection pool instad just for connections to the read
|
8
|
+
# slave. This is exactly what we are doing within AutoReplica.
|
9
|
+
#
|
10
|
+
# The setup to make it work is a bit involved. If you want to play with how ActiveRecord uses connections,
|
11
|
+
# the only actual "official" hook you can use is replacing the connection handler it uses. When a SQL request needs
|
12
|
+
# to be run AR walks the following dependency chain to arrive at the actual connection:
|
13
|
+
#
|
14
|
+
# * SomeRecord.connection calls ActiveRecord::Base.connection_handler
|
15
|
+
# * It then asks the connection handler for a connection for this specific ActiveRecord subclass, or barring that
|
16
|
+
# - for the connection for one of it's ancestors, ending with ActiveRecord::Base itself
|
17
|
+
# * The ConnectionHandler, in turn, asks one of it's managed ConnectionPool objects to give it a connection for use.
|
18
|
+
# * The connection is returned and then the query is ran against it.
|
19
|
+
#
|
20
|
+
# This is why the only integration point for this is ++ActiveRecord::Base.connection_hanler=++
|
21
|
+
# To make our trick work, here is what we do:
|
22
|
+
#
|
23
|
+
# First we wrap the original ConnectionHandler used by ActiveRecord in our own proxy. That proxy will ask the original
|
24
|
+
# ConnectionHandler for a connection to use, but it will also maintain it's own ConnectionPool just for the read
|
25
|
+
# replica connections.
|
26
|
+
#
|
27
|
+
# When a connection is returned by the original Handler, it is wrapped into a Adapter together with the
|
28
|
+
# read connection obtained from the special read pool. That proxy will intercept all of the SQL methods for SELECT
|
29
|
+
# and redirect them to the read connection instead. All of the other methods (including transactions)
|
30
|
+
# are still going to be executed on the master database.
|
31
|
+
#
|
32
|
+
# Once the block exits, the original connection handler is reassigned to the AR connection_pool.
|
33
|
+
module AutoReplica
|
34
|
+
# The first one is used in ActiveRecord 3+, the second one in 4+
|
35
|
+
ConnectionSpecification = ActiveRecord::Base::ConnectionSpecification rescue ActiveRecord::ConnectionAdapters::ConnectionSpecification
|
36
|
+
|
37
|
+
def self.using_read_replica_at(replica_connection_spec_hash_or_url)
|
38
|
+
return yield if @in_replica_context # This method should not be reentrant
|
39
|
+
|
40
|
+
# Resolve if there is a URL given
|
41
|
+
# Duplicate the hash so that we can change it if we have to
|
42
|
+
# (say by deleting :adapter)
|
43
|
+
config_hash = if replica_connection_spec_hash_or_url.is_a?(Hash)
|
44
|
+
replica_connection_spec_hash_or_url.dup
|
45
|
+
else
|
46
|
+
resolve_connection_url(replica_connection_spec_hash_or_url).dup
|
47
|
+
end
|
48
|
+
|
49
|
+
@in_replica_context = true
|
50
|
+
# Wrap the connection handler in our proxy
|
51
|
+
original_connection_handler = ActiveRecord::Base.connection_handler
|
52
|
+
custom_handler = ConnectionHandler.new(original_connection_handler, config_hash)
|
53
|
+
begin
|
54
|
+
ActiveRecord::Base.connection_handler = custom_handler
|
55
|
+
yield
|
56
|
+
ensure
|
57
|
+
ActiveRecord::Base.connection_handler = original_connection_handler
|
58
|
+
custom_handler.disconnect_read_pool!
|
59
|
+
@in_replica_context = false
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# Resolve an ActiveRecord connection URL
|
64
|
+
def self.resolve_connection_url(url_string)
|
65
|
+
if defined?(ActiveRecord::Base::ConnectionSpecification::Resolver) # AR3
|
66
|
+
resolver = ActiveRecord::Base::ConnectionSpecification::Resolver.new(url_string, {})
|
67
|
+
resolver.send(:connection_url_to_hash, url_string) # Because making this public was so hard
|
68
|
+
else # AR4
|
69
|
+
resolved = ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver.new(url_string).to_hash
|
70
|
+
resolved["database"].gsub!(/^\//, '') # which is not done by the resolver
|
71
|
+
resolved.symbolize_keys # which is also not done by the resolver
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# The connection handler that wraps the ActiveRecord one. Everything gets forwarded to the wrapped
|
76
|
+
# object, but a "spiked" connection adapter gets returned from retrieve_connection.
|
77
|
+
class ConnectionHandler # a proxy for ActiveRecord::ConnectionAdapters::ConnectionHandler
|
78
|
+
def initialize(original_handler, connection_specification_hash)
|
79
|
+
@original_handler = original_handler
|
80
|
+
# We need to maintain our own pool for read replica connections,
|
81
|
+
# aside from the one managed by Rails proper.
|
82
|
+
adapter_method = "%s_connection" % connection_specification_hash[:adapter]
|
83
|
+
connection_specification = ConnectionSpecification.new(connection_specification_hash, adapter_method)
|
84
|
+
@read_pool = ActiveRecord::ConnectionAdapters::ConnectionPool.new(connection_specification)
|
85
|
+
end
|
86
|
+
|
87
|
+
# Overridden method which gets called by ActiveRecord to get a connection related to a specific
|
88
|
+
# ActiveRecord::Base subclass.
|
89
|
+
def retrieve_connection(for_ar_class)
|
90
|
+
connection_for_writes = @original_handler.retrieve_connection(for_ar_class)
|
91
|
+
connection_for_reads = @read_pool.connection
|
92
|
+
Adapter.new(connection_for_writes, connection_for_reads)
|
93
|
+
end
|
94
|
+
|
95
|
+
# Close all the connections maintained by the read pool
|
96
|
+
def disconnect_read_pool!
|
97
|
+
@read_pool.disconnect!
|
98
|
+
end
|
99
|
+
|
100
|
+
# Disconnect both the original handler AND the read pool
|
101
|
+
def clear_all_connections!
|
102
|
+
disconnect_read_pool!
|
103
|
+
@original_handler.clear_all_connections!
|
104
|
+
end
|
105
|
+
|
106
|
+
# The duo for method proxying without delegate
|
107
|
+
def respond_to_missing?(method_name)
|
108
|
+
@original_handler.respond_to?(method_name)
|
109
|
+
end
|
110
|
+
def method_missing(method_name, *args, &blk)
|
111
|
+
@original_handler.public_send(method_name, *args, &blk)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
# Acts as a wrapping proxy that replaces an ActiveRecord Adapter object. This is the
|
116
|
+
# "connection adapter" object that ActiveRecord uses internally to run SQL queries
|
117
|
+
# against. We let it dispatch all SELECT queries to the read replica and all
|
118
|
+
# other queries to the master database.
|
119
|
+
#
|
120
|
+
# To achieve this, we make a delegator proxy that sends all methods prefixed with "select_"
|
121
|
+
# to the read connection, and all the others to the master connection.
|
122
|
+
class Adapter # a proxy for ActiveRecord::ConnectionAdapters::AbstractAdapter
|
123
|
+
def initialize(master_connection_adapter, replica_connection_adapter)
|
124
|
+
@master_connection = master_connection_adapter
|
125
|
+
@read_connection = replica_connection_adapter
|
126
|
+
end
|
127
|
+
|
128
|
+
# Under the hood, ActiveRecord uses methods for the most common database statements
|
129
|
+
# like "select_all", "select_one", "select_value" and so on. Those can be overridden by concrete
|
130
|
+
# connection adapters, but in the basic abstract Adapter they get included from
|
131
|
+
# DatabaseStatements. Therefore we can obtain a list of those methods (that we want to override)
|
132
|
+
# by grepping the instance method names of the DatabaseStatements module.
|
133
|
+
select_methods = ActiveRecord::ConnectionAdapters::DatabaseStatements.instance_methods.grep(/^select_/)
|
134
|
+
# ...and then for each of those "select_something" methods we can make a method override
|
135
|
+
# that will redirect the method to the read connection.
|
136
|
+
select_methods.each do | select_method_name |
|
137
|
+
define_method(select_method_name) do |*method_arguments|
|
138
|
+
@read_connection.send(select_method_name, *method_arguments)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
# The duo for method proxying without delegate
|
143
|
+
def respond_to_missing?(method_name)
|
144
|
+
@master_connection.respond_to?(method_name)
|
145
|
+
end
|
146
|
+
def method_missing(method_name, *args, &blk)
|
147
|
+
@master_connection.public_send(method_name, *args, &blk)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require_relative 'helper'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
describe AutoReplica do
|
5
|
+
|
6
|
+
before :all do
|
7
|
+
test_seed_name = SecureRandom.hex(4)
|
8
|
+
ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ('master_db_%s.sqlite3' % test_seed_name))
|
9
|
+
|
10
|
+
# Setup the master and replica connections
|
11
|
+
@master_connection_config = ActiveRecord::Base.connection_config.dup
|
12
|
+
@replica_connection_config = @master_connection_config.merge(database: ('replica_db_%s.sqlite3' % test_seed_name))
|
13
|
+
@replica_connection_config_url = 'sqlite3:/replica_db_%s.sqlite3' % test_seed_name
|
14
|
+
|
15
|
+
ActiveRecord::Migration.suppress_messages do
|
16
|
+
# Create both the master and the replica, with a simple small schema
|
17
|
+
[@master_connection_config, @replica_connection_config].each do | db_config |
|
18
|
+
ActiveRecord::Base.establish_connection(db_config)
|
19
|
+
ActiveRecord::Schema.define(:version => 1) do
|
20
|
+
create_table :things do |t|
|
21
|
+
t.string :description, :null => true
|
22
|
+
t.timestamps :null => false
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
after :all do
|
30
|
+
# Ensure database files get killed afterwards
|
31
|
+
[@master_connection_config, @replica_connection_config].map do | connection_config |
|
32
|
+
File.unlink(connection_config[:database]) rescue nil
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
before :each do
|
37
|
+
[@replica_connection_config, @master_connection_config].each do | config |
|
38
|
+
ActiveRecord::Base.establish_connection(config)
|
39
|
+
ActiveRecord::Base.connection.execute 'DELETE FROM things' # sqlite has no TRUNCATE
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
class TestThing < ActiveRecord::Base
|
44
|
+
self.table_name = 'things'
|
45
|
+
end
|
46
|
+
|
47
|
+
context 'using_read_replica_at' do
|
48
|
+
it 'has no reentrancy problems' do
|
49
|
+
id = described_class.using_read_replica_at(@replica_connection_config) do
|
50
|
+
described_class.using_read_replica_at(@replica_connection_config) do
|
51
|
+
described_class.using_read_replica_at(@replica_connection_config) do
|
52
|
+
thing = TestThing.create! description: 'A nice Thing in the master database'
|
53
|
+
expect {
|
54
|
+
TestThing.find(thing.id)
|
55
|
+
}.to raise_error(ActiveRecord::RecordNotFound)
|
56
|
+
|
57
|
+
thing.id # return to the outside of the block
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
found_on_master = TestThing.find(id)
|
62
|
+
expect(found_on_master.description).to eq('A nice Thing in the master database')
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'executes the SELECT query against the replica database and returns the result of the block' do
|
66
|
+
id = described_class.using_read_replica_at(@replica_connection_config) do
|
67
|
+
thing = TestThing.create! description: 'A nice Thing in the master database'
|
68
|
+
expect {
|
69
|
+
TestThing.find(thing.id)
|
70
|
+
}.to raise_error(ActiveRecord::RecordNotFound)
|
71
|
+
thing.id
|
72
|
+
end
|
73
|
+
found_on_master = TestThing.find(id)
|
74
|
+
expect(found_on_master.description).to eq('A nice Thing in the master database')
|
75
|
+
|
76
|
+
ActiveRecord::Base.establish_connection(@replica_connection_config)
|
77
|
+
thing_on_slave = TestThing.new(found_on_master.attributes)
|
78
|
+
thing_on_slave.id = found_on_master.id # gets ignored in attributes
|
79
|
+
thing_on_slave.description = 'A nice Thing that is in the slave database'
|
80
|
+
thing_on_slave.save!
|
81
|
+
|
82
|
+
ActiveRecord::Base.establish_connection(@master_connection_config)
|
83
|
+
described_class.using_read_replica_at(@replica_connection_config) do
|
84
|
+
thing_from_replica = TestThing.find(id)
|
85
|
+
expect(thing_from_replica.description).to eq('A nice Thing that is in the slave database')
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
it 'executes the SELECT query against the replica database with replica connection specification given as a URL' do
|
90
|
+
id = described_class.using_read_replica_at(@replica_connection_config_url) do
|
91
|
+
thing = TestThing.create! description: 'A nice Thing in the master database'
|
92
|
+
expect {
|
93
|
+
TestThing.find(thing.id)
|
94
|
+
}.to raise_error(ActiveRecord::RecordNotFound)
|
95
|
+
thing.id
|
96
|
+
end
|
97
|
+
found_on_master = TestThing.find(id)
|
98
|
+
expect(found_on_master.description).to eq('A nice Thing in the master database')
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
describe AutoReplica::ConnectionHandler do
|
103
|
+
it 'creates a read pool with the replica connection specification hash' do
|
104
|
+
original_handler = double('ActiveRecord_ConnectionHandler')
|
105
|
+
expect(ActiveRecord::ConnectionAdapters::ConnectionPool).to receive(:new).
|
106
|
+
with(instance_of(AutoReplica::ConnectionSpecification))
|
107
|
+
|
108
|
+
AutoReplica::ConnectionHandler.new(original_handler, @replica_connection_config)
|
109
|
+
end
|
110
|
+
|
111
|
+
it 'proxies all methods' do
|
112
|
+
original_handler = double('ActiveRecord_ConnectionHandler')
|
113
|
+
expect(original_handler).to receive(:do_that_thing) { :yes }
|
114
|
+
subject = AutoReplica::ConnectionHandler.new(original_handler, @replica_connection_config)
|
115
|
+
expect(subject.do_that_thing).to eq(:yes)
|
116
|
+
end
|
117
|
+
|
118
|
+
it 'enhances connection_for and returns an instance of the Adapter' do
|
119
|
+
original_handler = double('ActiveRecord_ConnectionHandler')
|
120
|
+
adapter_double = double('ActiveRecord_Adapter')
|
121
|
+
expect(original_handler).to receive(:retrieve_connection).with(TestThing) { adapter_double }
|
122
|
+
|
123
|
+
subject = AutoReplica::ConnectionHandler.new(original_handler, @replica_connection_config)
|
124
|
+
connection = subject.retrieve_connection(TestThing)
|
125
|
+
expect(connection).to be_kind_of(AutoReplica::Adapter)
|
126
|
+
end
|
127
|
+
|
128
|
+
it 'disconnects the read pool when asked to' do
|
129
|
+
original_handler = double('ActiveRecord_ConnectionHandler')
|
130
|
+
pool_double = double('ConnectionPool')
|
131
|
+
expect(ActiveRecord::ConnectionAdapters::ConnectionPool).to receive(:new).
|
132
|
+
with(instance_of(AutoReplica::ConnectionSpecification)) { pool_double }
|
133
|
+
subject = AutoReplica::ConnectionHandler.new(original_handler, @replica_connection_config)
|
134
|
+
|
135
|
+
expect(pool_double).to receive(:disconnect!)
|
136
|
+
subject.disconnect_read_pool!
|
137
|
+
end
|
138
|
+
|
139
|
+
it 'performs clear_all_connections! both on the contained handler and on the read pool' do
|
140
|
+
original_handler = double('ActiveRecord_ConnectionHandler')
|
141
|
+
pool_double = double('ConnectionPool')
|
142
|
+
expect(ActiveRecord::ConnectionAdapters::ConnectionPool).to receive(:new).
|
143
|
+
with(instance_of(AutoReplica::ConnectionSpecification)) { pool_double }
|
144
|
+
|
145
|
+
expect(original_handler).to receive(:clear_all_connections!)
|
146
|
+
expect(pool_double).to receive(:disconnect!)
|
147
|
+
|
148
|
+
subject = AutoReplica::ConnectionHandler.new(original_handler, @replica_connection_config)
|
149
|
+
subject.clear_all_connections!
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
describe AutoReplica::Adapter do
|
154
|
+
it 'mirrors select_ prefixed database statement methods in ActiveRecord::ConnectionAdapters::DatabaseStatements' do
|
155
|
+
master = double()
|
156
|
+
expect(master).not_to receive(:respond_to?)
|
157
|
+
subject = AutoReplica::Adapter.new(master, double())
|
158
|
+
|
159
|
+
select_methods = ActiveRecord::ConnectionAdapters::DatabaseStatements.instance_methods.grep(/^select_/)
|
160
|
+
expect(select_methods.length).to be > 1
|
161
|
+
|
162
|
+
select_methods.each do | select_method_in_database_statements |
|
163
|
+
expect(subject).to respond_to(select_method_in_database_statements)
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
it 'redirects calls to all select_ methods to the read connection and others to the master connection' do
|
168
|
+
master_adapter = double('Connection to the master DB')
|
169
|
+
replica_adapter = double('Connection to the replica DB')
|
170
|
+
subject = AutoReplica::Adapter.new(master_adapter, replica_adapter)
|
171
|
+
|
172
|
+
expect(master_adapter).to receive(:some_arbitrary_method) { :from_master }
|
173
|
+
expect(replica_adapter).to receive(:select_values).with(:a, :b, :c) { :from_replica }
|
174
|
+
expect(subject.some_arbitrary_method).to eq(:from_master)
|
175
|
+
expect(subject.select_values(:a, :b, :c)).to eq(:from_replica)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
data/spec/helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,141 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: activerecord_autoreplica
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Julik Tarkhanov
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-19 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activerecord
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '3.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '3.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: sqlite3
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '2.4'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '2.4'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rdoc
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.12'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.12'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: jeweler
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: " Redirect all SELECT queries to a separate connection within a block "
|
98
|
+
email: me@julik.nl
|
99
|
+
executables: []
|
100
|
+
extensions: []
|
101
|
+
extra_rdoc_files:
|
102
|
+
- LICENSE.txt
|
103
|
+
- README.md
|
104
|
+
files:
|
105
|
+
- ".document"
|
106
|
+
- ".gitlab-ci.yml"
|
107
|
+
- Gemfile
|
108
|
+
- LICENSE.txt
|
109
|
+
- README.md
|
110
|
+
- Rakefile
|
111
|
+
- activerecord_autoreplica.gemspec
|
112
|
+
- gemfiles/Gemfile.rails-3.2.x
|
113
|
+
- gemfiles/Gemfile.rails-4.1.x
|
114
|
+
- lib/activerecord_autoreplica.rb
|
115
|
+
- spec/activerecord_autoreplica_spec.rb
|
116
|
+
- spec/helper.rb
|
117
|
+
homepage: http://github.com/WeTransfer/activerecord_autoreplica
|
118
|
+
licenses:
|
119
|
+
- MIT
|
120
|
+
metadata: {}
|
121
|
+
post_install_message:
|
122
|
+
rdoc_options: []
|
123
|
+
require_paths:
|
124
|
+
- lib
|
125
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
127
|
+
- - ">="
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: '0'
|
130
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
131
|
+
requirements:
|
132
|
+
- - ">="
|
133
|
+
- !ruby/object:Gem::Version
|
134
|
+
version: '0'
|
135
|
+
requirements: []
|
136
|
+
rubyforge_project:
|
137
|
+
rubygems_version: 2.2.2
|
138
|
+
signing_key:
|
139
|
+
specification_version: 4
|
140
|
+
summary: Palatable-size read replica adapter for ActiveRecord
|
141
|
+
test_files: []
|