redis_directory 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +24 -0
- data/Rakefile +26 -0
- data/lib/redis_directory.rb +90 -0
- data/redis_directory.gemspec +47 -0
- data/test/connection_test.rb +129 -0
- data/test/helper.rb +7 -0
- metadata +140 -0
data/README
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
= Redis::Directory
|
2
|
+
|
3
|
+
Redis::Directory assumes a Redis installation running on a default
|
4
|
+
port and database 0 that will contain connection information for various
|
5
|
+
other Redis databases. For example, if you were using a Redis database to
|
6
|
+
store the content of cached pages, and this was running on a cluster of
|
7
|
+
two Redis instances, with multiple applications connecting partitioned by
|
8
|
+
database, then your connection might look like this:
|
9
|
+
|
10
|
+
require "redis"
|
11
|
+
require "redis/distributed"
|
12
|
+
|
13
|
+
# The ACME Corp database is #27
|
14
|
+
cache = Redis::Distributed.new %w( redis://redis.example:4400/27 redis://redis.example:4401/27 )
|
15
|
+
|
16
|
+
Redis::Directory uses a centralized Redis database to store the
|
17
|
+
connection information so you don't have to remember "magic numbers" for
|
18
|
+
each client/database mapping, and can easily update port-numbers, hostnames
|
19
|
+
and cluster-members as necessary. The same connection with
|
20
|
+
Redis::Directory would look like this:
|
21
|
+
|
22
|
+
require "redis_directory"
|
23
|
+
|
24
|
+
cache = Redis::Directory.new("redis.example").get("cache", "acme")
|
data/Rakefile
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require "rake"
|
2
|
+
require "rake/clean"
|
3
|
+
require "rake/testtask"
|
4
|
+
require "rake/rdoctask"
|
5
|
+
|
6
|
+
task :default => :test
|
7
|
+
|
8
|
+
Rake::TestTask.new(:test) do |t|
|
9
|
+
t.libs << "test"
|
10
|
+
t.test_files = FileList["test/**/*_test.rb"]
|
11
|
+
t.verbose = true
|
12
|
+
end
|
13
|
+
|
14
|
+
CLEAN.include ["*.gem", "rdoc"]
|
15
|
+
RDOC_OPTS = [ "--quiet", "--inline-source", "--line-numbers", "--title", "Redis Directory: A database connection manager for Redis", "--main", "README" ]
|
16
|
+
|
17
|
+
Rake::RDocTask.new do |rdoc|
|
18
|
+
rdoc.rdoc_dir = "rdoc"
|
19
|
+
rdoc.options += RDOC_OPTS
|
20
|
+
rdoc.rdoc_files.add %w"README MIT-LICENSE lib/redis_directory.rb"
|
21
|
+
end
|
22
|
+
|
23
|
+
desc "Package redis_directory"
|
24
|
+
task :package do
|
25
|
+
sh %{gem build redis_directory.gemspec}
|
26
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require "redis"
|
2
|
+
require "redis/distributed"
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
class Redis::Directory
|
6
|
+
|
7
|
+
SERVICES_KEY = "services"
|
8
|
+
DATABASES_KEY = "databases"
|
9
|
+
|
10
|
+
# Hard-coded, because I don't know how to get this from the Redis connection at runtime.
|
11
|
+
MAXIMUM_DATABASE_COUNT = 65535
|
12
|
+
|
13
|
+
# This error is thrown when you request a service that is not defined in the directory's "services" key.
|
14
|
+
class UndefinedServiceError < StandardError
|
15
|
+
def initialize(directory, service_name)
|
16
|
+
super "#{service_name} is not an available service! (#{directory.services.keys.sort})"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# If you are unable to reserve a database during a connection attempt, this error is raised.
|
21
|
+
class ReservationError < StandardError
|
22
|
+
def initialize(directory, service_name, connection_name)
|
23
|
+
super "Unable to reserve a database for #{service_name}:#{connection_name}!\nCurrent databases for service: #{directory.redis.hgetall("#{service_name}-service")}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# You must provide the +connection_options+ to the directory server.
|
28
|
+
def initialize(connection_options)
|
29
|
+
@redis = Redis.connect(connection_options)
|
30
|
+
end
|
31
|
+
|
32
|
+
def services
|
33
|
+
if redis.exists(SERVICES_KEY)
|
34
|
+
redis.hgetall(SERVICES_KEY).inject({}) do |h,(k,v)|
|
35
|
+
h[k] = JSON.parse(v); h
|
36
|
+
end
|
37
|
+
else
|
38
|
+
{}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
# This locates the next available database for a given service, starting at an index of 1.
|
43
|
+
# The 0 database is reserved in case you have a default connection, or local redis server
|
44
|
+
# (in which case 0 should be the directory database to avoid conflicts).
|
45
|
+
def next_db(service_name)
|
46
|
+
raise UndefinedServiceError.new(self, service_name) unless services.keys.include?(service_name)
|
47
|
+
databases = redis.hvals("#{service_name}-service").map { |i| i.to_i }.sort
|
48
|
+
(1..MAXIMUM_DATABASE_COUNT).select { |i| break i unless databases.include? i }
|
49
|
+
end
|
50
|
+
|
51
|
+
def reserve(service_name, connection_name)
|
52
|
+
new_db = nil
|
53
|
+
# redis.multi do
|
54
|
+
if redis.hexists("#{service_name}-service", connection_name)
|
55
|
+
new_db = redis.hget("#{service_name}-service", connection_name)
|
56
|
+
else
|
57
|
+
new_db = next_db(service_name)
|
58
|
+
redis.hset("#{service_name}-service", connection_name, new_db)
|
59
|
+
end
|
60
|
+
# end
|
61
|
+
new_db
|
62
|
+
end
|
63
|
+
|
64
|
+
def get(service_name, connection_name)
|
65
|
+
db = reserve(service_name, connection_name)
|
66
|
+
raise ReservationError.new(self, service_name, connection_name) if db.nil?
|
67
|
+
|
68
|
+
connection_list = services[service_name].map do |server|
|
69
|
+
if server =~ /^redis\:\/\//
|
70
|
+
"#{server}/#{db}"
|
71
|
+
else
|
72
|
+
"redis://#{server}/#{db}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
connection = nil
|
76
|
+
|
77
|
+
if connection_list.size == 1
|
78
|
+
connection = Redis.connect(:url => connection_list.first)
|
79
|
+
else
|
80
|
+
connection = Redis::Distributed.new(connection_list)
|
81
|
+
end
|
82
|
+
|
83
|
+
connection.set("connection-name", connection_name)
|
84
|
+
connection
|
85
|
+
end
|
86
|
+
|
87
|
+
def redis
|
88
|
+
@redis
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = "redis_directory"
|
3
|
+
s.version = "1.0.4"
|
4
|
+
s.platform = Gem::Platform::RUBY
|
5
|
+
s.summary = "Redis Directory for retriving/storing redis connections in a central database."
|
6
|
+
|
7
|
+
s.description = <<-EOF
|
8
|
+
Redis::Directory assumes a Redis installation running on a default
|
9
|
+
port and database 0 that will contain connection information for various
|
10
|
+
other Redis databases. For example, if you were using a Redis database to
|
11
|
+
store the content of cached pages, and this was running on a cluster of
|
12
|
+
two Redis instances, with multiple applications connecting partitioned by
|
13
|
+
database, then your connection might look like this:
|
14
|
+
|
15
|
+
require "redis"
|
16
|
+
require "redis/distributed"
|
17
|
+
|
18
|
+
# The ACME Corp database is #27
|
19
|
+
cache = Redis::Distributed.new "redis://redis.example:4400/27", "redis://redis.example:4401/27"
|
20
|
+
|
21
|
+
Redis::Directory uses a centralized Redis database to store the
|
22
|
+
connection information so you don't have to remember "magic numbers" for
|
23
|
+
each client/database mapping, and can easily update port-numbers/hostnames,
|
24
|
+
cluster-members as necessary. The same connection with
|
25
|
+
Redis::Directory would look like this:
|
26
|
+
|
27
|
+
require "redis_directory"
|
28
|
+
|
29
|
+
cache = Redis::Directory.new("redis.example").connect("cache", "acme")
|
30
|
+
EOF
|
31
|
+
|
32
|
+
s.files = Dir[ "README", "lib/**/*", "test/**/*", "redis_directory.gemspec", "Rakefile" ]
|
33
|
+
s.require_path = "lib"
|
34
|
+
s.test_files = Dir["test/**/*_test.rb"]
|
35
|
+
|
36
|
+
s.author = "Sam Smoot"
|
37
|
+
s.email = "ssmoot@gmail.com"
|
38
|
+
s.homepage = "https://github.com/wiecklabs/redis_directory"
|
39
|
+
s.license = "MIT"
|
40
|
+
|
41
|
+
s.has_rdoc = true
|
42
|
+
s.rdoc_options = [ "--inline-source", "--line-numbers", "--title", "Redis Directory: A database connection manager for Redis", "README", "MIT-LICENSE", "lib" ]
|
43
|
+
|
44
|
+
s.add_dependency "redis"
|
45
|
+
s.add_dependency "minitest" if RUBY_VERSION =~ /1\.8/
|
46
|
+
s.add_dependency "json"
|
47
|
+
end
|
@@ -0,0 +1,129 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/helper"
|
2
|
+
|
3
|
+
class ConnectionTest < MiniTest::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
@directory = Redis::Directory.new(:url => (ENV["REDIS_DIRECTORY"] || "redis://localhost"))
|
7
|
+
|
8
|
+
@redis = @directory.redis
|
9
|
+
@redis.hset Redis::Directory::SERVICES_KEY, "cache", [ "redis://localhost" ].to_json
|
10
|
+
@redis.hset Redis::Directory::SERVICES_KEY, "sessions", [ "redis://localhost" ].to_json
|
11
|
+
@redis.hset Redis::Directory::SERVICES_KEY, "queue", [ "redis://localhost" ].to_json
|
12
|
+
end
|
13
|
+
|
14
|
+
def teardown
|
15
|
+
@redis.flushall
|
16
|
+
@redis.quit
|
17
|
+
end
|
18
|
+
|
19
|
+
# Our first test verifies that the "services" key (populated here in #setup) is not empty.
|
20
|
+
# Since we can't know the hostname(s) and ports for the Redis members you've setup, you
|
21
|
+
# must pre-populate this.
|
22
|
+
# This can be as easy as using the redis-cli on the command-line to populate your directory:
|
23
|
+
#
|
24
|
+
# redis-cli -h redis.example.com -p 8440 hset services cache '["redis.example.com:9100", "redis.example.com:9101"]'
|
25
|
+
# redis-cli -h redis.example.com -p 8440 hset services sessions '["redis.example.com:9200", "redis.example.com:9201"]'
|
26
|
+
# redis-cli -h redis.example.com -p 8440 hset services queue '["redis.example.com:9300"]'
|
27
|
+
#
|
28
|
+
# Then +@directory.services+ would be equal to:
|
29
|
+
#
|
30
|
+
# {
|
31
|
+
# "cache" => ["redis.example.com:9100", "redis.example.com:9101"],
|
32
|
+
# "sessions" => ["redis.example.com:9200", "redis.example.com:9201"],
|
33
|
+
# "queue" => ["redis.example.com:9300"]
|
34
|
+
# }
|
35
|
+
#
|
36
|
+
# These are the definitions for your cluster(s) of services.
|
37
|
+
# It's recommended that you dedicate individual use-cases to individual clusters.
|
38
|
+
# This helps you balance your loads more evenly across multi-core machines
|
39
|
+
# (since Redis is single-threaded) as well as allowing you to tune memory limits,
|
40
|
+
# flush-to-disk intervals, etc on a per-cluster basis.
|
41
|
+
def test_services_are_staticly_assigned
|
42
|
+
refute_empty @directory.services
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_services_is_a_hash_of_connection_strings
|
46
|
+
assert_kind_of Hash, @directory.services
|
47
|
+
end
|
48
|
+
|
49
|
+
# Because we use a Redis::Distributed connection, you should consistently
|
50
|
+
# use Arrays for your cluster connections even if there is only one item
|
51
|
+
# in the cluster.
|
52
|
+
def test_service_value_is_an_array
|
53
|
+
assert_kind_of Array, @directory.services["queue"]
|
54
|
+
end
|
55
|
+
|
56
|
+
# Here we're just confirming that when we ask for it, the next availble
|
57
|
+
# database number for a given service is retrieved.
|
58
|
+
def test_database_number_is_retrieved
|
59
|
+
assert_kind_of Fixnum, @directory.next_db("sessions")
|
60
|
+
end
|
61
|
+
|
62
|
+
# If the service/cluster is undefined, then we should raise a
|
63
|
+
# UndefinedServiceError when requesting it's next database number.
|
64
|
+
def test_database_number_errors_for_service_not_exist
|
65
|
+
assert_raises(Redis::Directory::UndefinedServiceError) do
|
66
|
+
@directory.next_db("quack")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Once we've made a connection (+Redis::Directory#get+), we should be able
|
71
|
+
# to confirm that the database we've connected to is for this name.
|
72
|
+
def test_database_contains_connection_name
|
73
|
+
assert_equal "acme", @directory.get("cache", "acme").get("connection-name")
|
74
|
+
end
|
75
|
+
|
76
|
+
# If we reserve a database, it should be tracked in a Hash describing the
|
77
|
+
# currently reserved services for that name. That way we can look it up later
|
78
|
+
# and re-use it instead of always provisioning new databases every time you
|
79
|
+
# make a new request.
|
80
|
+
def test_database_is_reserved
|
81
|
+
@directory.reserve("cache", "acme")
|
82
|
+
assert @redis.hexists("cache-service", "acme")
|
83
|
+
refute_nil @redis.hget("cache-service", "acme")
|
84
|
+
end
|
85
|
+
|
86
|
+
# This test confirms an internal implementation detail, ensuring that we don't
|
87
|
+
# have collisions between multiple database names for a given service.
|
88
|
+
def test_additional_cache_connection_increments
|
89
|
+
@directory.reserve("cache", "acme")
|
90
|
+
@directory.reserve("cache", "trioptimum")
|
91
|
+
assert_equal @redis.hlen("cache-service"), @redis.hget("cache-service", "trioptimum").to_i
|
92
|
+
end
|
93
|
+
|
94
|
+
# When there are "gaps" in the list of tracked databases for a service, those
|
95
|
+
# should be filled to keep the maximum database number as low as possible.
|
96
|
+
# This way as long as you clean up disused databases by removing them from the
|
97
|
+
# service keys, eg:
|
98
|
+
#
|
99
|
+
# redis-cli hdel cache-service acme
|
100
|
+
#
|
101
|
+
# Then we'll re-use that database number, meaning that as long as you've set
|
102
|
+
# (in your redis.conf) your maximum databases equal to or great than the number
|
103
|
+
# you plan to use, +Redis::Directory+ should not exceed that number.
|
104
|
+
def test_sparse_ids_are_filled
|
105
|
+
@redis.hset("cache-service", "corporatron", 5)
|
106
|
+
|
107
|
+
@directory.reserve("cache", "bnl")
|
108
|
+
assert_equal 1, @redis.hget("cache-service", "bnl").to_i
|
109
|
+
|
110
|
+
@directory.reserve("cache", "axiom")
|
111
|
+
assert_equal 2, @redis.hget("cache-service", "axiom").to_i
|
112
|
+
end
|
113
|
+
|
114
|
+
# In order to use as few database numbers as possible, and again, allowing you
|
115
|
+
# to tune Redis configuration settings like +databases+ on a per service/cluster
|
116
|
+
# basis, database numbers may not be consistent across services/clusters for a
|
117
|
+
# given client/name. This is especially true if not all clients consume all services,
|
118
|
+
# as the example below demonstrates.
|
119
|
+
def test_database_ids_are_unbalanced
|
120
|
+
bnl = @directory.reserve("cache", "bnl")
|
121
|
+
hacker = @directory.reserve("cache", "trioptimum")
|
122
|
+
acme = @directory.reserve("cache", "acme")
|
123
|
+
|
124
|
+
shodan = @directory.reserve("sessions", "trioptimum")
|
125
|
+
|
126
|
+
refute_equal hacker, shodan
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
data/test/helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,140 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: redis_directory
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 31
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 4
|
10
|
+
version: 1.0.4
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Sam Smoot
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-03-03 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: redis
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 3
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
version: "0"
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: minitest
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 3
|
43
|
+
segments:
|
44
|
+
- 0
|
45
|
+
version: "0"
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: json
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
hash: 3
|
57
|
+
segments:
|
58
|
+
- 0
|
59
|
+
version: "0"
|
60
|
+
type: :runtime
|
61
|
+
version_requirements: *id003
|
62
|
+
description: |
|
63
|
+
Redis::Directory assumes a Redis installation running on a default
|
64
|
+
port and database 0 that will contain connection information for various
|
65
|
+
other Redis databases. For example, if you were using a Redis database to
|
66
|
+
store the content of cached pages, and this was running on a cluster of
|
67
|
+
two Redis instances, with multiple applications connecting partitioned by
|
68
|
+
database, then your connection might look like this:
|
69
|
+
|
70
|
+
require "redis"
|
71
|
+
require "redis/distributed"
|
72
|
+
|
73
|
+
# The ACME Corp database is #27
|
74
|
+
cache = Redis::Distributed.new "redis://redis.example:4400/27", "redis://redis.example:4401/27"
|
75
|
+
|
76
|
+
Redis::Directory uses a centralized Redis database to store the
|
77
|
+
connection information so you don't have to remember "magic numbers" for
|
78
|
+
each client/database mapping, and can easily update port-numbers/hostnames,
|
79
|
+
cluster-members as necessary. The same connection with
|
80
|
+
Redis::Directory would look like this:
|
81
|
+
|
82
|
+
require "redis_directory"
|
83
|
+
|
84
|
+
cache = Redis::Directory.new("redis.example").connect("cache", "acme")
|
85
|
+
|
86
|
+
email: ssmoot@gmail.com
|
87
|
+
executables: []
|
88
|
+
|
89
|
+
extensions: []
|
90
|
+
|
91
|
+
extra_rdoc_files: []
|
92
|
+
|
93
|
+
files:
|
94
|
+
- README
|
95
|
+
- lib/redis_directory.rb
|
96
|
+
- test/connection_test.rb
|
97
|
+
- test/helper.rb
|
98
|
+
- redis_directory.gemspec
|
99
|
+
- Rakefile
|
100
|
+
homepage: https://github.com/wiecklabs/redis_directory
|
101
|
+
licenses:
|
102
|
+
- MIT
|
103
|
+
post_install_message:
|
104
|
+
rdoc_options:
|
105
|
+
- --inline-source
|
106
|
+
- --line-numbers
|
107
|
+
- --title
|
108
|
+
- "Redis Directory: A database connection manager for Redis"
|
109
|
+
- README
|
110
|
+
- MIT-LICENSE
|
111
|
+
- lib
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
none: false
|
116
|
+
requirements:
|
117
|
+
- - ">="
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
hash: 3
|
120
|
+
segments:
|
121
|
+
- 0
|
122
|
+
version: "0"
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
none: false
|
125
|
+
requirements:
|
126
|
+
- - ">="
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
hash: 3
|
129
|
+
segments:
|
130
|
+
- 0
|
131
|
+
version: "0"
|
132
|
+
requirements: []
|
133
|
+
|
134
|
+
rubyforge_project:
|
135
|
+
rubygems_version: 1.8.12
|
136
|
+
signing_key:
|
137
|
+
specification_version: 3
|
138
|
+
summary: Redis Directory for retriving/storing redis connections in a central database.
|
139
|
+
test_files:
|
140
|
+
- test/connection_test.rb
|