redis_pipeline 0.0.5 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +17 -8
- data/lib/redis_pipeline/commands.rb +4 -0
- data/lib/redis_pipeline/errors.rb +4 -0
- data/lib/redis_pipeline/redis_pipeline.rb +30 -19
- data/lib/redis_pipeline/version.rb +1 -1
- data/lib/redis_pipeline.rb +2 -0
- data/redis_pipeline.gemspec +1 -1
- data/test/test_redis_pipeline.rb +28 -16
- metadata +12 -10
data/README.md
CHANGED
@@ -22,7 +22,7 @@ Create the configuration yaml file
|
|
22
22
|
|
23
23
|
rails generate redis_pipeline
|
24
24
|
|
25
|
-
Populate the configuration file with the redis uri and batch size.
|
25
|
+
Populate the configuration file with the redis uri and batch size. The configuration file is stored at `config/redis_pipeline.yml`
|
26
26
|
|
27
27
|
* uri defaults to `redis://localhost:6379` if you don't have a configuration file
|
28
28
|
* batch size defaults to 1000 if you don't have a configuration file
|
@@ -32,28 +32,37 @@ Populate the configuration file with the redis uri and batch size.
|
|
32
32
|
Create a new pipeline
|
33
33
|
|
34
34
|
```ruby
|
35
|
-
pipeline = RedisPipeline::RedisPineline.new
|
35
|
+
pipeline = RedisPipeline::RedisPineline.new
|
36
36
|
```
|
37
37
|
|
38
|
-
Queue up commands
|
38
|
+
Queue up commands by adding them. You can add a single command as a string. Within a command the | character is used to separate the parts of the command.
|
39
39
|
|
40
40
|
```ruby
|
41
|
-
pipeline.
|
41
|
+
pipeline.add_command('set|hello|world')
|
42
42
|
```
|
43
43
|
|
44
|
-
|
44
|
+
Or pass an array of commands.
|
45
45
|
|
46
46
|
```ruby
|
47
47
|
array = ['hset|gem|first_name|redis', '|hset|gem|last_name|pipeline']
|
48
|
-
pipeline.
|
48
|
+
pipeline.add_command(array)
|
49
49
|
```
|
50
50
|
|
51
|
-
|
51
|
+
The shovel operator works as well.
|
52
52
|
|
53
53
|
```ruby
|
54
|
-
|
54
|
+
pipeline << 'set|hello|world'
|
55
|
+
pipeline << ['hset|gem|first_name|redis', '|hset|gem|last_name|pipeline']
|
55
56
|
```
|
56
57
|
|
58
|
+
Send the commands to redis with <tt>execute</tt>. Commands are sent using redis-rb's pipelined mode in batches, the size of which are controlled by your configuration. Returns false if there is an error, true if all commands succeed.
|
59
|
+
|
60
|
+
```ruby
|
61
|
+
pipeline.execute
|
62
|
+
```
|
63
|
+
|
64
|
+
See the pipeline errors by iterating through `pipeline.errors`.
|
65
|
+
|
57
66
|
## Contributing
|
58
67
|
|
59
68
|
1. Fork it
|
@@ -7,39 +7,50 @@ module RedisPipeline
|
|
7
7
|
|
8
8
|
attr_reader :errors
|
9
9
|
|
10
|
+
# Instantiates and configures a redis pipeline.
|
10
11
|
def initialize()
|
11
12
|
configure
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
self.redis = open_redis_connection
|
14
|
+
self.commands = Commands.new
|
15
|
+
self.errors = Errors.new
|
15
16
|
end
|
16
17
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
def execute_commands
|
23
|
-
response = true
|
18
|
+
# Sends each command to the redis pipline, where it is processed.
|
19
|
+
# Commands are removed from the command collection when sent to the pipeline.
|
20
|
+
# Returns <tt>true</tt> if all commands succeed. Returns <tt>false</tt> if any command fails.
|
21
|
+
def execute
|
24
22
|
begin
|
25
|
-
while
|
23
|
+
while commands.length > 0
|
26
24
|
pipeline_commands(command_batch)
|
27
25
|
end
|
28
26
|
rescue => error
|
29
|
-
|
30
|
-
response = false
|
27
|
+
errors << error.message
|
31
28
|
end
|
32
|
-
|
29
|
+
|
30
|
+
errors.empty?
|
33
31
|
end
|
34
32
|
|
33
|
+
# Adds commands to the piplined. Pass either a single command in a string or an array of commands
|
34
|
+
# pipeline.add_command('set|hello|world') => [all_commands]
|
35
|
+
# pipeline.add_command(['hset|gem|first_name|redis', '|hset|gem|last_name|pipeline']) => [all_commands]
|
36
|
+
# Within a command the | character is used to separate the parts of the command.
|
37
|
+
# Returns an array of all commands that have been added.
|
38
|
+
# commands are removed from the array when they are executed
|
39
|
+
def <<(new_commands)
|
40
|
+
new_commands = [new_commands] if !new_commands.respond_to?(:each)
|
41
|
+
commands.concat(new_commands)
|
42
|
+
end
|
43
|
+
alias_method :add_command, :<<
|
44
|
+
|
35
45
|
private
|
36
46
|
|
37
|
-
|
47
|
+
attr_writer :errors
|
48
|
+
attr_accessor :redis, :commands
|
38
49
|
|
39
50
|
def command_batch
|
40
51
|
command_batch = []
|
41
|
-
|
42
|
-
command_batch <<
|
52
|
+
commands.first(@settings[:batch_size]).count.times do
|
53
|
+
command_batch << commands.shift
|
43
54
|
end
|
44
55
|
command_batch
|
45
56
|
end
|
@@ -54,10 +65,10 @@ module RedisPipeline
|
|
54
65
|
end
|
55
66
|
|
56
67
|
def pipeline_commands(command_batch)
|
57
|
-
|
68
|
+
redis.pipelined do
|
58
69
|
command_batch.each do |command|
|
59
70
|
redis_args = command.split("|")
|
60
|
-
|
71
|
+
redis.send(*redis_args)
|
61
72
|
end
|
62
73
|
end
|
63
74
|
end
|
data/lib/redis_pipeline.rb
CHANGED
data/redis_pipeline.gemspec
CHANGED
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
|
|
14
14
|
gem.name = "redis_pipeline"
|
15
15
|
gem.require_paths = ["lib"]
|
16
16
|
gem.version = RedisPipeline::VERSION
|
17
|
-
gem.add_dependency('gem_configurator')
|
17
|
+
gem.add_dependency('gem_configurator', '~> 1.0.0')
|
18
18
|
gem.add_dependency 'redis'
|
19
19
|
gem.add_development_dependency 'rake'
|
20
20
|
end
|
data/test/test_redis_pipeline.rb
CHANGED
@@ -23,28 +23,40 @@ class TestRedisPipeline < Test::Unit::TestCase
|
|
23
23
|
|
24
24
|
def test_add_commands_adds_array_of_commands_as_seperate_commands
|
25
25
|
commands = ["hset|person:0|first_name|joe", "hest|person:0|last_name|smith"]
|
26
|
-
@pipeline.
|
26
|
+
@pipeline.add_command(commands)
|
27
|
+
assert_equal commands.length, @pipeline.send(:commands).length
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_shovel_command_adds_array_of_commands_as_seperate_commands
|
31
|
+
commands = ["hset|person:0|first_name|joe", "hest|person:0|last_name|smith"]
|
32
|
+
@pipeline << commands
|
27
33
|
assert_equal commands.length, @pipeline.send(:commands).length
|
28
34
|
end
|
29
35
|
|
30
36
|
def test_add_commands_adds_string_as_single_command
|
31
37
|
commands = "hset|person:0 first_name joe"
|
32
|
-
@pipeline.
|
38
|
+
@pipeline.add_command(commands)
|
39
|
+
assert_equal 1, @pipeline.send(:commands).length
|
40
|
+
end
|
41
|
+
|
42
|
+
def test_shovel_command_adds_string_as_single_command
|
43
|
+
commands = "hset|person:0 first_name joe"
|
44
|
+
@pipeline << commands
|
33
45
|
assert_equal 1, @pipeline.send(:commands).length
|
34
46
|
end
|
35
47
|
|
36
48
|
def test_add_commands_queues_commands_at_end
|
37
49
|
commands = ["hset|person:0|first_name|joe", "hest|person:0|last_name|smith"]
|
38
|
-
@pipeline.
|
50
|
+
@pipeline.add_command(commands)
|
39
51
|
last_command = "hset|person:1|first_name|jane"
|
40
|
-
@pipeline.
|
52
|
+
@pipeline.add_command(last_command)
|
41
53
|
assert_equal commands[0], @pipeline.send(:commands).first
|
42
54
|
assert_equal last_command, @pipeline.send(:commands).last
|
43
55
|
end
|
44
56
|
|
45
57
|
def test_command_batch_returns_batch_size_number_of_items
|
46
58
|
full_command_set = three_batches_of_commands
|
47
|
-
@pipeline.
|
59
|
+
@pipeline.add_command(full_command_set)
|
48
60
|
single_batch = @pipeline.send(:command_batch)
|
49
61
|
|
50
62
|
upper_limit = (@pipeline.settings[:batch_size] - 1)
|
@@ -52,7 +64,7 @@ class TestRedisPipeline < Test::Unit::TestCase
|
|
52
64
|
end
|
53
65
|
|
54
66
|
def test_command_batch_takes_batch_size_items_out_of_commands
|
55
|
-
@pipeline.
|
67
|
+
@pipeline.add_command(three_batches_of_commands)
|
56
68
|
count = @pipeline.send(:commands).length
|
57
69
|
@pipeline.send(:command_batch)
|
58
70
|
assert_equal (count - @pipeline.settings[:batch_size]), @pipeline.send(:commands).length
|
@@ -61,39 +73,39 @@ class TestRedisPipeline < Test::Unit::TestCase
|
|
61
73
|
def test_execute_sends_commands_to_redis
|
62
74
|
uri_parsed = URI.parse(@uri)
|
63
75
|
redis = Redis.new(:host => uri_parsed.host, :port => uri_parsed.port, :password => uri_parsed.password)
|
64
|
-
@pipeline.
|
76
|
+
@pipeline.add_command(three_batches_of_commands)
|
65
77
|
|
66
78
|
first_command = @pipeline.send(:commands).first.split("|")
|
67
79
|
last_command = @pipeline.send(:commands).last.split("|")
|
68
80
|
assert_equal Hash.new(), redis.send("hgetall", first_command[1])
|
69
81
|
assert_equal Hash.new(), redis.send("hgetall", last_command[1])
|
70
82
|
|
71
|
-
@pipeline.
|
83
|
+
@pipeline.execute
|
72
84
|
assert_equal first_command.last, redis.send("hget", *first_command[1..-2])
|
73
85
|
assert_equal last_command.last, redis.send("hget", *last_command[1..-2])
|
74
86
|
end
|
75
87
|
|
76
88
|
def test_after_execute_no_items_in_command
|
77
|
-
@pipeline.
|
78
|
-
@pipeline.
|
89
|
+
@pipeline.add_command(three_batches_of_commands)
|
90
|
+
@pipeline.execute
|
79
91
|
assert_equal 0, @pipeline.send(:commands).length
|
80
92
|
end
|
81
93
|
|
82
94
|
def test_execute_commands_returns_true_if_successful
|
83
|
-
@pipeline.
|
84
|
-
assert @pipeline.
|
95
|
+
@pipeline.add_command(three_batches_of_commands)
|
96
|
+
assert @pipeline.execute
|
85
97
|
end
|
86
98
|
|
87
99
|
def test_execute_commands_returns_false_if_error
|
88
100
|
mismatched_commands = ['set|"string_key"|"string_value"', 'hget|"string_key"|"string_not_a_hash"']
|
89
|
-
@pipeline.
|
90
|
-
assert_equal false, @pipeline.
|
101
|
+
@pipeline.add_command(mismatched_commands)
|
102
|
+
assert_equal false, @pipeline.execute
|
91
103
|
end
|
92
104
|
|
93
105
|
def test_execute_commands_populates_errors_if_error
|
94
106
|
mismatched_commands = ['set|"string_key"|"string_value"', 'hget|"string_key"|"string_not_a_hash"']
|
95
|
-
@pipeline.
|
96
|
-
@pipeline.
|
107
|
+
@pipeline.add_command(mismatched_commands)
|
108
|
+
@pipeline.execute
|
97
109
|
assert_equal 1, @pipeline.errors.count, "#{@pipeline.errors.inspect}"
|
98
110
|
assert_equal 'ERR Operation against a key holding the wrong kind of value', @pipeline.errors[0]
|
99
111
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis_pipeline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 1.0.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,22 +10,22 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2012-
|
13
|
+
date: 2012-09-05 00:00:00.000000000Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: gem_configurator
|
17
|
-
requirement: &
|
17
|
+
requirement: &2169692560 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
|
-
- -
|
20
|
+
- - ~>
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version:
|
22
|
+
version: 1.0.0
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
|
-
version_requirements: *
|
25
|
+
version_requirements: *2169692560
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
27
|
name: redis
|
28
|
-
requirement: &
|
28
|
+
requirement: &2169692140 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
31
31
|
- - ! '>='
|
@@ -33,10 +33,10 @@ dependencies:
|
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
|
-
version_requirements: *
|
36
|
+
version_requirements: *2169692140
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
38
|
name: rake
|
39
|
-
requirement: &
|
39
|
+
requirement: &2169691680 !ruby/object:Gem::Requirement
|
40
40
|
none: false
|
41
41
|
requirements:
|
42
42
|
- - ! '>='
|
@@ -44,7 +44,7 @@ dependencies:
|
|
44
44
|
version: '0'
|
45
45
|
type: :development
|
46
46
|
prerelease: false
|
47
|
-
version_requirements: *
|
47
|
+
version_requirements: *2169691680
|
48
48
|
description: Send commands to a redis server in pipelined batches
|
49
49
|
email:
|
50
50
|
- iwhitney@ssa-i.org
|
@@ -62,6 +62,8 @@ files:
|
|
62
62
|
- lib/generators/redis_pipeline/install/install_generator.rb
|
63
63
|
- lib/generators/redis_pipeline/install/templates/redis_pipeline.example.yml
|
64
64
|
- lib/redis_pipeline.rb
|
65
|
+
- lib/redis_pipeline/commands.rb
|
66
|
+
- lib/redis_pipeline/errors.rb
|
65
67
|
- lib/redis_pipeline/redis_pipeline.rb
|
66
68
|
- lib/redis_pipeline/version.rb
|
67
69
|
- redis_pipeline.gemspec
|