redis_pipeline 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +18 -0
- data/Gemfile +5 -0
- data/LICENSE +22 -0
- data/README.md +63 -0
- data/Rakefile +11 -0
- data/lib/generators/redis_pipeline_generator.rb +7 -0
- data/lib/redis_pipeline/redis_pipeline.rb +62 -0
- data/lib/redis_pipeline/version.rb +3 -0
- data/lib/redis_pipeline.rb +8 -0
- data/lib/templates/redis_pipeline.example.yml +19 -0
- data/redis_pipeline.gemspec +21 -0
- data/test/test_helper.rb +2 -0
- data/test/test_redis_pipeline.rb +113 -0
- metadata +105 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Davin Lagerroos
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
# RedisPipeline
|
2
|
+
|
3
|
+
Creates a connection and a queue for pipelining commands to a redis server. Uses redis-rb. Intended for mass inserting of data.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'redis_pipeline'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install redis_pipeline
|
18
|
+
|
19
|
+
## Setup
|
20
|
+
|
21
|
+
Create the configuration yaml file
|
22
|
+
|
23
|
+
rails generate redis_pipeline
|
24
|
+
|
25
|
+
Populate the configuration file with the redis uri and batch size.
|
26
|
+
|
27
|
+
* uri defaults to `redis://localhost:6379` if you don't have a configuration file
|
28
|
+
* batch size defaults to 1000 if you don't have a configuration file
|
29
|
+
|
30
|
+
## Usage
|
31
|
+
|
32
|
+
Create a new pipeline
|
33
|
+
|
34
|
+
```ruby
|
35
|
+
pipeline = RedisPipeline::RedisPineline.new('redis_uri')
|
36
|
+
```
|
37
|
+
|
38
|
+
Queue up commands with add_commands either as a single string
|
39
|
+
|
40
|
+
```ruby
|
41
|
+
pipeline.add_commands('set hello world')
|
42
|
+
```
|
43
|
+
|
44
|
+
or an array
|
45
|
+
|
46
|
+
```ruby
|
47
|
+
array = ['hset gem first_name redis', 'hset gem last_name pipeline']
|
48
|
+
pipeline.add_commands(array)
|
49
|
+
```
|
50
|
+
|
51
|
+
Send them with execute_commands. Commands are sent using redis-rb's pipelined mode in batches, the size of which are controlled by your configuration. Returns false if there is an error
|
52
|
+
|
53
|
+
```ruby
|
54
|
+
pipeline.execute_commands
|
55
|
+
```
|
56
|
+
|
57
|
+
## Contributing
|
58
|
+
|
59
|
+
1. Fork it
|
60
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
61
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
62
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
63
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
module RedisPipeline
|
2
|
+
class RedisPipeline
|
3
|
+
# include GemConfigurator
|
4
|
+
|
5
|
+
require 'uri'
|
6
|
+
require 'redis'
|
7
|
+
|
8
|
+
def initialize()
|
9
|
+
configure
|
10
|
+
@redis = open_redis_connection
|
11
|
+
@commands = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def add_commands(new_commands)
|
15
|
+
new_commands = [new_commands] if new_commands.class == String
|
16
|
+
@commands.concat(new_commands)
|
17
|
+
end
|
18
|
+
|
19
|
+
def execute_commands
|
20
|
+
response = true
|
21
|
+
begin
|
22
|
+
while @commands.length > 0
|
23
|
+
pipeline_commands(command_batch)
|
24
|
+
end
|
25
|
+
rescue
|
26
|
+
response = false
|
27
|
+
end
|
28
|
+
response
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
attr_accessor :commands, :redis
|
34
|
+
|
35
|
+
def command_batch
|
36
|
+
command_batch = []
|
37
|
+
@commands.first(@settings[:batch_size]).count.times do
|
38
|
+
command_batch << @commands.shift
|
39
|
+
end
|
40
|
+
command_batch
|
41
|
+
end
|
42
|
+
|
43
|
+
def default_settings
|
44
|
+
{:uri => 'redis://localhost:6379', :batch_size => 1000}
|
45
|
+
end
|
46
|
+
|
47
|
+
def open_redis_connection
|
48
|
+
uri = URI.parse(settings[:uri])
|
49
|
+
Redis.new(:host => uri.host, :port => uri.port, :password => uri.password)
|
50
|
+
end
|
51
|
+
|
52
|
+
def pipeline_commands(command_batch)
|
53
|
+
@redis.pipelined do
|
54
|
+
command_batch.each do |command|
|
55
|
+
redis_args = command.split(" ")
|
56
|
+
@redis.send(*redis_args)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
#
|
2
|
+
# Format:
|
3
|
+
#
|
4
|
+
# <stage name>:
|
5
|
+
# uri: <redis uri, defaults to 'redis://localhost:6379'>
|
6
|
+
# pipeline_batch_size: <Number of commands to pipeline, defaults to 1000>
|
7
|
+
|
8
|
+
development: &defaults
|
9
|
+
uri: 'redis://localhost:6379'
|
10
|
+
pipeline_batch_size: 1000
|
11
|
+
|
12
|
+
test:
|
13
|
+
<<: *defaults
|
14
|
+
|
15
|
+
staging:
|
16
|
+
<<: *defaults
|
17
|
+
|
18
|
+
production:
|
19
|
+
<<: *defaults
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/redis_pipeline/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Davin Lagerroos"]
|
6
|
+
gem.email = ["dlagerroos@ssa-i.org"]
|
7
|
+
gem.description = %q{Send commands to a redis server in pipelined batches}
|
8
|
+
gem.summary = %q{Establishes a connection to a redis server. You can then pass it commands that will be queued to be pipelined. Intended for mass upload.}
|
9
|
+
gem.homepage = ""
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "redis_pipeline"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = RedisPipeline::VERSION
|
17
|
+
gem.add_dependency('gem_configurator')
|
18
|
+
gem.add_dependency('active_support')
|
19
|
+
gem.add_dependency 'redis'
|
20
|
+
gem.add_development_dependency 'rake'
|
21
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class TestRedisPipeline < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
@pipeline = RedisPipeline::RedisPipeline.new
|
7
|
+
@uri = @pipeline.settings[:uri]
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
uri_parsed = URI.parse(@uri)
|
12
|
+
redis = Redis.new(:host => uri_parsed.host, :port => uri_parsed.port, :password => uri_parsed.password)
|
13
|
+
redis.flushall
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_pipeline_establishes_connection_to_redis_server
|
17
|
+
assert_not_nil @pipeline
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_pipeline_has_empty_command_list
|
21
|
+
assert_equal 0, @pipeline.send(:commands).length
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_add_commands_adds_array_of_commands_as_seperate_commands
|
25
|
+
commands = ["hset person:0 first_name joe", "hest person:0 last_name smith"]
|
26
|
+
@pipeline.add_commands(commands)
|
27
|
+
assert_equal commands.length, @pipeline.send(:commands).length
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_add_commands_adds_string_as_single_command
|
31
|
+
commands = "hset person:0 first_name joe"
|
32
|
+
@pipeline.add_commands(commands)
|
33
|
+
assert_equal 1, @pipeline.send(:commands).length
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_add_commands_queues_commands_at_end
|
37
|
+
commands = ["hset person:0 first_name joe", "hest person:0 last_name smith"]
|
38
|
+
@pipeline.add_commands(commands)
|
39
|
+
last_command = "hset person:1 first_name jane"
|
40
|
+
@pipeline.add_commands(last_command)
|
41
|
+
assert_equal commands[0], @pipeline.send(:commands).first
|
42
|
+
assert_equal last_command, @pipeline.send(:commands).last
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_command_batch_returns_batch_size_number_of_items
|
46
|
+
full_command_set = three_batches_of_commands
|
47
|
+
@pipeline.add_commands(full_command_set)
|
48
|
+
single_batch = @pipeline.send(:command_batch)
|
49
|
+
|
50
|
+
upper_limit = (@pipeline.settings[:batch_size] - 1)
|
51
|
+
assert_equal full_command_set[0..upper_limit], single_batch
|
52
|
+
end
|
53
|
+
|
54
|
+
def test_command_batch_takes_batch_size_items_out_of_commands
|
55
|
+
@pipeline.add_commands(three_batches_of_commands)
|
56
|
+
count = @pipeline.send(:commands).length
|
57
|
+
@pipeline.send(:command_batch)
|
58
|
+
assert_equal (count - @pipeline.settings[:batch_size]), @pipeline.send(:commands).length
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_execute_sends_commands_to_redis
|
62
|
+
uri_parsed = URI.parse(@uri)
|
63
|
+
redis = Redis.new(:host => uri_parsed.host, :port => uri_parsed.port, :password => uri_parsed.password)
|
64
|
+
@pipeline.add_commands(three_batches_of_commands)
|
65
|
+
|
66
|
+
first_command = @pipeline.send(:commands).first.split(" ")
|
67
|
+
last_command = @pipeline.send(:commands).last.split(" ")
|
68
|
+
assert_equal Hash.new(), redis.send("hgetall", first_command[1])
|
69
|
+
assert_equal Hash.new(), redis.send("hgetall", last_command[1])
|
70
|
+
|
71
|
+
@pipeline.execute_commands
|
72
|
+
assert_equal first_command.last, redis.send("hget", *first_command[1..-2])
|
73
|
+
assert_equal last_command.last, redis.send("hget", *last_command[1..-2])
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_after_execute_no_items_in_command
|
77
|
+
@pipeline.add_commands(three_batches_of_commands)
|
78
|
+
@pipeline.execute_commands
|
79
|
+
assert_equal 0, @pipeline.send(:commands).length
|
80
|
+
end
|
81
|
+
|
82
|
+
def test_execute_commands_returns_true_if_successful
|
83
|
+
@pipeline.add_commands(three_batches_of_commands)
|
84
|
+
assert @pipeline.execute_commands
|
85
|
+
end
|
86
|
+
|
87
|
+
def test_execute_commands_returns_false_if_error
|
88
|
+
mismatched_commands = ['set "string_key" "string_value"', 'hget "string_key" "string_not_a_hash"']
|
89
|
+
@pipeline.add_commands(mismatched_commands)
|
90
|
+
assert_equal false, @pipeline.execute_commands
|
91
|
+
end
|
92
|
+
|
93
|
+
private
|
94
|
+
|
95
|
+
def three_batches_of_commands
|
96
|
+
first_names = ["Lindsey", "Dodie", "Tommie", "Aletha", "Matilda", "Robby", "Forest", "Sherrie", "Elroy", "Darlene", "Blossom", "Preston", "Ivan", "Denisha", "Antonietta", "Lenora", "Fatimah", "Alvaro", "Madeleine", "Johnsie", "Jacki"]
|
97
|
+
last_names = ["Austino", "Egnor", "Mclauglin", "Vettel", "Osornio", "Kloke", "Neall", "Licon", "Bergren", "Guialdo", "Heu", "Lilla", "Fogt", "Ellinghuysen", "Banner", "Gammage", "Fleniken", "Byerley", "Mccandless", "Hatchet", "Segal", "Bagnall", "Mangum", "Marinella", "Hunke", "Klis", "Skonczewski", "Aiava", "Masson", "Hochhauser", "Pfost", "Cripps", "Surrell", "Carstens", "Moeder", "Feller", "Turri", "Plummer", "Liuzza", "Macaskill", "Pirie", "Haase", "Gummersheimer", "Caden", "Balich", "Franssen", "Barbur", "Bonker", "Millar", "Armijo", "Canales", "Kucera", "Ahlstrom", "Marcoux", "Dagel", "Vandonsel", "Lagrasse", "Bolten", "Smyer", "Spiker", "Detz", "Munar", "Oieda", "Westin", "Levenson", "Ramagos", "Lipson", "Crankshaw", "Polton", "Seibt", "Genrich", "Shempert", "Bonillas", "Stout", "Caselli", "Jaji", "Kudo", "Feauto", "Hetland", "Hsieh", "Iwasko", "Jayme", "Lebby", "Dircks", "Hainley", "Gielstra", "Dozois", "Suss", "Matern", "Mcloud", "Fassio", "Blumstein", "Qin", "Gindi", "Petrizzo", "Beath", "Tonneson", "Fraga", "Tamura", "Cappellano", "Galella"]
|
98
|
+
|
99
|
+
# each first_name,last_name pair is 2 commands so to get 2 batches plus extra we only need batch_size number pairs plus some extra * 1.33
|
100
|
+
number_of_commands = (@pipeline.settings[:batch_size] * 1.33)
|
101
|
+
commands = []
|
102
|
+
(0..number_of_commands).each do |i|
|
103
|
+
first = first_names.shift
|
104
|
+
last = last_names.shift
|
105
|
+
commands << "hset person:#{i} first_name #{first}"
|
106
|
+
commands << "hset person:#{i} lsst_name #{last}"
|
107
|
+
first_names.push(first)
|
108
|
+
last_names.push(last)
|
109
|
+
end
|
110
|
+
commands
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
metadata
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: redis_pipeline
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Davin Lagerroos
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-08-23 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: gem_configurator
|
16
|
+
requirement: &2153231100 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *2153231100
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: active_support
|
27
|
+
requirement: &2153230680 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2153230680
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: redis
|
38
|
+
requirement: &2153230260 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *2153230260
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rake
|
49
|
+
requirement: &2153229840 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :development
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *2153229840
|
58
|
+
description: Send commands to a redis server in pipelined batches
|
59
|
+
email:
|
60
|
+
- dlagerroos@ssa-i.org
|
61
|
+
executables: []
|
62
|
+
extensions: []
|
63
|
+
extra_rdoc_files: []
|
64
|
+
files:
|
65
|
+
- .gitignore
|
66
|
+
- Gemfile
|
67
|
+
- LICENSE
|
68
|
+
- README.md
|
69
|
+
- Rakefile
|
70
|
+
- lib/generators/redis_pipeline_generator.rb
|
71
|
+
- lib/redis_pipeline.rb
|
72
|
+
- lib/redis_pipeline/redis_pipeline.rb
|
73
|
+
- lib/redis_pipeline/version.rb
|
74
|
+
- lib/templates/redis_pipeline.example.yml
|
75
|
+
- redis_pipeline.gemspec
|
76
|
+
- test/test_helper.rb
|
77
|
+
- test/test_redis_pipeline.rb
|
78
|
+
homepage: ''
|
79
|
+
licenses: []
|
80
|
+
post_install_message:
|
81
|
+
rdoc_options: []
|
82
|
+
require_paths:
|
83
|
+
- lib
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
none: false
|
86
|
+
requirements:
|
87
|
+
- - ! '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
|
+
none: false
|
92
|
+
requirements:
|
93
|
+
- - ! '>='
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
requirements: []
|
97
|
+
rubyforge_project:
|
98
|
+
rubygems_version: 1.8.16
|
99
|
+
signing_key:
|
100
|
+
specification_version: 3
|
101
|
+
summary: Establishes a connection to a redis server. You can then pass it commands
|
102
|
+
that will be queued to be pipelined. Intended for mass upload.
|
103
|
+
test_files:
|
104
|
+
- test/test_helper.rb
|
105
|
+
- test/test_redis_pipeline.rb
|