redis_pipeline 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.rvmrc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in redis_pipeline.gemspec
+gemspec
+gem 'gem_configurator', :git => 'git://github.com/SeniorServiceAmerica/gem_configurator'

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Davin Lagerroos
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+# RedisPipeline
+
+Creates a connection and a queue for pipelining commands to a redis server. Uses redis-rb. Intended for mass inserting of data. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'redis_pipeline'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install redis_pipeline
+
+## Setup 
+
+Create the configuration yaml file
+
+    rails generate redis_pipeline
+
+Populate the configuration file with the redis uri and batch size.
+
+* uri defaults to `redis://localhost:6379` if you don't have a configuration file
+* batch size defaults to 1000 if you don't have a configuration file
+
+## Usage
+
+Create a new pipeline 
+
+```ruby
+ pipeline =  RedisPipeline::RedisPineline.new('redis_uri')
+```
+
+Queue up commands with add_commands either as a single string 
+
+```ruby
+ pipeline.add_commands('set hello world')
+```
+
+or an array
+
+```ruby
+	array = ['hset gem first_name redis', 'hset gem last_name pipeline']
+	pipeline.add_commands(array)
+```
+
+Send them with execute_commands. Commands are sent using redis-rb's pipelined mode in batches, the size of which are controlled by your configuration. Returns false if there is an error
+
+```ruby
+  pipeline.execute_commands
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/lib/generators/redis_pipeline_generator.rb

@@ -0,0 +1,7 @@
+class RedisPipelineGenerator < Rails::Generators::Base
+  source_root File.expand_path("../../templates", __FILE__)
+
+  def copy_initializer_file
+    copy_file "redis_pipeline.example.yml", "config/redis_pipeline.yml"
+  end
+end

data/lib/redis_pipeline/redis_pipeline.rb

@@ -0,0 +1,62 @@
+module RedisPipeline
+  class RedisPipeline
+    # include GemConfigurator
+    
+    require 'uri'
+    require 'redis'
+
+    def initialize()
+      configure
+      @redis = open_redis_connection
+      @commands = []
+    end
+    
+    def add_commands(new_commands)
+      new_commands = [new_commands] if new_commands.class == String
+      @commands.concat(new_commands)
+    end
+    
+    def execute_commands
+      response = true
+      begin
+        while @commands.length > 0
+          pipeline_commands(command_batch)
+        end
+      rescue 
+        response = false
+      end
+      response
+    end
+    
+    private
+      
+      attr_accessor :commands, :redis
+      
+      def command_batch
+        command_batch = []
+        @commands.first(@settings[:batch_size]).count.times do 
+          command_batch << @commands.shift
+        end
+        command_batch
+      end
+      
+      def default_settings
+        {:uri => 'redis://localhost:6379', :batch_size => 1000}
+      end
+
+      def open_redis_connection
+        uri = URI.parse(settings[:uri])
+        Redis.new(:host => uri.host, :port => uri.port, :password => uri.password)
+      end
+      
+      def pipeline_commands(command_batch)
+        @redis.pipelined do 
+          command_batch.each do |command|
+            redis_args = command.split(" ")
+            @redis.send(*redis_args)
+          end
+        end
+      end
+      
+  end
+end

data/lib/redis_pipeline/version.rb

@@ -0,0 +1,3 @@
+module RedisPipeline
+  VERSION = "0.0.1"
+end

data/lib/redis_pipeline.rb

@@ -0,0 +1,8 @@
+require_relative 'redis_pipeline/version'
+require 'redis'
+require 'active_support/inflector'
+require_relative 'redis_pipeline/redis_pipeline'
+
+require 'bundler'
+Bundler.require
+include GemConfigurator

data/lib/templates/redis_pipeline.example.yml

@@ -0,0 +1,19 @@
+#
+# Format:
+# 
+# <stage name>:
+#   uri: <redis uri, defaults to 'redis://localhost:6379'>
+#   pipeline_batch_size: <Number of commands to pipeline, defaults to 1000>
+
+development: &defaults
+  uri: 'redis://localhost:6379'
+  pipeline_batch_size: 1000
+
+test:
+  <<: *defaults
+
+staging:
+  <<: *defaults
+  
+production:
+  <<: *defaults

data/redis_pipeline.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/redis_pipeline/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Davin Lagerroos"]
+  gem.email         = ["dlagerroos@ssa-i.org"]
+  gem.description   = %q{Send commands to a redis server in pipelined batches}
+  gem.summary       = %q{Establishes a connection to a redis server. You can then pass it commands that will be queued to be pipelined. Intended for mass upload.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "redis_pipeline"
+  gem.require_paths = ["lib"]
+  gem.version       = RedisPipeline::VERSION
+  gem.add_dependency('gem_configurator')
+  gem.add_dependency('active_support')
+  gem.add_dependency              'redis' 
+  gem.add_development_dependency  'rake'  
+end

data/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require 'redis_pipeline'

data/test/test_redis_pipeline.rb

@@ -0,0 +1,113 @@
+require 'test_helper'
+
+class TestRedisPipeline < Test::Unit::TestCase
+
+  def setup 
+    @pipeline = RedisPipeline::RedisPipeline.new
+    @uri = @pipeline.settings[:uri]
+  end
+  
+  def teardown 
+    uri_parsed = URI.parse(@uri)
+    redis = Redis.new(:host => uri_parsed.host, :port => uri_parsed.port, :password => uri_parsed.password)
+    redis.flushall
+  end
+
+  def test_pipeline_establishes_connection_to_redis_server
+    assert_not_nil @pipeline
+  end
+  
+  def test_pipeline_has_empty_command_list
+    assert_equal 0, @pipeline.send(:commands).length
+  end
+  
+  def test_add_commands_adds_array_of_commands_as_seperate_commands
+    commands = ["hset person:0 first_name joe", "hest person:0 last_name smith"]
+    @pipeline.add_commands(commands)
+    assert_equal commands.length, @pipeline.send(:commands).length
+  end
+  
+  def test_add_commands_adds_string_as_single_command
+    commands = "hset person:0 first_name joe"
+    @pipeline.add_commands(commands)
+    assert_equal 1, @pipeline.send(:commands).length
+  end
+  
+  def test_add_commands_queues_commands_at_end
+    commands = ["hset person:0 first_name joe", "hest person:0 last_name smith"]
+    @pipeline.add_commands(commands)
+    last_command = "hset person:1 first_name jane"
+    @pipeline.add_commands(last_command)
+    assert_equal commands[0], @pipeline.send(:commands).first
+    assert_equal last_command, @pipeline.send(:commands).last
+  end
+  
+  def test_command_batch_returns_batch_size_number_of_items
+    full_command_set = three_batches_of_commands
+    @pipeline.add_commands(full_command_set)
+    single_batch = @pipeline.send(:command_batch)
+    
+    upper_limit = (@pipeline.settings[:batch_size] - 1)
+    assert_equal full_command_set[0..upper_limit], single_batch 
+  end
+  
+  def test_command_batch_takes_batch_size_items_out_of_commands
+    @pipeline.add_commands(three_batches_of_commands)
+    count = @pipeline.send(:commands).length
+    @pipeline.send(:command_batch)
+    assert_equal (count - @pipeline.settings[:batch_size]), @pipeline.send(:commands).length
+  end
+  
+  def test_execute_sends_commands_to_redis
+    uri_parsed = URI.parse(@uri)
+    redis = Redis.new(:host => uri_parsed.host, :port => uri_parsed.port, :password => uri_parsed.password)    
+    @pipeline.add_commands(three_batches_of_commands)
+    
+    first_command = @pipeline.send(:commands).first.split(" ")
+    last_command = @pipeline.send(:commands).last.split(" ")
+    assert_equal Hash.new(), redis.send("hgetall", first_command[1])
+    assert_equal Hash.new(), redis.send("hgetall", last_command[1])
+    
+    @pipeline.execute_commands
+    assert_equal first_command.last, redis.send("hget", *first_command[1..-2])
+    assert_equal last_command.last, redis.send("hget", *last_command[1..-2])
+  end
+  
+  def test_after_execute_no_items_in_command
+    @pipeline.add_commands(three_batches_of_commands)
+    @pipeline.execute_commands
+    assert_equal 0, @pipeline.send(:commands).length
+  end
+  
+  def test_execute_commands_returns_true_if_successful
+    @pipeline.add_commands(three_batches_of_commands)
+    assert @pipeline.execute_commands
+  end
+  
+  def test_execute_commands_returns_false_if_error
+    mismatched_commands = ['set "string_key" "string_value"', 'hget "string_key" "string_not_a_hash"']
+    @pipeline.add_commands(mismatched_commands)
+    assert_equal false, @pipeline.execute_commands
+  end
+  
+  private
+
+    def three_batches_of_commands
+      first_names = ["Lindsey", "Dodie", "Tommie", "Aletha", "Matilda", "Robby", "Forest", "Sherrie", "Elroy", "Darlene", "Blossom", "Preston", "Ivan", "Denisha", "Antonietta", "Lenora", "Fatimah", "Alvaro", "Madeleine", "Johnsie", "Jacki"]
+      last_names = ["Austino", "Egnor", "Mclauglin", "Vettel", "Osornio", "Kloke", "Neall", "Licon", "Bergren", "Guialdo", "Heu", "Lilla", "Fogt", "Ellinghuysen", "Banner", "Gammage", "Fleniken", "Byerley", "Mccandless", "Hatchet", "Segal", "Bagnall", "Mangum", "Marinella", "Hunke", "Klis", "Skonczewski", "Aiava", "Masson", "Hochhauser", "Pfost", "Cripps", "Surrell", "Carstens", "Moeder", "Feller", "Turri", "Plummer", "Liuzza", "Macaskill", "Pirie", "Haase", "Gummersheimer", "Caden", "Balich", "Franssen", "Barbur", "Bonker", "Millar", "Armijo", "Canales", "Kucera", "Ahlstrom", "Marcoux", "Dagel", "Vandonsel", "Lagrasse", "Bolten", "Smyer", "Spiker", "Detz", "Munar", "Oieda", "Westin", "Levenson", "Ramagos", "Lipson", "Crankshaw", "Polton", "Seibt", "Genrich", "Shempert", "Bonillas", "Stout", "Caselli", "Jaji", "Kudo", "Feauto", "Hetland", "Hsieh", "Iwasko", "Jayme", "Lebby", "Dircks", "Hainley", "Gielstra", "Dozois", "Suss", "Matern", "Mcloud", "Fassio", "Blumstein", "Qin", "Gindi", "Petrizzo", "Beath", "Tonneson", "Fraga", "Tamura", "Cappellano", "Galella"]
+
+      # each first_name,last_name pair is 2 commands so to get 2 batches plus extra we only need batch_size number pairs plus some extra * 1.33
+      number_of_commands = (@pipeline.settings[:batch_size]  * 1.33)
+      commands = []
+      (0..number_of_commands).each do |i|
+        first = first_names.shift
+        last = last_names.shift        
+        commands << "hset person:#{i} first_name #{first}"
+        commands << "hset person:#{i} lsst_name #{last}" 
+        first_names.push(first)
+        last_names.push(last)
+      end
+      commands 
+    end
+  
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: redis_pipeline
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Davin Lagerroos
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-23 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gem_configurator
+  requirement: &2153231100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153231100
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: &2153230680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153230680
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &2153230260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153230260
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2153229840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2153229840
+description: Send commands to a redis server in pipelined batches
+email:
+- dlagerroos@ssa-i.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/generators/redis_pipeline_generator.rb
+- lib/redis_pipeline.rb
+- lib/redis_pipeline/redis_pipeline.rb
+- lib/redis_pipeline/version.rb
+- lib/templates/redis_pipeline.example.yml
+- redis_pipeline.gemspec
+- test/test_helper.rb
+- test/test_redis_pipeline.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.16
+signing_key: 
+specification_version: 3
+summary: Establishes a connection to a redis server. You can then pass it commands
+  that will be queued to be pipelined. Intended for mass upload.
+test_files:
+- test/test_helper.rb
+- test/test_redis_pipeline.rb