redis_open3 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 060a708ef7b2533866b91169cd1b2890500f7315
+  data.tar.gz: 370b16a073547778f1c8c66a03ea0d281c20907f
+SHA512:
+  metadata.gz: 39d569cdc09b5295fcc10c14ecb40d0faafed89548b5d936a3c678b123a631f0d453354aa10e9df91171a1c7a8f310bfc152dcb37f9fa3ef795449f444f0dc6e
+  data.tar.gz: e4e8d305f0fb35b8b77c69a21a6a1bb4d1e920941190adb79db01d704a8690e58a061604aa24eb4585bac0345d462859745e6169270f5ece947e4a735fe33d57

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in redis_open3.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Tyler Hartland
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,80 @@
+# RedisOpen3
+
+Provides Open3 like syntax for passing data through Redis.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'redis_open3'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install redis_open3
+
+## Usage
+
+Initialize a RedisOpen3 object giving a method which will yield redis connections.
+```
+# example using ConnectinPool gem
+conn_pool = ConnectionPool.new(size: 9, timeout: 10) { Redis.new }
+ropen3 = RedisOpen3.new(conn_pool.method(:with))
+```
+
+Use #popen3 to start passing data through Redis.
+```
+# example using RedisOpen3 to pass data to a different Sidekiq instance
+ropen3.popen3 do |redis_in, redis_out, redis_err, uuids|
+  # start the foreign Sidekiq job
+  # The worker must be pre-configured to use a queue worked by the foreign process
+  ForeignWorker.perform_async(args, uuids)
+
+  # Then feed data across Redis
+  # Using threads allows true streaming between processes/workers
+  threads = [] << Thread.new do
+    input_data.each { |row| redis_in << row }
+    # Important to signal that we're done sending data
+    redis_in.close
+  end
+
+  # Catch your return data
+  threads << Thread.new do
+    redis_out.each { |row| output_file << row }
+  end
+
+  # Catch logging/error messages
+  threads << Thread.new do
+    redis_err.each { |row| log_file << row }
+  end
+
+  threads.each(&:join)
+  # You should have already closed redis_in yourself, but it will also be closed if you haven't
+  # redis_out and redis_err lists will be deleted
+end
+```
+
+Use #process3 to receive input and return data
+```
+# This could be used as the body of the perform method on a Sidekiq worker
+# Assumes uuids were send across as args
+ropen3.process3(uuids) do |redis_in, redis_out, redis_err|
+  # Pull your input data from Redis
+  # And send your results back across Redis
+  redis_in.each { |row| redis_out << process(row) }
+
+  # Any exceptions raised from this block will be sent to redis_err
+  # redis_out and redis_err will be closed
+  # redis_in will be deleted
+end
+```
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/redis_open3/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/redis_open3/enum.rb

@@ -0,0 +1,55 @@
+require 'redis'
+require 'redis_open3'
+
+class RedisOpen3
+  class Enum
+    class TimeoutError < RedisOpen3::Error; end
+
+    include Enumerable
+    attr_reader :list_name
+
+    # magic :(
+    EOF = '90829c12-3ac3-4af9-aeb4-8b63d1cc1d23'
+    EOF_REGX = /\A#{EOF}\z/
+
+    def initialize(list_name, opts={})
+      @list_name = list_name
+      @timeout   = opts[:timeout]
+      @redis     = opts[:redis] || Redis.new(opts[:redis_opts] || {})
+    end
+
+    def <<(input)
+      push(input)
+    end
+
+    def close
+      push(EOF)
+    end
+
+    def each(&block)
+      Enumerator.new do |y|
+        until (line = pop.chomp) =~ EOF_REGX
+          y << line
+        end
+      end.each(&block)
+    end
+
+    def delete
+      @redis.del(list_name)
+    end
+
+    private
+
+    def push(string)
+      @redis.rpush(list_name, string)
+    end
+
+    def pop
+      if popped = @redis.blpop(list_name, @timeout)
+        popped.last
+      else
+        raise TimeoutError, "Timeout of #{@timeout.inspect} seconds expired on list_name #{list_name.inspect}."
+      end
+    end
+  end
+end

data/lib/redis_open3/version.rb

@@ -0,0 +1,3 @@
+class RedisOpen3
+  VERSION = "0.0.1"
+end

data/lib/redis_open3.rb

@@ -0,0 +1,70 @@
+require "redis_open3/version"
+require 'redis'
+
+class RedisOpen3
+  class Error < StandardError; end
+
+  IN_KEY  = 'redis_open3_in'
+  OUT_KEY = 'redis_open3_out'
+  ERR_KEY = 'redis_open3_err'
+
+  def initialize(redis_pool, opts={})
+    @redis_pool = redis_pool
+    @timeout    = opts[:timeout] || 900
+  end
+
+  def open3
+    with_enums(generated_uuids) do |redis_in, redis_out, redis_err, uuids|
+      begin
+        yield redis_in, redis_out, redis_err, uuids
+      ensure
+        redis_in.close
+        redis_out.delete
+        redis_err.delete
+      end
+    end
+  end
+
+  def process3(uuids)
+    with_enums(uuids) do |redis_in, redis_out, redis_err|
+      begin
+        yield redis_in, redis_out, redis_err
+      rescue Exception => e
+        ([e.inspect] + e.backtrace).each { |row| redis_err << row }
+        raise e
+      ensure
+        redis_in.delete
+        redis_out.close
+        redis_err.close
+      end
+    end
+  end
+
+  private
+
+  def with_conns
+    @redis_pool.call do |redis1|
+      @redis_pool.call do |redis2|
+        @redis_pool.call do |redis3|
+          yield redis1, redis2, redis3
+        end
+      end
+    end
+  end
+
+  def with_enums(uuids)
+    with_conns do |redis1, redis2, redis3|
+      redis_in  = Enum.new(uuids[IN_KEY],  redis: redis1, timeout: @timeout)
+      redis_out = Enum.new(uuids[OUT_KEY], redis: redis2, timeout: @timeout)
+      redis_err = Enum.new(uuids[ERR_KEY], redis: redis3, timeout: @timeout)
+      yield redis_in, redis_out, redis_err, uuids
+    end
+  end
+
+  def generated_uuids
+    [ IN_KEY, OUT_KEY, ERR_KEY ].inject({}) do |result, key|
+      result[key] = SecureRandom.uuid
+      result
+    end
+  end
+end

data/redis_open3.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'redis_open3/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "redis_open3"
+  spec.version       = RedisOpen3::VERSION
+  spec.authors       = ["Tyler Hartland"]
+  spec.email         = ["tyler.hartland@code42.com"]
+  spec.summary       = %q{Use Open3 like sematics to pass data through Redis.}
+  # spec.description   = %q{TODO: Write a longer description. Optional.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency 'rspec', '~> 3.0.0'
+
+  spec.add_runtime_dependency 'redis', '~> 3.1.0'
+end

data/spec/lib/redis_open3/enum_spec.rb

@@ -0,0 +1,71 @@
+require 'redis_open3/enum'
+
+describe RedisOpen3::Enum do
+  let(:list_name) { 'redis_open3:redis_enum:test_list' }
+  let(:enum) { RedisOpen3::Enum.new(list_name, redis: RedisConn.conn, timeout: 1) }
+  let(:eof) { RedisOpen3::Enum::EOF }
+
+  after { enum.delete }
+
+  describe '#<<' do
+    it 'adds an item to the redis list' do
+      RedisConn.with do |redis|
+        expect {
+          enum << 'test_item'
+        }.to change {
+          redis.lpop(list_name)
+        }.from(nil).to('test_item')
+      end
+    end
+  end
+
+  describe '#close' do
+    it 'adds the EOF uuid to the redis list' do
+      RedisConn.with do |redis|
+        expect {
+          enum.close
+        }.to change {
+          redis.lpop(list_name)
+        }.from(nil).to(eof)
+      end
+    end
+  end
+
+  describe '#each' do
+    let(:expected_items) { ['item1', 'item2', 'item3'] }
+
+    it 'blocks until it can yield each item in the list' do
+      Thread.new do
+        RedisConn.with do |redis|
+          expected_items.each do |item|
+            redis.rpush(list_name, item)
+            sleep 0.02
+          end
+          redis.rpush(list_name, eof)
+        end
+      end
+
+      found_items = enum.each.to_a
+      expected_items.each_with_index { |item, i| expect(found_items[i]).to eq item }
+    end
+  end
+
+  describe '#delete' do
+    it 'deletes the redis list' do
+      RedisConn.with do |redis|
+        enum << 'junk'
+        expect {
+          enum.delete
+        }.to change {
+          redis.exists(list_name)
+        }.from(true).to(false)
+      end
+    end
+  end
+
+  context 'timeout' do
+    it 'raises a timeout error if the timeout expires' do
+      expect { enum.each.to_a }.to raise_error(RedisOpen3::Enum::TimeoutError)
+    end
+  end
+end

data/spec/lib/redis_open3_spec.rb

@@ -0,0 +1,73 @@
+require 'redis_open3'
+
+describe RedisOpen3 do
+  let(:ropen) { RedisOpen3.new(conns, timeout: 1) }
+  let(:conns) { RedisConn.method(:with) }
+
+  describe '#popen3' do
+    it 'yields 3 redis enum objects and an array with their list uuids' do
+      ropen.open3 do |r_in, r_out, r_err, uuids|
+        expect(uuids[RedisOpen3::IN_KEY]).to  eq r_in.list_name
+        expect(uuids[RedisOpen3::OUT_KEY]).to eq r_out.list_name
+        expect(uuids[RedisOpen3::ERR_KEY]).to eq r_err.list_name
+      end
+    end
+
+    it 'sends EOF to redis in enum after block closes' do
+      r_in = nil
+      ropen.open3 { |redis_in| r_in = redis_in }
+      expect(r_in.each.to_a).to eq []
+    end
+
+    it 'deletes the redis_out and redis_err list' do
+      out_name = nil
+      err_name = nil
+      conns.call do |redis|
+        ropen.open3 do |_, redis_out, redis_err|
+          out_name = redis_out.list_name
+          err_name = redis_err.list_name
+          redis_out << 'junk'
+          redis_err << 'junk'
+          expect(redis.exists(out_name)).to eq true
+          expect(redis.exists(err_name)).to eq true
+        end
+        expect(redis.exists(out_name)).to eq false
+        expect(redis.exists(err_name)).to eq false
+      end
+    end
+  end
+
+  describe '#process3' do
+    let(:uuids) {{
+      RedisOpen3::IN_KEY  => SecureRandom.uuid,
+      RedisOpen3::OUT_KEY => SecureRandom.uuid,
+      RedisOpen3::ERR_KEY => SecureRandom.uuid
+    }}
+
+    it 'yields 3 redis enum objects with given list names' do
+      ropen.process3(uuids) do |r_in, r_out, r_err|
+        expect(r_in.list_name).to  eq uuids[RedisOpen3::IN_KEY]
+        expect(r_out.list_name).to eq uuids[RedisOpen3::OUT_KEY]
+        expect(r_err.list_name).to eq uuids[RedisOpen3::ERR_KEY]
+      end
+    end
+
+    it 'sends EOF to redis out and redis err enums after block closes' do
+      r_out = nil
+      r_err = nil
+      ropen.process3(uuids) { |_, redis_out, redis_err| r_out = redis_out; r_err = redis_err }
+      expect(r_out.each.to_a).to eq []
+      expect(r_err.each.to_a).to eq []
+    end
+
+    it 'deletes the redis_in list' do
+      conns.call do |redis|
+        ropen.process3(uuids) do |redis_in|
+          redis_in << 'junk'
+          expect(redis.exists(uuids[RedisOpen3::IN_KEY])).to eq true
+        end
+        expect(redis.exists(uuids[RedisOpen3::IN_KEY])).to eq false
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,103 @@
+lib_path = "#{File.dirname(__FILE__)}/../lib"
+$LOAD_PATH << lib_path unless $LOAD_PATH.include?(lib_path)
+
+class RedisConn
+  def self.with
+    redis = new_conn
+    yield redis
+    redis.quit
+  end
+
+  def self.conn
+    new_conn
+  end
+
+  def self.flushdb
+    with { |redis| redis.flushdb }
+  end
+
+  private
+
+  def self.new_conn
+    Redis.new(db: 14)
+  end
+end
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.after(:suite) { RedisConn.flushdb }
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: redis_open3
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Tyler Hartland
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: 
+email:
+- tyler.hartland@code42.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/redis_open3.rb
+- lib/redis_open3/enum.rb
+- lib/redis_open3/version.rb
+- redis_open3.gemspec
+- spec/lib/redis_open3/enum_spec.rb
+- spec/lib/redis_open3_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Use Open3 like sematics to pass data through Redis.
+test_files:
+- spec/lib/redis_open3/enum_spec.rb
+- spec/lib/redis_open3_spec.rb
+- spec/spec_helper.rb