execache 0.1.0

data/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+*.gem
+.bundle
+Gemfile.lock
+pkg
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2010
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of 
+this software and associated documentation files (the "Software"), to deal in 
+the Software without restriction, including without limitation the rights to 
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 
+the Software, and to permit persons to whom the Software is furnished to do so, 
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all 
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+Execache
+========
+
+Run commands in parallel and cache the output. Redis queues jobs and stores the result.
+
+Requirements
+------------
+
+<pre>
+gem install execache
+</pre>
+
+How Your Binaries Should Behave
+-------------------------------
+
+Execache assumes that the script or binary you are executing has multiple results and sometimes multiple groups of results.
+
+Example output:
+
+    $ bin/some/binary preliminary_arg arg1a arg1b arg2a arg2b
+    $ arg1_result_1
+    $ arg1_result_2
+    $ [END]
+    $ arg2_result_1
+    $ arg2_result_2
+
+Your binary may take zero or more preliminary arguments (e.g. `preliminary_arg`), followed by argument "groups" that dictate output (e.g. `arg1a arg1b`).
+
+Configure
+---------
+
+Given the above example, our `execache.yml` looks like this:
+
+    redis: localhost:6379/0
+    some_binary:
+      command: '/bin/some/binary'
+      separators:
+        result: "\n"
+        group: "[END]"
+
+Start the Server
+----------------
+
+    $ execache /path/to/execache.yml
+
+Execute Commands
+----------------
+
+    require 'rubygems'
+    require 'execache'
+
+    client = Execache::Client.new("localhost:6379/0")
+    
+    results = client.exec(
+      :some_binary => {
+        :args => 'preliminary_arg',
+        :groups => [
+          {
+            :args => 'arg1a arg1b',
+            :ttl => 60
+          },
+          {
+            :args => 'arg2a arg2b',
+            :ttl => 60
+          }
+        ]
+      }
+    )
+
+    results == {
+      :some_binary => [
+        [ 'arg1_result_1', 'arg1_result_2' ],
+        [ 'arg2_result_1', 'arg2_result_2' ]
+      ]
+    }

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/bin/execache

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/execache", __FILE__)

data/execache.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+root = File.expand_path('../', __FILE__)
+lib = "#{root}/lib"
+
+$:.unshift lib unless $:.include?(lib)
+
+Gem::Specification.new do |s|
+  s.name        = "execache"
+  s.version     = '0.1.0'
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = [ "Winton Welsh" ]
+  s.email       = [ "mail@wintoni.us" ]
+  s.homepage    = "http://github.com/winton/execache"
+  s.summary     = %q{Run commands in parallel and cache the output, controlled by Redis}
+  s.description = %q{Run commands in parallel and cache the output. Redis queues jobs and stores the result.}
+
+  s.executables = `cd #{root} && git ls-files bin/*`.split("\n").collect { |f| File.basename(f) }
+  s.files = `cd #{root} && git ls-files`.split("\n")
+  s.require_paths = %w(lib)
+  s.test_files = `cd #{root} && git ls-files -- {features,test,spec}/*`.split("\n")
+
+  s.add_development_dependency "rspec", "~> 1.0"
+
+  s.add_dependency "redis", "~> 2.2.2"
+  s.add_dependency "yajl-ruby", "~> 1.0.0"
+end

data/lib/execache.rb

@@ -0,0 +1,117 @@
+require "digest/sha1"
+require "timeout"
+require "yaml"
+
+gem "yajl-ruby", "~> 1.0.0"
+require "yajl"
+
+gem "redis", "~> 2.2.2"
+require "redis"
+
+$:.unshift File.dirname(__FILE__)
+
+require 'execache/client'
+
+class Execache
+
+  def initialize(yaml)
+    options = YAML.load(File.read(yaml))
+
+    puts "\nStarting execache server (redis @ #{options['redis']})..."
+
+    redis = Redis.connect(:url => "redis://#{options['redis']}")
+    retries = 0
+    
+    begin
+      while true
+        request = redis.lpop('execache:request')
+        if request
+          Thread.new do
+            request = Yajl::Parser.parse(request)
+            channel = request.delete('channel')
+            commands = []
+
+            request.each do |cmd_type, cmd_options|
+              # Command with preliminary args
+              command = [
+                options[cmd_type]['command'],
+                cmd_options['args']
+              ]
+              
+              # Fill results with caches if present
+              cmd_options['groups'].each do |group|
+                cache_key = Digest::SHA1.hexdigest(
+                  "#{cmd_options['args']} #{group['args']}"
+                )
+                group['cache_key'] = cache_key = "execache:cache:#{cache_key}"
+                cache = redis.get(cache_key)
+                
+                if cache
+                  group['result'] = Yajl::Parser.parse(cache)
+                else
+                  command << group['args']
+                  nil
+                end
+              end
+              
+              # Add command to be executed if not all args are cached
+              if command.length > 2
+                cmd_options['cmd'] = command.join(' ')
+              end
+            end
+
+            # Build response
+            response = request.inject({}) do |hash, (cmd_type, cmd_options)|
+              hash[cmd_type] = []
+
+              if cmd_options['cmd']
+                separators = options[cmd_type]['separators'] || {}
+                separators['group'] ||= "[END]"
+                separators['result'] ||= "\n"
+                output = `#{cmd_options['cmd']}`
+                output = output.split(separators['group'] + separators['result'])
+                output = output.collect { |r| r.split(separators['result']) }
+              end
+
+              cmd_options['groups'].each do |group|
+                if group['result']
+                  hash[cmd_type] << group['result']
+                else
+                  hash[cmd_type] << output.shift
+                  redis.set(
+                    group['cache_key'],
+                    Yajl::Encoder.encode(hash[cmd_type].last)
+                  )
+                  if group['ttl']
+                    redis.expire(group['cache_key'], group['ttl'])
+                  end
+                end
+              end
+
+              hash
+            end
+            
+            redis.publish(
+              "execache:response:#{channel}",
+              Yajl::Encoder.encode(response)
+            )
+          end
+        end
+        sleep(1.0 / 1000.0)
+      end
+    rescue Interrupt
+      shut_down
+    rescue Exception => e
+      puts "\nError: #{e.message}"
+      puts "\t#{e.backtrace.join("\n\t")}"
+      retries += 1
+      shut_down if retries >= 10
+      retry
+    end
+  end
+
+  def shut_down
+    puts "\nShutting down execache server..."
+    exit
+  end
+end

data/lib/execache/client.rb

@@ -0,0 +1,31 @@
+class Execache
+  class Client
+
+    attr_reader :redis_1, :redis_2
+
+    def initialize(redis_url)
+      @redis_1 = Redis.connect(:url => "redis://#{redis_url}")
+      @redis_2 = Redis.connect(:url => "redis://#{redis_url}")
+    end
+
+    def exec(options)
+      options[:channel] = Digest::SHA1.hexdigest("#{rand}")
+      response = nil
+
+      Timeout.timeout(60) do
+        @redis_1.subscribe("execache:response:#{options[:channel]}") do |on|
+          on.subscribe do |channel, subscriptions|
+            @redis_2.rpush "execache:request", Yajl::Encoder.encode(options)
+          end
+
+          on.message do |channel, message|
+            response = Yajl::Parser.parse(message)
+            @redis_1.unsubscribe
+          end
+        end
+      end
+
+      response
+    end
+  end
+end

data/spec/execache_spec.rb

@@ -0,0 +1,123 @@
+require 'spec_helper'
+
+describe Execache do
+
+  def client_exec
+    @client.exec(
+      :some_binary => {
+        :args => 'preliminary_arg',
+        :groups => [
+          {
+            :args => 'arg1a arg1b',
+            :ttl => 60
+          },
+          {
+            :args => 'arg2a arg2b',
+            :ttl => 60
+          }
+        ]
+      }
+    )
+  end
+
+  before(:all) do
+    @thread = Thread.new do
+      Execache.new("#{$root}/spec/fixtures/execache.yml")
+    end
+    @client = Execache::Client.new("localhost:6379/0")
+    @client.redis_1.keys("execache:cache:*").each do |key|
+      @client.redis_1.del(key)
+    end
+  end
+
+  after(:all) do
+    @thread.kill
+  end
+
+  it "should return proper results" do
+    client_exec.should == {
+      "some_binary" => [
+        ["arg1_result_1", "arg1_result_2"],
+        ["arg2_result_1", "arg2_result_2"]
+      ]
+    }
+  end
+
+  it "should write to cache" do
+    keys = @client.redis_1.keys("execache:cache:*")
+    keys.length.should == 2
+  end
+
+  it "should read from cache" do
+    keys = @client.redis_1.keys("execache:cache:*")
+    @client.redis_1.set(keys[0], "[\"cached!\"]")
+    @client.redis_1.set(keys[1], "[\"cached!\"]")
+    client_exec.should == {
+      "some_binary" => [
+        ["cached!"],
+        ["cached!"]
+      ]
+    }
+  end
+
+  it "should read from cache for individual groups" do
+    @client.exec(
+      :some_binary => {
+        :args => 'preliminary_arg',
+        :groups => [
+          {
+            :args => 'arg2a arg2b',
+            :ttl => 60
+          }
+        ]
+      }
+    ).should == {
+      "some_binary" => [
+        ["cached!"]
+      ]
+    }
+
+    @client.exec(
+      :some_binary => {
+        :args => 'preliminary_arg',
+        :groups => [
+          {
+            :args => 'arg1a arg1b',
+            :ttl => 60
+          }
+        ]
+      }
+    ).should == {
+      "some_binary" => [
+        ["cached!"]
+      ]
+    }
+  end
+
+  it "should not read cache if preliminary arg changes" do
+    @client.exec(
+      :some_binary => {
+        :args => 'preliminary_arg2',
+        :groups => [
+          {
+            :args => 'arg2a arg2b',
+            :ttl => 60
+          }
+        ]
+      }
+    ).should == {
+      "some_binary" => [
+        ["arg1_result_1", "arg1_result_2"]
+      ]
+    }
+  end
+
+  it "should still read from original cache" do
+    client_exec.should == {
+      "some_binary" => [
+        ["cached!"],
+        ["cached!"]
+      ]
+    }
+  end
+end

data/spec/fixtures/execache.yml

@@ -0,0 +1,6 @@
+redis: localhost:6379/0
+some_binary:
+  command: 'ruby spec/fixtures/fixture.rb'
+  separators:
+    result: "\n"
+    group: "[END]"

data/spec/fixtures/fixture.rb

@@ -0,0 +1,5 @@
+puts "arg1_result_1
+arg1_result_2
+[END]
+arg2_result_1
+arg2_result_2"

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require "pp"
+require "bundler"
+
+Bundler.require(:default)
+Bundler.require(:development)
+
+$root = File.expand_path('../../', __FILE__)
+
+require "#{$root}/lib/execache"

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: execache
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Winton Welsh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-11 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70093750276540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: *70093750276540
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &70093750276060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70093750276060
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: &70093750275600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70093750275600
+description: Run commands in parallel and cache the output. Redis queues jobs and
+  stores the result.
+email:
+- mail@wintoni.us
+executables:
+- execache
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/execache
+- execache.gemspec
+- lib/execache.rb
+- lib/execache/client.rb
+- spec/execache_spec.rb
+- spec/fixtures/execache.yml
+- spec/fixtures/fixture.rb
+- spec/spec_helper.rb
+homepage: http://github.com/winton/execache
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Run commands in parallel and cache the output, controlled by Redis
+test_files:
+- spec/execache_spec.rb
+- spec/fixtures/execache.yml
+- spec/fixtures/fixture.rb
+- spec/spec_helper.rb