rector 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - jruby-19mode
+  - rbx-19mode
+script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in rector.gemspec
+gemspec

data/README.md

@@ -0,0 +1,74 @@
+# Rector
+
+[![Build Status](https://secure.travis-ci.org/alindeman/rector.png)](http://travis-ci.org/alindeman/rector)
+
+** RECTOR IS CURRENTLY VAPORWARE; THIS README IS SIMPLY MY THOUGHTS ON
+HOW IT MIGHT WORK **
+
+Rector allows coordination of a number of jobs spawned with a mechanism
+like Resque (though any job manager will do). If you are able to parallelize
+the processing of a task, yet all these tasks are generating metrics,
+statistics, or other data that need to be combined, Rector might be for you.
+
+## Requirements
+
+* Ruby >= 1.9.2 (or 1.9 mode of JRuby or Rubinius)
+
+## Configuration
+
+Rector currently supports Redis as a backend for job coordination and
+data storage.
+
+### Redis Server
+
+```ruby
+Rector.configure do |c|
+  c.redis = Redis.new(:host => "10.0.1.1", :port => 6380)
+end
+```
+
+## Job Creation (Master)
+
+Rector requires that some process be designated as the "master" process.
+This is usually the process that is also responsible for spawning the
+worker jobs.
+
+```ruby
+job = Rector::Job.new
+
+# e.g., processing files in parallel
+files.each do |file|
+  worker = job.workers.create
+
+  # e.g., using Resque for job management; Rector doesn't really care
+  Resque.enqueue(WordCounterJob, worker.id, file)
+end
+
+# wait for all the workers to complete
+job.join
+
+# get aggregated data from all the jobs
+job.data.each do |word, count|
+  puts "#{word} was seen #{count} times across all files"
+end
+
+job.cleanup
+```
+
+## Job Processing (Workers)
+
+```ruby
+class ProcessFileJob
+  def self.perform(worker_id, file)
+    worker = Rector::Worker.new(worker_id)
+
+    words = File.read(file).split(/\W/)
+    words.reject(&:blank?).each do |word|
+      worker.data[word] ||= 0 
+      worker.data[word]  += 1
+    end
+
+    worker.finish
+  end
+end
+```

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/rector.rb

@@ -0,0 +1,27 @@
+require "redis"
+require "redis-namespace"
+
+require_relative "rector/configuration"
+require_relative "rector/worker"
+require_relative "rector/job"
+require_relative "rector/backends"
+
+module Rector
+  class << self
+    def configuration
+      @configuration ||= Rector::Configuration.new
+    end
+
+    def reset
+      @configuration = nil
+    end
+
+    def configure
+      yield configuration
+    end
+
+    def backend_for(job_id)
+      Rector::Backends::Redis.new(job_id)
+    end
+  end
+end

data/lib/rector/backends.rb

@@ -0,0 +1 @@
+require_relative "backends/redis"

data/lib/rector/backends/redis.rb

@@ -0,0 +1,74 @@
+module Rector
+  module Backends
+    class Redis
+      KEY_LIST_SET    = "__keys__"
+      WORKER_LIST_SET = "__workers__"
+
+      attr_reader :job_id
+
+      def initialize(job_id)
+        @job_id = job_id
+      end
+
+      def update_job_data_from_hash(hsh)
+        redis.multi do
+          redis.sadd(KEY_LIST_SET, *hsh.keys)
+
+          hsh.each do |key, val|
+            case val
+            when Numeric
+              redis.incrby(key, val)
+            when Set
+              redis.sadd(key, *val)
+            when Enumerable
+              redis.rpush(key, *val)
+            end
+          end
+        end
+      end
+
+      def read_job_data_to_hash
+        Hash[keys.map { |k| [k, read(k)] }]
+      end
+
+      def add_worker(worker_id)
+        redis.sadd(WORKER_LIST_SET, worker_id)
+      end
+
+      def finish_worker(worker_id)
+        redis.srem(WORKER_LIST_SET, worker_id)
+      end
+
+      def workers_working?
+        redis.scard(WORKER_LIST_SET).to_i > 0
+      end
+
+      def cleanup
+        redis.del(*keys)
+        redis.del(KEY_LIST_SET, WORKER_LIST_SET)
+      end
+
+      private
+
+      def redis
+        @redis ||=
+          ::Redis::Namespace.new(@job_id, redis: Rector.configuration.redis)
+      end
+
+      def keys
+        redis.smembers(KEY_LIST_SET)
+      end
+
+      def read(key)
+        case redis.type(key)
+        when "string"
+          redis.get(key).to_i
+        when "set"
+          Set.new(redis.smembers(key))
+        when "list"
+          redis.lrange(key, 0, -1)
+        end
+      end
+    end
+  end
+end

data/lib/rector/configuration.rb

@@ -0,0 +1,5 @@
+require "ostruct"
+
+module Rector
+  Configuration = ::OpenStruct
+end

data/lib/rector/job.rb

@@ -0,0 +1,54 @@
+require "delegate"
+require "securerandom"
+
+module Rector
+  class Job
+    class WorkerCollection < SimpleDelegator
+      def initialize(job)
+        @job = job
+
+        # Wraps an array
+        super(Array.new)
+      end
+
+      def create
+        Rector::Worker.new(@job.allocate_worker_id).tap do |worker|
+          self << worker
+        end
+      end
+    end
+
+    attr_reader :workers
+
+    def initialize
+      @workers = WorkerCollection.new(self)
+      @backend = Rector.backend_for(id)
+    end
+
+    def id
+      # TODO: Obviously there's a small chance of jobs overlapping here
+      # Can do something more reliable for ID generation?
+      @id ||= SecureRandom.hex(10)
+    end
+
+    def allocate_worker_id
+      # TODO: Obviously there's a small chance of jobs overlapping here
+      # Can do something more reliable for ID generation?
+      "#{id}:#{SecureRandom.hex(8)}"
+    end
+
+    def join
+      while @backend.workers_working?
+        sleep 5
+      end
+    end
+
+    def data
+      @data ||= @backend.read_job_data_to_hash
+    end
+
+    def cleanup
+      @backend.cleanup
+    end
+  end
+end

data/lib/rector/worker.rb

@@ -0,0 +1,22 @@
+module Rector
+  class Worker
+    attr_reader :id, :data
+
+    def initialize(id)
+      @id      = id
+      @data    = Hash.new
+
+      @backend = Rector.backend_for(job_id)
+      @backend.add_worker(@id)
+    end
+
+    def job_id
+      @id.split(":").first
+    end
+
+    def finish
+      @backend.update_job_data_from_hash(@data)
+      @backend.finish_worker(@id)
+    end
+  end
+end

data/rector.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "rector"
+  s.version     = "0.0.1"
+  s.authors     = ["Andy Lindeman"]
+  s.email       = ["alindeman@gmail.com"]
+  s.homepage    = "https://github.com/alindeman/rector"
+  s.summary     = %q{Rector coordinates parallelized jobs that generate metrics or other data together}
+  s.description = <<-EOF
+    Rector allows coordination of a number of jobs spawned with a mechanism
+    like Resque (though any job manager will do). If you are able to parallelize
+    the processing of a task, yet all these tasks are generating metrics,
+    statistics, or other data that need to be combined, Rector might be for you.
+  EOF
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "redis"
+  s.add_dependency "redis-namespace"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec", ">=2.8.0"
+  s.add_development_dependency "mocha", ">=0.10.0"
+end

data/spec/lib/backends/redis_spec.rb

@@ -0,0 +1,108 @@
+require "spec_helper"
+
+describe Rector::Backends::Redis do
+  let(:redis) { stub_everything("redis") }
+
+  before do
+    def redis.multi
+      yield
+    end
+
+    Rector.configure do |c|
+      c.redis = redis
+    end
+  end
+
+  let(:job_id) { "abc123" }
+  subject         { described_class.new(job_id) }
+
+  describe "writing" do
+    it "stores a list of keys" do
+      hsh = {
+        "foo" => 1,
+        "bar" => 2
+      }
+
+      redis.expects(:sadd).with("#{job_id}:__keys__", "foo", "bar")
+      subject.update_job_data_from_hash(hsh)
+    end
+
+    it "stores integers" do
+      hsh = { "foo" => 1 }
+
+      redis.expects(:incrby).with("#{job_id}:foo", 1)
+      subject.update_job_data_from_hash(hsh)
+    end
+
+    it "stores lists" do
+      hsh = { "foo" => ["a", "b", "c"] }
+
+      redis.expects(:rpush).with("#{job_id}:foo", "a", "b", "c")
+      subject.update_job_data_from_hash(hsh)
+    end
+
+    it "stores sets" do
+      hsh = { "foo" => Set.new(["a", "b", "c"]) }
+
+      redis.expects(:sadd).with("#{job_id}:foo", "a", "b", "c")
+      subject.update_job_data_from_hash(hsh)
+    end
+  end
+
+  describe "reading" do
+    it "reads integers" do
+      redis.stubs(:smembers).with("#{job_id}:__keys__").returns(["foo"])
+
+      redis.stubs(:type).with("#{job_id}:foo").returns("string")
+      redis.stubs(:get).with("#{job_id}:foo").returns("5")
+
+      subject.read_job_data_to_hash.should == { "foo" => 5 }
+    end
+
+    it "reads lists" do
+      redis.stubs(:smembers).with("#{job_id}:__keys__").returns(["foo"])
+
+      redis.stubs(:type).with("#{job_id}:foo").returns("list")
+      redis.stubs(:lrange).with("#{job_id}:foo", 0, -1).returns(["bar"])
+
+      subject.read_job_data_to_hash.should == { "foo" => ["bar"] }
+    end
+
+    it "reads sets" do
+      redis.stubs(:smembers).with("#{job_id}:__keys__").returns(["foo"])
+
+      redis.stubs(:type).with("#{job_id}:foo").returns("set")
+      redis.stubs(:smembers).with("#{job_id}:foo").returns(["bar"])
+
+      subject.read_job_data_to_hash.should == { "foo" => Set.new(["bar"]) }
+    end
+  end
+
+  describe "workers" do
+    it "adds a worker to a set" do
+      redis.expects(:sadd).with("#{job_id}:__workers__", "1234:5678")
+      subject.add_worker("1234:5678")
+    end
+
+    it "removes a worker from the set when it is finished" do
+      redis.expects(:srem).with("#{job_id}:__workers__", "1234:5678")
+      subject.finish_worker("1234:5678")
+    end
+
+    it "knows if workers are still working" do
+      redis.stubs(:scard).with("#{job_id}:__workers__").returns("1")
+      subject.workers_working?.should be_true
+
+      redis.stubs(:scard).with("#{job_id}:__workers__").returns("0")
+      subject.workers_working?.should be_false
+    end
+  end
+
+  it "cleans up when requests" do
+    redis.stubs(:smembers).returns(["a", "b"])
+    redis.expects(:del).with("#{job_id}:a", "#{job_id}:b")
+    redis.expects(:del).with("#{job_id}:__keys__", "#{job_id}:__workers__")
+
+    subject.cleanup
+  end
+end

data/spec/lib/configuration_spec.rb

@@ -0,0 +1,9 @@
+require "spec_helper"
+
+describe Rector::Configuration do
+  it "accepts and stores arbitrary configuration items" do
+    subject.backend = :redis
+
+    subject.backend.should == :redis
+  end
+end

data/spec/lib/job_spec.rb

@@ -0,0 +1,33 @@
+require "spec_helper"
+require "timeout"
+
+describe Rector::Job do
+  let(:backend) { stub_everything("backend") }
+  before do
+    Rector.stubs(:backend_for).returns(backend)
+  end
+
+  it "constructs workers" do
+    worker = subject.workers.create
+    worker.should be_a(Rector::Worker)
+  end
+
+  it "waits for workers to complete" do
+    backend.expects(:workers_working?).at_least_once.returns(true).then.returns(false)
+    subject.stubs(:sleep)
+
+    Timeout.timeout(2) do
+      subject.join
+    end
+  end
+
+  it "loads data from the backend" do
+    backend.stubs(:read_job_data_to_hash).returns("foo" => "bar")
+    subject.data.should == { "foo" => "bar" }
+  end
+
+  it "delegates to the backend for cleanup" do
+    backend.expects(:cleanup)
+    subject.cleanup
+  end
+end

data/spec/lib/rector_spec.rb

@@ -0,0 +1,18 @@
+require "spec_helper"
+
+describe Rector do
+  it "allows configuration by yielding a block to #configure" do
+    described_class.configure do |c|
+      c.foo = :bar
+    end
+
+    described_class.configuration.foo.should == :bar
+  end
+
+  it "creates backend objects for jobs" do
+    backend = described_class.backend_for("abc123")
+
+    backend.should be_a(Rector::Backends::Redis)
+    backend.job_id.should == "abc123"
+  end
+end

data/spec/lib/worker_spec.rb

@@ -0,0 +1,38 @@
+require "spec_helper"
+
+describe Rector::Worker do
+  let(:worker_id) { "zyx987:abc123" }
+  subject         { described_class.new(worker_id) }
+
+  let(:backend)   { stub_everything("backend") }
+  before do
+    Rector.stubs(:backend_for).returns(backend)
+  end
+
+  it "is initialized with a worker ID" do
+    subject.id.should == worker_id
+  end
+
+  it "knows its job ID" do
+    subject.job_id.should == "zyx987"
+  end
+
+  it "notifies the backend of workers being created" do
+    backend.expects(:add_worker).with(worker_id)
+    subject
+  end
+
+  describe "#finish" do
+    it "notifies the backend" do
+      backend.expects(:finish_worker).with(worker_id)
+      subject.finish
+    end
+
+    it "saves data" do
+      subject.data["foo"] = "bar"
+
+      backend.expects(:update_job_data_from_hash).with("foo" => "bar")
+      subject.finish
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require "rspec"
+require "mocha"
+
+require_relative "../lib/rector"
+
+RSpec.configure do |c|
+  c.mock_with :mocha
+
+  c.before do
+    Rector.reset
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: rector
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Andy Lindeman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &70197558869260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70197558869260
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: &70197558868820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70197558868820
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70197558868400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70197558868400
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70197558867900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :development
+  prerelease: false
+  version_requirements: *70197558867900
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: &70197558867340 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :development
+  prerelease: false
+  version_requirements: *70197558867340
+description: ! "    Rector allows coordination of a number of jobs spawned with a
+  mechanism\n    like Resque (though any job manager will do). If you are able to
+  parallelize\n    the processing of a task, yet all these tasks are generating metrics,\n
+  \   statistics, or other data that need to be combined, Rector might be for you.\n"
+email:
+- alindeman@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- lib/rector.rb
+- lib/rector/backends.rb
+- lib/rector/backends/redis.rb
+- lib/rector/configuration.rb
+- lib/rector/job.rb
+- lib/rector/worker.rb
+- rector.gemspec
+- spec/lib/backends/redis_spec.rb
+- spec/lib/configuration_spec.rb
+- spec/lib/job_spec.rb
+- spec/lib/rector_spec.rb
+- spec/lib/worker_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/alindeman/rector
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -3768330157712875762
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -3768330157712875762
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.14
+signing_key: 
+specification_version: 3
+summary: Rector coordinates parallelized jobs that generate metrics or other data
+  together
+test_files:
+- spec/lib/backends/redis_spec.rb
+- spec/lib/configuration_spec.rb
+- spec/lib/job_spec.rb
+- spec/lib/rector_spec.rb
+- spec/lib/worker_spec.rb
+- spec/spec_helper.rb