pipes 0.1.0

data/spec/pipes/resque_hooks_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+require 'pipes/resque_hooks'
+
+describe Pipes::ResqueHooks do
+  class ResqueWorker
+    extend Pipes::ResqueHooks
+  end
+
+  describe '.after_perform_pipes' do
+    it 'lets the Redis store know it is finished working' do
+      Pipes::Store.should_receive(:done)
+      ResqueWorker.after_perform_pipes
+    end
+  end
+
+  describe '.on_failure_pipes' do
+    it 'clears the current job, forfeiting any remaining stages' do
+      Pipes::Store.should_receive(:done)
+      ResqueWorker.on_failure_pipes(Exception.new)
+    end
+  end
+end

data/spec/pipes/runner_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+require 'mock_jobs'
+
+describe Pipes::Runner do
+
+  describe '.run' do
+
+    let(:jobs) { [Writers::ContentWriter, Writers::AnotherContentWriter, Publishers::Publisher] }
+    let(:arg)  { 'en-US' }
+
+    before do
+      Pipes.configure do |config|
+        config.stages do
+          content_writers [{Writers::ContentWriter => :publishers}]
+          publishers      [Publishers::Publisher]
+        end
+      end
+    end
+
+    context 'with dependency resolution turned off' do
+
+      let(:options) { {resolve: false} }
+
+      it 'adds a pipe for all the jobs and runs it, filtering out jobs that are not configured' do
+        pipe = [
+          {name: :content_writers, jobs: [{class: Writers::ContentWriter, args: [arg]}]},
+          {name: :publishers,      jobs: [{class: Publishers::Publisher,  args: [arg]}]}
+        ]
+
+        Pipes::Store.should_receive(:add_pipe).with(pipe, options)
+        Pipes::Runner.run(jobs, arg, options)
+      end
+
+      it 'can be turned off in the config' do
+        original = Pipes.resolve
+        Pipes.resolve = false
+
+        Pipes::Store.should_receive(:add_pipe).with([{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: []}]}], {})
+        Pipes::Runner.run(Writers::ContentWriter)
+
+        Pipes.resolve = original
+      end
+
+      it 'accepts an array of strings of jobs' do
+        Pipes::Store.should_receive(:add_pipe).with([{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: []}]}], options)
+        Pipes::Runner.run(['Writers::ContentWriter'], options)
+      end
+
+      it 'accepts an array of symbols referring to stages' do
+        pipe = [
+          {name: :content_writers, jobs: [{class: Writers::ContentWriter, args: []}]},
+          {name: :publishers,      jobs: [{class: Publishers::Publisher,  args: []}]}
+        ]
+
+        Pipes::Store.should_receive(:add_pipe).with(pipe, options)
+        Pipes::Runner.run([:content_writers, :publishers], options)
+      end
+
+      it 'accepts a singular class' do
+        Pipes::Store.should_receive(:add_pipe).with([{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: []}]}], options)
+        Pipes::Runner.run(Writers::ContentWriter, options)
+      end
+
+      it 'accepts a singular string referring to a class' do
+        Pipes::Store.should_receive(:add_pipe).with([{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: []}]}], options)
+        Pipes::Runner.run('Writers::ContentWriter', options)
+      end
+
+      it 'accepts a symbol referring to a stage' do
+        Pipes::Store.should_receive(:add_pipe).with([{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: []}]}], options)
+        Pipes::Runner.run(:content_writers, options)
+      end
+
+    end
+
+    context 'with dependency resolution turned on' do
+
+      let(:options) { {resolve: true} }
+      let(:pipe) {
+        [
+          {name: :content_writers, jobs: [{class: Writers::ContentWriter, args: []}]},
+          {name: :publishers,      jobs: [{class: Publishers::Publisher,  args: []}]}
+        ]
+      }
+
+      it 'includes all dependencies of the provided job' do
+        Pipes::Store.should_receive(:add_pipe).with(pipe, options)
+        Pipes::Runner.run(Writers::ContentWriter, options)
+      end
+
+      it 'is the default' do
+        Pipes::Store.should_receive(:add_pipe).with(pipe, {})
+        Pipes::Runner.run(Writers::ContentWriter)
+      end
+
+      it 'can overwrite the value defined in the config' do
+        original = Pipes.resolve
+        Pipes.resolve = false
+
+        Pipes::Store.should_receive(:add_pipe).with(pipe, options)
+        Pipes::Runner.run(Writers::ContentWriter, options)
+
+        Pipes.resolve = original
+      end
+
+    end
+
+  end
+
+end

data/spec/pipes/stage_parser_spec.rb

@@ -0,0 +1,169 @@
+require 'spec_helper'
+require 'mock_jobs'
+
+describe Pipes::StageParser do
+  subject { Pipes::StageParser }
+
+  describe '#stage_names' do
+    it 'returns the name of the stages, in the same order' do
+      stages = {
+                 writers:    [ Writers::ContentWriter ],
+                 publishers: [ Publishers::Publisher ]
+               }
+
+      subject.new(stages).stage_names.should == [:writers, :publishers]
+    end
+  end
+
+  describe '#jobs_in_stage' do
+    it 'returns all jobs for a given stage, without dependents' do
+      stages = {
+                 writers:    [ { Writers::ContentWriter => Publishers::Publisher } ],
+                 publishers: [ Publishers::Publisher ]
+               }
+
+      subject.new(stages).jobs_in_stage(:writers).should == [Writers::ContentWriter]
+    end
+  end
+
+  describe '#dependents_for' do
+    let(:stages) {
+      {
+        writers:    [ { Writers::ContentWriter => :publishers } ],
+        publishers: [ Publishers::Publisher => Emailers::Email ],
+        emailers:   [ Emailers::Email ]
+      }
+    }
+
+    it 'returns an empty array for jobs that do not exist' do
+      subject.new(stages).dependents_for(Writers::AnotherContentWriter).should == []
+    end
+
+    it 'returns an array containing the recursive dependents' do
+      expected = [Publishers::Publisher, Emailers::Email]
+
+      subject.new(stages).dependents_for(Writers::ContentWriter).should == expected
+    end
+  end
+
+  describe '#stages_with_resolved_dependencies' do
+    context 'when the stage contains only job classes' do
+      let(:stages) {
+        {
+          writers:    [ Writers::ContentWriter ],
+          publishers: [ Publishers::Publisher ]
+        }
+      }
+
+      it 'returns a set of nested arrays, keeping the defined order, each representing a stage and its jobs with empty dependents' do
+        expected = {
+                     writers:    [ { Writers::ContentWriter => [] } ],
+                     publishers: [ { Publishers::Publisher => [] } ]
+                   }
+
+        subject.new(stages).stages_with_resolved_dependencies.should == expected
+      end
+    end
+
+    context 'when the stage containing a hash defining a dependent class' do
+      let(:stages) {
+        { writers: [ { Writers::ContentWriter => Publishers::Publisher } ] }
+      }
+
+      it 'adds the dependent class to the list' do
+        expected = {
+                     writers: [ { Writers::ContentWriter => [Publishers::Publisher] } ]
+                   }
+
+        subject.new(stages).stages_with_resolved_dependencies.should == expected
+      end
+    end
+
+    context 'with the stage containing a hash defining a dependent stage' do
+      let(:stages) {
+        {
+          writers:    [ { Writers::ContentWriter => :publishers } ],
+          publishers: [ Publishers::Publisher ]
+        }
+      }
+
+      it 'adds all classes within the dependent stage to the list' do
+        expected = {
+                     writers:    [ { Writers::ContentWriter => [Publishers::Publisher] } ],
+                     publishers: [ { Publishers::Publisher  => [] } ]
+                   }
+
+        subject.new(stages).stages_with_resolved_dependencies.should == expected
+      end
+    end
+
+    context 'with the stage containing a hash defining an array of dependent classes' do
+      let(:stages) {
+        {
+          writers:    [ { Writers::ContentWriter => [Publishers::Publisher] } ],
+          publishers: [ Publishers::Publisher ]
+        }
+      }
+
+      it 'adds all classes within the array to the list' do
+        expected = {
+                     writers:    [ { Writers::ContentWriter => [Publishers::Publisher] } ],
+                     publishers: [ { Publishers::Publisher  => [] } ]
+                   }
+
+        subject.new(stages).stages_with_resolved_dependencies.should == expected
+      end
+    end
+
+    context 'with the stage containing a hash defining an array of dependent stages' do
+      let(:stages) {
+        {
+          writers:    [ { Writers::ContentWriter => [:publishers] } ],
+          publishers: [ Publishers::Publisher ]
+        }
+      }
+
+      it 'adds all classes within the dependent stages to the list' do
+        expected = {
+                     writers:    [ { Writers::ContentWriter => [Publishers::Publisher] } ],
+                     publishers: [ { Publishers::Publisher  => [] } ]
+                   }
+
+        subject.new(stages).stages_with_resolved_dependencies.should == expected
+      end
+    end
+
+    context 'with a comlex configuration, intermixing dependent types' do
+      let (:stages) {
+        {
+          writers:    [ { Writers::ContentWriter        => [:publishers, Uploaders::Rsync] },
+                        { Writers::AnotherContentWriter => [Emailers::Email] }
+                      ],
+          publishers: [ { Publishers::Publisher => :emailers } ],
+          messengers: [ { Messengers::SMS => :uploaders } ],
+          uploaders:  [ { Uploaders::Rsync => Notifiers::Twitter } ],
+          emailers:   [ Emailers::Email, Emailers::AnotherEmail ],
+          notifiers:  [ Notifiers::Twitter ]
+        }
+      }
+
+      it 'resolves all dependencies' do
+        expected = {
+                     writers:    [ { Writers::ContentWriter        => [Publishers::Publisher, Emailers::Email, Emailers::AnotherEmail,
+                                                                       Uploaders::Rsync, Notifiers::Twitter] },
+                                   { Writers::AnotherContentWriter => [Emailers::Email] }
+                                 ],
+                     publishers: [ { Publishers::Publisher  => [Emailers::Email, Emailers::AnotherEmail] } ],
+                     messengers: [ { Messengers::SMS        => [Uploaders::Rsync, Notifiers::Twitter] } ],
+                     uploaders:  [ { Uploaders::Rsync       => [Notifiers::Twitter] } ],
+                     emailers:   [ { Emailers::Email        => [] },
+                                   { Emailers::AnotherEmail => [] }
+                                 ],
+                     notifiers:  [ { Notifiers::Twitter     => [] } ]
+                   }
+
+        subject.new(stages).stages_with_resolved_dependencies.should == expected
+      end
+    end
+  end
+end

data/spec/pipes/store_spec.rb

@@ -0,0 +1,181 @@
+require 'spec_helper'
+require 'mock_jobs'
+
+describe Pipes::Store do
+
+  def mock_stages
+    Pipes::Store.stub!(:stages) { stages }
+    stages
+  end
+
+  def mock_pending_jobs(stage)
+    list = send("pending_jobs_#{stage}".to_sym)
+    Pipes::Store.stub!(:pending_jobs).with(stage) { list }
+    list
+  end
+
+  let(:stages) { [] }
+  let(:pending_jobs_content_writers) { [] }
+  let(:pending_jobs_publishers)      { [] }
+
+  let(:job_options) { 'en-US' }
+  let(:pipe)        { [{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: [job_options]}]},
+                       {name: :publishers,      jobs: [{class: Publishers::Publisher,  args: [job_options]}]}] }
+
+  before do
+    Pipes::Store.clear_all
+  end
+
+  describe '.add_pipe' do
+    before do
+      @writers = mock_pending_jobs(:content_writers)
+      @publishers = mock_pending_jobs(:publishers)
+      Pipes::Store.stub!(:next_stage)
+    end
+
+    it 'adds the job to Redis and fires off the next available stage' do
+      Pipes::Store.should_receive(:next_stage)
+      Pipes::Store.add_pipe(pipe)
+
+      @writers.should == [{class: Writers::ContentWriter, args: [job_options]}]
+      @publishers.should == [{class: Publishers::Publisher,  args: [job_options]}]
+    end
+
+    it 'does not add duplicate jobs to stages' do
+      another_pipe = [{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: [job_options]},
+                                                      {class: Writers::AnotherContentWriter, args: [job_options]}]},
+                      {name: :publishers,      jobs: [{class: Publishers::Publisher,  args: [job_options]}]}]
+
+      Pipes::Store.add_pipe(pipe)
+      Pipes::Store.add_pipe(another_pipe)
+
+      @writers.should == [{class: Writers::ContentWriter, args: [job_options]}, {class: Writers::AnotherContentWriter, args: [job_options]}]
+      @publishers.should == [{class: Publishers::Publisher,  args: [job_options]}]
+    end
+
+    context 'with allow_duplicates set' do
+      it 'does not add jobs that have the same class already in the queue' do
+        mock_stages
+
+        Pipes::Store.add_pipe(pipe)
+
+        @writers.should_receive(:<<)
+        @publishers.should_not_receive(:<<)
+
+        another_pipe = [{name: :content_writers, jobs: [{class: Writers::ContentWriter, args: ['some arg']}]},
+                        {name: :publishers,      jobs: [{class: Publishers::Publisher,  args: ['some arg']}]}]
+
+        Pipes::Store.add_pipe(another_pipe, {allow_duplicates: [:content_writers]})
+      end
+    end
+
+  end
+
+  describe '.next_stage' do
+    context 'with jobs left in the queue' do
+      it 'pulls the next stage with pending jobs off the queue and runs it' do
+        Pipes::Store.stub!(:stages) { [:content_writers, :publishers] }
+
+        writers = mock_pending_jobs(:content_writers) << {class: Writers::ContentWriter, args: []}
+        publishers = mock_pending_jobs(:publishers) << {class: Publishers::Publisher,  args: []}
+
+        Pipes::Store.should_receive(:run_stage).with([{class: Writers::ContentWriter, args: []}])
+        Pipes::Store.should_not_receive(:run_stage).with([{class: Publishers::Publisher, args: []}])
+
+        Pipes::Store.next_stage
+      end
+
+      it 'runs stages in the order determined by the stages list' do
+        Pipes::Store.stub!(:stages) { [:publishers, :content_writers] }
+
+        writers = mock_pending_jobs(:content_writers) << {class: Writers::ContentWriter, args: []}
+        publishers = mock_pending_jobs(:publishers) << {class: Publishers::Publisher,  args: []}
+
+        Pipes::Store.should_not_receive(:run_stage).with([{class: Writers::ContentWriter, args: []}])
+        Pipes::Store.should_receive(:run_stage).with([{class: Publishers::Publisher, args: []}])
+
+        Pipes::Store.next_stage
+      end
+    end
+
+    context 'without jobs left in the queue' do
+      it 'fires off the next pipe' do
+        mock_stages
+        Pipes::Store.should_not_receive(:run_stage)
+        Pipes::Store.next_stage
+      end
+    end
+  end
+
+  describe '.run_stage' do
+    it 'sets the remaining counter and enqueues the jobs' do
+      stage = [{class: Writers::ContentWriter, args: [job_options]},
+               {class: Writers::AnotherContentWriter, args: [job_options]}]
+
+      remaining = mock('Remaining Jobs')
+      Pipes::Store.stub!(:remaining_jobs) { remaining }
+
+      remaining.should_receive(:clear)
+      remaining.should_receive(:incr).with(2)
+
+      Resque.should_receive(:enqueue).with(Writers::ContentWriter, 'en-US')
+      Resque.should_receive(:enqueue).with(Writers::AnotherContentWriter, 'en-US')
+
+      Pipes::Store.run_stage(stage)
+    end
+  end
+
+  describe '.done' do
+    context 'as the last job in a stage' do
+      it 'fires off the next stage' do
+        remaining = mock('Remaining Jobs', {decrement: 0}) { 1 }
+        Pipes::Store.stub!(:remaining_jobs) { remaining }
+
+        Pipes::Store.should_receive(:next_stage)
+        Pipes::Store.done
+      end
+    end
+
+    context 'with jobs still running' do
+      it 'decrements the remaining jobs but does not run the next stage' do
+        remaining = mock('Remaining Jobs') { 2 }
+        Pipes::Store.stub!(:remaining_jobs) { remaining }
+
+        remaining.should_receive(:decrement) { 1 }
+        Pipes::Store.should_not_receive(:next_stage)
+        Pipes::Store.done
+      end
+    end
+  end
+
+  describe '.clear' do
+    it 'clears out the jobs for the given stage' do
+      writers = mock_pending_jobs(:content_writers)
+      writers << {class: Writers::ContentWriter, args: [job_options]}
+
+      Pipes::Store.clear(:content_writers)
+      writers.should == []
+    end
+  end
+
+  describe '.clear_all' do
+    it 'deletes Redis keys for all stages, even those not currently being used' do
+      Redis::List.new('pipes:stages:some_stage_used_for_testing') << 'stategy'
+      Redis::List.new('pipes:stages:any_stage_test') << 'stategy'
+
+      Pipes::Store.clear_all
+
+      Redis::List.new('pipes:stages:some_stage_used_for_testing').should == []
+      Redis::List.new('pipes:stages:any_stage_test').should == []
+    end
+
+    it 'resets the remaining jobs counter' do
+      remaining = mock('Remaining Jobs')
+      Pipes::Store.stub!(:remaining_jobs) { remaining }
+
+      remaining.should_receive(:clear)
+
+      Pipes::Store.clear_all
+    end
+  end
+end