processor 1.0.0 → 2.0.0

data/lib/processor/subroutine/name.rb

@@ -0,0 +1,28 @@
+require 'delegate'
+
+module Processor
+  module Subroutine
+    class Name < ::SimpleDelegator
+      def name
+        return super if __getobj__.respond_to? :name
+
+        # underscore a class name
+       @name ||= real_object.class.name.to_s.
+          gsub(/::/, '_').
+          gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+          gsub(/([a-z\d])([A-Z])/,'\1_\2').
+          downcase
+      end
+
+      private
+      def real_object
+        object = __getobj__
+        while object.is_a? SimpleDelegator
+          object = object.__getobj__
+        end
+
+        object
+      end
+    end
+  end
+end

data/lib/processor/subroutine/recursion.rb

@@ -0,0 +1,29 @@
+require_relative 'counter'
+require 'delegate'
+
+module Processor
+  module Subroutine
+    class Recursion < ::SimpleDelegator
+      def initialize(processor)
+        # recursion depends on counter subroutine
+        processor = Counter.new(processor) unless processor.respond_to? :processed_records_count
+
+        super processor
+      end
+
+      def process(*)
+        recursion_preventer
+        super
+      end
+
+      private
+      def recursion_preventer
+        raise Exception, "Processing fall into recursion. Check logs." if processed_records_count >= max_records_to_process
+      end
+
+      def max_records_to_process
+        @max_records_to_process ||= (total_records * 1.1).round + 10
+      end
+    end
+  end
+end

data/lib/processor/thread.rb

@@ -1,11 +1,15 @@
 require 'processor/runner'
+require 'processor/event_processor'
 require 'processor/process_runner/successive'
 require 'processor/process_runner/threads'
+require 'processor/subroutine/recursion'
 
 module Processor
   class Thread
     def initialize(data_processor, *observers)
-      @runner = Runner.new data_processor, EventsRegistrator.new(observers)
+      data_processor = Subroutine::Recursion.new(data_processor)
+
+      @runner = Runner.new EventProcessor.new(data_processor, observers)
     end
 
     def run_as(&process_runner)

data/lib/processor/version.rb

@@ -1,3 +1,3 @@
 module Processor
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/lib/processor.rb

@@ -1,10 +1,11 @@
+require "processor/environment"
 require "processor/version"
 
 require "processor/data/array_processor"
 require "processor/data/batch_processor"
 require "processor/data/null_processor"
-require "processor/data/solr_processor"
 require "processor/data/solr_pages_processor"
+require "processor/data/csv_processor"
 
 require "processor/observer/logger"
 require "processor/observer/null_observer"

data/processor.gemspec

@@ -2,15 +2,15 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'processor/version'
+require 'processor/environment'
 
 Gem::Specification.new do |gem|
   gem.name          = "processor"
   gem.version       = Processor::VERSION
   gem.authors       = ["Alexander Paramonov"]
   gem.email         = ["alexander.n.paramonov@gmail.com"]
-  gem.summary       = %q{Universal processor for data migration and reports generation.}
-  gem.description   = %q{Processor could execute any DataProcessor you specify and log entire process.
-  You may add own observers for monitoring background tasks on even send email to bussiness with generated report.}
+  gem.summary       = %q{Universal processor for iteration over a collection with threads, logging and post processing}
+  gem.description   = %q{Processor is a tool that helps to iterate over collection and perform complex actions on a result. It is extremely useful in data migrations, report generation, etc. }
   gem.homepage      = "http://github.com/AlexParamonov/processor"
   gem.license       = "MIT"
 
@@ -21,6 +21,8 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec", ">= 2.6"
-  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry" unless Processor::RUNNING_ON_CI
+  gem.add_development_dependency "pry-plus" if "ruby" == RUBY_ENGINE && false == Processor::RUNNING_ON_CI
+  gem.add_development_dependency "coveralls" if Processor::RUNNING_ON_CI
 end
 

data/spec/processor/data/array_processor_spec.rb

@@ -3,8 +3,7 @@ require 'processor/data/array_processor'
 
 describe Processor::Data::ArrayProcessor do
   it "should have total records count equals to count of records" do
-    records = *1..5
-    subject.stub(records: records)
+    subject.stub(records: 1..5)
     subject.total_records.should eq 5
   end
 end

data/spec/processor/data/null_processor_spec.rb

@@ -1,9 +1,34 @@
 require 'spec_helper_lite'
 require 'processor/data/null_processor'
+require 'processor/runner'
+require 'processor/process_runner/successive'
 
 describe Processor::Data::NullProcessor do
-  it "should have a name equals to underscored class name" do
-    subject.name.should eq "processor_data_null_processor"
+  it "should have zero records" do
+    subject.records.should be_empty
+    subject.total_records.should be_zero
+  end
+
+  describe "required methods" do
+    it "should respond to all happy path methods" do
+      collector = Class.new do
+        def records; [1] end
+        def method_missing(method, *)
+          @methods = @methods.to_a << method
+        end
+        attr_reader :methods
+      end.new
+
+      runner = Processor::Runner.new collector
+      runner.run Processor::ProcessRunner::Successive.new
+      collector.methods.each do |method|
+        subject.should respond_to method
+      end
+    end
+
+    %w[record_error error total_records].each do |method|
+      it { subject.should respond_to method }
+    end
   end
 end
 

data/spec/processor/event_processor_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper_lite'
+require 'processor/event_processor'
+
+describe Processor::EventProcessor do
+  let(:processor) { stub.as_null_object }
+  let(:observer) { stub.as_null_object }
+  let(:observers) { [observer] }
+  subject { Processor::EventProcessor.new(processor, observers) }
+
+
+  it "should delegate methods to processor" do
+    processor.should_receive(:method_name).and_return("result")
+    subject.method_name.should eq "result"
+  end
+
+  it "should send a processor by update method" do
+    observer.should_receive(:update).with(/method_name/, processor)
+    subject.method_name
+  end
+
+  describe "before and after events" do
+    it "should fire before_method_name event before calling processor.method_name" do
+      observer.should_receive(:update).ordered do |event|
+        event.should eq :before_method_name
+      end
+      processor.should_receive(:method_name).ordered
+      subject.method_name
+    end
+
+    it "should fire after_method_name event after calling processor.method_name" do
+      observer.should_receive(:update)
+      processor.should_receive(:method_name, processor).ordered
+      observer.should_receive(:update).ordered do |event|
+        event.should eq :after_method_name
+      end
+      subject.method_name
+    end
+
+    it "should send result of processing to after_ event" do
+      result = stub
+      processor.stub(method_name: result)
+      observer.should_receive(:update).with(:after_method_name, processor, result).ordered
+      subject.method_name
+    end
+  end
+
+  describe "broadcasting of events" do
+    let(:observers) do
+      3.times.map do
+        stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event, processor).once }
+      end
+    end
+
+    it "should broadcast events to all observers" do
+      subject.register :test_event
+    end
+  end
+end

data/spec/processor/logger_messages_spec.rb

@@ -0,0 +1,69 @@
+require "spec_helper_lite"
+require "logger"
+require "processor/logger_messages"
+
+describe Processor::LoggerMessages do
+  let(:messages) { Processor::LoggerMessages.new logger }
+  shared_examples_for "a null logger" do
+    it "should not produce any output" do
+      messages.finished.should eq ""
+      messages.initialized.should eq ""
+    end
+  end
+
+  describe "Ruby Logger" do
+    describe "file" do
+      let(:filename) { "spec/support/dummy_file" }
+      let(:logger) { Logger.new File.new(filename) }
+
+      it "should include file name into message" do
+        messages.finished.should include filename
+        messages.initialized.should include filename
+      end
+    end
+
+    describe "filename" do
+      let(:filename) { "spec/support/dummy_file" }
+      let(:logger) { Logger.new filename }
+
+      it "should include file name into message" do
+        messages.finished.should include filename
+        messages.initialized.should include filename
+      end
+    end
+
+    describe "null" do
+      let(:logger) { ::Logger.new(File::NULL) }
+      it_behaves_like "a null logger"
+    end
+
+    describe "STDOUT" do
+      let(:logger) { Logger.new(STDOUT) }
+
+      it "should include IO in the message" do
+        messages.initialized.should include "IO"
+        messages.finished.should eq ""
+      end
+    end
+
+    describe "STDERR" do
+      let(:logger) { Logger.new(STDERR) }
+
+      it "should include IO in the message" do
+        messages.initialized.should include "IO"
+        messages.finished.should eq ""
+      end
+    end
+  end
+
+  describe "nil" do
+    let(:logger) { nil }
+    it_behaves_like "a null logger"
+  end
+
+  describe "unknown object" do
+    let(:logger) { Object.new }
+    it_behaves_like "a null logger"
+  end
+
+end

data/spec/processor/messenger_spec.rb

@@ -21,6 +21,21 @@ describe Processor::Messenger do
     messenger.fatal "fatal error"
   end
 
+  it "should not send empty messages" do
+    io.should_not_receive(:write)
+    messenger = Processor::Messenger.new :info, io
+    [nil, ""].each do |empty_message|
+      messenger.message empty_message
+    end
+  end
+
+  it "should send messages from a name of sender" do
+    sender = "File Logger"
+    io.should_receive(:write).with /#{sender}/
+    messenger = Processor::Messenger.new :info, io, sender
+    messenger.error "failed to create log file"
+  end
+
   describe "messages" do
     let(:messenger) { Processor::Messenger.new :debug, io }
     %w[debug info error fatal].each do |message_level|

data/spec/processor/observer/logger_spec.rb

@@ -2,31 +2,62 @@ require 'spec_helper_lite'
 require 'processor/observer/logger'
 
 describe Processor::Observer::Logger do
+  subject { Processor::Observer::Logger.new logger, messenger: messenger, messages: messages, processor: processor }
+
   let(:processor) { stub.as_null_object }
-  let(:no_messages) { ::Logger.new("/dev/null") }
+  let(:messages) { stub.as_null_object }
+  let(:messenger) { ::Logger.new("/dev/null") }
+  let(:logger) { ::Logger.new("/dev/null") }
+
+  describe "logger" do
+    describe "as proc" do
+      let(:external_logger) { stub }
+      let(:logger) { -> name { external_logger } }
+
+      it "should be assepted" do
+        subject.logger.should eq external_logger
+      end
+    end
+
+    describe "as object" do
+      let(:external_logger) { stub }
+      let(:logger) { external_logger }
+
+      it "should be assepted" do
+        subject.logger.should eq external_logger
+      end
+    end
 
-  subject { Processor::Observer::Logger }
-    it "accepts logger builder as parameter" do
-      external_logger = mock
-      logger_observer = subject.new -> name { external_logger }, messenger: no_messages
+    describe "nil" do
+      let(:logger) { nil }
 
-      external_logger.should_receive(:info)
-      logger_observer.processing_started processor
+      it "uses ruby Logger" do
+        ruby_logger = stub.as_null_object
+        observer = subject
+        ::Logger.should_receive(:new).and_return(ruby_logger)
+        observer.logger.should eq ruby_logger
+      end
     end
+  end
 
-    it "accepts logger as parameter" do
-      external_logger = mock
-      logger_observer = subject.new external_logger, messenger: no_messages
+  describe "messages" do
+    describe "as object" do
+      let(:messages) { stub }
 
-      external_logger.should_receive(:info)
-      logger_observer.processing_started processor
+      it "should be assepted" do
+        messages.should_receive(:initialized)
+        subject.after_start nil
+      end
     end
 
-    it "use ruby Logger if no external logger provided" do
-      logger_observer = subject.new nil, messenger: no_messages
+    describe "as hash" do
+      let(:messages) { { initialized: "INIT" } }
 
-      Logger.should_receive(:new).and_return(stub.as_null_object)
-      logger_observer.processing_started processor
+      it "should be assepted" do
+        messenger.should_receive(:info).with("INIT")
+        subject.after_start nil
+      end
     end
+  end
 end
 

data/spec/processor/observer/null_observer_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper_lite'
+require 'processor/observer/null_observer'
+
+describe Processor::Observer::NullObserver do
+
+  it "should send update message to itself if know how to respond to this message" do
+    subject.should_receive(:know_how)
+    subject.update :know_how
+  end
+
+  it "should ignore update message if dont know how to respond to this message" do
+    subject.stub(:respond_to?).with(:unknown_method).and_return(false)
+    subject.should_not_receive(:unknown_method)
+    subject.update :unknown_method
+  end
+
+  it "should blow up if got undefined method call" do
+    expect { subject.undefined_method }.to raise_error(NoMethodError)
+  end
+
+  describe "processor" do
+    let(:processor) { stub }
+    it "should set a processor" do
+      observer = Processor::Observer::NullObserver.new processor: processor
+      observer.processor.should eq processor
+    end
+
+    specify "update method should set a processor" do
+      subject.update :some_method, processor
+      subject.processor.should eq processor
+    end
+  end
+end

data/spec/processor/process_runner/specs.rb

@@ -1,49 +1,38 @@
 shared_examples_for "a records processor" do
   let(:process_runner) { described_class.new }
   let(:records) { 1..2 }
-  let(:processor) { stub.tap { |p| p.stub(records: records) } }
-
-  let(:no_recursion_preventer) { Proc.new{} }
-  let(:no_events) { stub.as_null_object }
+  let(:processor) { stub.tap { |p| p.stub(records: records) }.as_null_object }
 
   it "should fetch records from processor" do
     processor.should_receive(:records).and_return([])
-    process_runner.call processor, no_events, no_recursion_preventer
+    process_runner.call processor
   end
 
   it "should send each found record to processor" do
     records.each { |record| processor.should_receive(:process).with(record) }
-    process_runner.call processor, no_events, no_recursion_preventer
+    process_runner.call processor
   end
 
   describe "exception handling" do
     describe "processing a record raised StandardError" do
+      let(:records) { 1..3 }
+
       it "should continue processing" do
-        processor.should_receive(:process).twice.and_raise(StandardError)
-        expect { process_runner.call processor, no_events, no_recursion_preventer }.to_not raise_error
+        processor.should_receive(:process).exactly(3).times.and_raise(StandardError)
+        expect { process_runner.call processor }.to_not raise_error
       end
 
-      it "should register a record_processing_error event" do
-        event = mock.tap { |event| event.should_receive(:trigger).any_number_of_times }
-
-        events_registrator = stub
-        events_registrator.should_receive(:register) do |event_name, failed_record, exception|
-          next if event_name != :record_processing_error
-          event_name.should eq :record_processing_error
-          exception.should be_a StandardError
-          event.trigger
-        end.any_number_of_times
-
-        processor.stub(:process).and_raise(StandardError)
-
-        process_runner.call processor, events_registrator, no_recursion_preventer rescue nil
+      it "should call record_error" do
+        processor.should_receive(:process).at_least(1).and_raise(StandardError)
+        processor.should_receive(:record_error).at_least(1)
+        process_runner.call processor
       end
     end
 
     describe "processing a record raised Exception" do
       it "should break processing" do
         processor.should_receive(:process).once.and_raise(Exception)
-        expect { process_runner.call processor, no_events, no_recursion_preventer }.to raise_error(Exception)
+        expect { process_runner.call processor }.to raise_error(Exception)
       end
     end
   end

data/spec/processor/process_runner/threads_spec.rb

@@ -3,8 +3,6 @@ require_relative 'specs'
 require 'processor/process_runner/threads'
 
 describe Processor::ProcessRunner::Threads do
-  let(:no_recursion_preventer) { Proc.new{} }
-  let(:no_events) { stub.as_null_object }
   it_behaves_like "a records processor"
 
   it "should run in defined number of threads" do
@@ -20,6 +18,6 @@ describe Processor::ProcessRunner::Threads do
     processor.stub(records: 1..9)
     processor.should_receive(:process).exactly(9).times
 
-    process_runner.call(processor, no_events, no_recursion_preventer)
+    process_runner.call processor
   end
 end

data/spec/processor/runner_spec.rb

@@ -2,70 +2,44 @@ require 'spec_helper_lite'
 require 'processor/runner'
 
 describe Processor::Runner do
-  let(:runner) { Processor::Runner.new(processor, events_registrator) }
-  let(:processor) { stub }
-  let(:events_registrator) { stub.as_null_object }
+  let(:runner) { Processor::Runner.new(processor) }
+  let(:processor) { stub.as_null_object }
   let(:no_process_runner) { Proc.new{} }
 
-  describe "exception handling" do
+  it "should start processing" do
+    processor.should_receive(:start)
+    runner.run no_process_runner
+  end
+
+  it "should finish processing" do
+    processor.should_receive(:finish)
+    runner.run no_process_runner
+  end
+
+  it "should finalize processing" do
+    processor.should_receive(:finalize)
+    runner.run no_process_runner
+  end
+
+  describe "processing error" do
     describe "processing records raised" do
+      let(:process_runner) { Proc.new { raise RuntimeError } }
+
       it "should break processing and rerise" do
         expect do
-          runner.run Proc.new { raise RuntimeError }
+          runner.run process_runner
         end.to raise_error(RuntimeError)
       end
 
-      it "should register a processing_error event" do
-        register_processing_error_event mock.tap { |event| event.should_receive :trigger }
-        runner.run Proc.new { raise RuntimeError } rescue nil
+      it "should send error to processor" do
+        processor.should_receive(:error)
+        runner.run process_runner rescue nil
       end
-    end
-
-    private
-    def register_processing_error_event(event)
-      # Check that processing_error event was register
-      events_registrator.should_receive(:register) do |event_name, current_processor, exception|
-        next if event_name != :processing_error
-        event_name.should eq :processing_error
-        current_processor.should eq processor
-        exception.should be_a RuntimeError
-        event.trigger
-      end.any_number_of_times
-    end
-  end
-
-  describe "recursion" do
-    before(:each) do
-      processor.stub(total_records: 100)
-      processor.stub(records: 1..Float::INFINITY)
-    end
-
-    it "should not fall into recursion" do
-      processor.should_receive(:process).at_most(1000).times
 
-      expect do
-        process_runner = Proc.new do |data_processor, events, recursion_preventer|
-          data_processor.records.each do |record|
-            recursion_preventer.call
-            data_processor.process record
-          end
-        end
-        runner.run process_runner
-      end.to raise_error(Exception, /Processing fall into recursion/)
-    end
-
-    it "should have 10% + 10 rerurns window" do
-      processor.should_receive(:process).exactly(120).times
-
-      expect do
-        process_runner = Proc.new do |data_processor, events, recursion_preventer|
-          data_processor.records.each do |record|
-            recursion_preventer.call
-            data_processor.process record
-          end
-        end
-        runner.run process_runner
-      end.to raise_error(Exception, /Processing fall into recursion/)
+      it "should finalize processing" do
+        processor.should_receive(:finalize)
+        runner.run process_runner rescue nil
+      end
     end
   end
 end

data/spec/processor/subroutine/counter_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper_lite'
+require 'processor/subroutine/counter'
+
+describe Processor::Subroutine::Counter do
+  let(:records) { 1..10 }
+  let(:processor) { stub(total_records: 10, process: true) }
+  let(:subroutine) { Processor::Subroutine::Counter.new processor }
+
+  it "should count remaining records" do
+    subroutine.remaining_records_count.should eq 10
+    records.each do
+      expect { subroutine.process }.to change { subroutine.remaining_records_count }.by -1
+    end
+  end
+
+  it "should count processed records" do
+    subroutine.processed_records_count.should eq 0
+    records.each do
+      expect { subroutine.process }.to change { subroutine.processed_records_count }.by 1
+    end
+  end
+
+  describe "over process" do
+    before(:each) do
+      10.times { subroutine.process }
+    end
+
+    specify "remaining_records_count should be zero" do
+      subroutine.remaining_records_count.should eq 0
+      5.times do
+        expect { subroutine.process }.to_not change { subroutine.remaining_records_count }.from 0
+      end
+    end
+
+    specify "processed_records_count should keep counting" do
+      subroutine.processed_records_count.should eq 10
+      5.times do
+        expect { subroutine.process }.to change { subroutine.processed_records_count }.by 1
+      end
+    end
+  end
+
+end
+