processor 0.0.1 → 1.0.0.alpha

data/.travis.yml

@@ -3,11 +3,15 @@ rvm:
   - 1.9.3
   - ruby-head
   - rbx-19mode
+  - jruby-19mode
+  - jruby-head
 
 matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx-19mode
+    - rvm: jruby-19mode
+    - rvm: jruby-head
 
 before_install: gem install bundler --pre
 

data/README.md

@@ -3,7 +3,10 @@ Processor
 [![Build Status](https://travis-ci.org/AlexParamonov/processor.png?branch=master)](http://travis-ci.org/AlexParamonov/processor)
 [![Gemnasium Build Status](https://gemnasium.com/AlexParamonov/processor.png)](http://gemnasium.com/AlexParamonov/processor)  
 
-Universal processor for data migration
+Processor could execute any `DataProcessor` you specify and log entire process using any number of loggers you need.  
+You may add own observers for monitoring background tasks on even send an email to bussiness with generated report.  
+Processor provide customisation for almost every part of it.  
+
 
 Contents
 ---------
@@ -41,42 +44,61 @@ Requirements
 
 Usage
 ------------
-1. Implement a `DataProcessor`. See `Processor::Example::Migration`, `Processor::Example::SolrMigration`, `Processor::Example::SolrPagesMigration`
+1. Implement a `DataProcessor`. See `Processor::Example::Migration`
+and `processor/data` directory. CSV and Solr data processors are usabe
+but not yet finished and tested.
 1. Run your `DataProcessor`:
 
 ``` ruby
+data_processor = UserLocationMigration.new
 thread = Processor::Thread.new data_processor
 thread.run_successive
 ```
-See `spec/processor/thread_spec.rb` and `spec/example_spec.rb` and `example` directory for other usage examples.  
-Below are specs for most valuable parts of the gem.
-
-``` rspec
-Processor::Thread
-  should run a migration using provided block
-  should run a migration successive
-  should run a migration in threads
-
-Processor::DataProcessor
-  should have a name equals to underscored class name
-  should be done when there are 0 records to process
-  should respond to done?
-  should respond to fetch_records
-  should respond to total_records
-  should respond to process
-
-Processor::Observer::Logger
-  accepts logger builder as parameter
-  accepts logger as parameter
-  use ruby Logger if no external logger provided
+See `spec/processor/thread_spec.rb` and `spec/example_spec.rb` and
+`example` directory for other usage examples.  
+
+
+It is recomended to wrap Processor::Thread in your classes like:
+
+```
+WeeklyReport
+TaxonomyMigration
+UserDataImport
+```
+to hide configuration of observers or use your own API to run
+migrations:
+
 ```
+weekly_report.create_and_deliver
+user_data_import.import_from_csv(file)
+etc.
+```
+
+Sure, it is possible to use it raw, but please dont fear to add a
+wrapper class for it:
+
+```
+csv_data_processor = Processor::Data::CsvProcessor.new file
+stdout_notifier = Processor::Observer::Logger.new(Logger.new(STDOUT))
+logger_observer = Processor::Observer::Logger.new
+Processor::Thread.new(
+  csv_data_processor,
+  stdout_notifier,
+  logger_observer,
+  email_notification_observer
+).run_in_threads 5
+```
+
+### Observers
+Observers should respond to `update` method but if you inherit from
+`Processor::Observers::NullObserver` you'll get a bunch of methods to
+use. See `Processor::Observers::Logger` for example.
 
 Compatibility
 -------------
 tested with Ruby
 
 * 1.9.3
-* jruby-19mode
 * rbx-19mode
 * ruby-head
 

data/example/migration.rb

@@ -1,29 +1,18 @@
-require 'processor/data_processor'
+require 'processor/data/array_processor'
 
 module Processor
   module Example
-    class Migration < DataProcessor
+    class Migration < Data::ArrayProcessor
       attr_reader :records
+
       def initialize(records)
         @records = records
       end
 
-      def done?(records)
-        records.count < 1
-      end
-
       def process(record)
         record.do_something
         "OK"
       end
-
-      def fetch_records
-        records.shift(2)
-      end
-
-      def total_records
-        records.count
-      end
     end
   end
 end

data/lib/processor/data/array_processor.rb

@@ -0,0 +1,11 @@
+require_relative 'null_processor'
+
+module Processor
+  module Data
+    class ArrayProcessor < NullProcessor
+      def records
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/processor/data/batch_processor.rb

@@ -0,0 +1,37 @@
+require_relative 'null_processor'
+
+module Processor
+  module Data
+    class BatchProcessor < NullProcessor
+      def initialize(batch_size = 10)
+        @batch_size = batch_size
+      end
+
+      def records
+        Enumerator.new do |result|
+          loop do
+            fetch_batch.each do |record|
+              result << record
+            end
+          end
+        end
+      end
+
+      def fetch_batch
+        @fetcher ||= query.each_slice(batch_size)
+        @fetcher.next
+      end
+
+      def total_records
+        @total_records ||= query.count
+      end
+
+      def query
+        raise NotImplementedError
+      end
+
+      private
+      attr_reader :batch_size
+    end
+  end
+end

data/lib/processor/data/csv_processor.rb

@@ -0,0 +1,39 @@
+require_relative 'batch_processor'
+
+module Processor
+  module Data
+    class CsvProcessor < BatchProcessor
+      def initialize(file, csv_options = {})
+        @file = file
+        @separator = separator
+        @csv_options = {
+          col_sep: ";",
+          headers: true,
+        }.merge csv_options
+      end
+
+      def process(row)
+        raise NotImplementedError
+      end
+
+      def records
+        Enumerator.new do |result|
+          ::CSV.foreach(file, csv_options) do |record|
+            result << record
+          end
+        end
+      end
+
+      def total_records
+        @total_records ||= File.new(file).readlines.size
+      end
+
+      private
+      attr_reader :file, :separator, :csv_options
+
+      def fetch_field(field_name, row)
+        row[field_name].to_s.strip
+      end
+    end
+  end
+end

data/lib/processor/data/null_processor.rb

@@ -0,0 +1,26 @@
+module Processor
+  module Data
+    class NullProcessor
+      def process(record)
+        # do nothing
+      end
+
+      def records
+        []
+      end
+
+      def total_records
+        @total_records ||= records.count
+      end
+
+      def name
+        # underscore a class name
+        self.class.name.to_s.
+          gsub(/::/, '_').
+          gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+          gsub(/([a-z\d])([A-Z])/,'\1_\2').
+          downcase
+      end
+    end
+  end
+end

data/lib/processor/data/solr_pages_processor.rb

@@ -0,0 +1,30 @@
+require_relative "batch_processor"
+
+module Processor
+  module Data
+    class SolrPagesProcessor < BatchProcessor
+      def process(record)
+        raise NotImplementedError
+      end
+
+      def query(requested_page)
+        raise NotImplementedError
+      end
+
+      def fetch_batch
+        query(next_page).results
+      end
+
+      def total_records
+        @total_records ||= query(1).total
+      end
+
+      private
+      def next_page
+        @page ||= 0
+        @page += 1
+      end
+    end
+  end
+end
+

data/lib/processor/data/solr_processor.rb

@@ -0,0 +1,23 @@
+require_relative "batch_processor"
+
+module Processor
+  module Data
+    class SolrProcessor < BatchProcessor
+      def process(record)
+        raise NotImplementedError
+      end
+
+      def query
+        raise NotImplementedError
+      end
+
+      def fetch_batch
+        query.results
+      end
+
+      def total_records
+        @total_records ||= query.total
+      end
+    end
+  end
+end

data/lib/processor/observer/logger.rb

@@ -26,7 +26,7 @@ module Processor
       end
 
       def after_record_processing(record, result)
-        logger.info "Successfully processed #{id_for record}: #{result}"
+        logger.info "Processed #{id_for record}: #{result}"
       end
 
       def processing_finished(processor)
@@ -39,7 +39,7 @@ module Processor
       end
 
       def processing_error(processor, exception)
-        logger.fatal "Processing #{processor.name} failed: #{exception}"
+        logger.fatal "Processing #{processor.name} FAILED: #{exception.backtrace}"
       end
 
       private
@@ -79,7 +79,7 @@ module Processor
           return record[method.to_s] if record.key? method.to_s
         end if record.respond_to?(:key?) && record.respond_to?(:[])
 
-        record.to_s
+        record.to_s.strip
       end
     end
   end

data/lib/processor/process_runner/successive.rb

@@ -0,0 +1,12 @@
+require_relative "threads"
+
+module Processor
+  module ProcessRunner
+    class Successive < Threads
+      private
+      def new_thread(processor, record, &block)
+        block.call processor, record
+      end
+    end
+  end
+end

data/lib/processor/process_runner/threads.rb

@@ -0,0 +1,51 @@
+module Processor
+  module ProcessRunner
+    class Threads
+      def initialize(number_of_threads = 1)
+        @number_of_threads = number_of_threads
+        @threads = []
+      end
+
+      def call(processor, events, recursion_preventer)
+        join_threads
+
+        begin
+          processor.records.each do |record|
+            recursion_preventer.call
+            if threads_created >= number_of_threads then join_threads end
+
+            new_thread(processor, record) do |thread_data_processor, thread_record|
+              begin
+                events.register :before_record_processing, thread_record
+
+                result = thread_data_processor.process(thread_record)
+
+                events.register :after_record_processing, thread_record, result
+              rescue StandardError => exception
+                events.register :record_processing_error, thread_record, exception
+              end
+            end
+
+          end
+        ensure # join already created threads
+          join_threads
+        end
+      end
+
+      private
+      attr_reader :threads_created, :number_of_threads
+
+      def new_thread(processor, record, &block)
+        @threads << ::Thread.new(processor, record, &block)
+        @threads_created += 1
+      end
+
+      def join_threads
+        @threads.each(&:join)
+        @threads_created = 0
+        @threads = []
+      end
+
+    end
+  end
+end

data/lib/processor/runner.rb

@@ -7,11 +7,10 @@ module Processor
       @events = events_registrator
     end
 
-    def run(records_processor)
+    def run(process_runner)
       events.register :processing_started, processor
-      until processor.done?(records = processor.fetch_records)
-        records_processor.call records, processor, events, method(:recursion_preventer)
-      end
+
+      process_runner.call processor, events, method(:recursion_preventer)
 
       events.register :processing_finished, processor
     rescue Exception => exception
@@ -33,7 +32,7 @@ module Processor
     end
 
     def max_records_to_process
-      (processor.total_records * 1.1).round + 10
+      @max_records_to_process ||= (processor.total_records * 1.1).round + 10
     end
   end
 end

data/lib/processor/thread.rb

@@ -1,6 +1,6 @@
 require 'processor/runner'
-require 'processor/records_processor/successive'
-require 'processor/records_processor/threads'
+require 'processor/process_runner/successive'
+require 'processor/process_runner/threads'
 
 module Processor
   class Thread
@@ -8,16 +8,16 @@ module Processor
       @runner = Runner.new data_processor, EventsRegistrator.new(observers)
     end
 
-    def run_as(&records_processor)
-      runner.run records_processor
+    def run_as(&process_runner)
+      runner.run process_runner
     end
 
     def run_successive
-      runner.run RecordsProcessor::Successive.new
+      runner.run ProcessRunner::Successive.new
     end
 
-    def run_in_threads
-      runner.run RecordsProcessor::Threads.new
+    def run_in_threads(number_of_threads = 2)
+      runner.run ProcessRunner::Threads.new number_of_threads
     end
 
     private

data/lib/processor/version.rb

@@ -1,3 +1,3 @@
 module Processor
-  VERSION = "0.0.1"
+  VERSION = "1.0.0.alpha"
 end

data/lib/processor.rb

@@ -1,4 +1,17 @@
 require "processor/version"
 
+require "processor/data/array_processor"
+require "processor/data/batch_processor"
+require "processor/data/null_processor"
+
+require "processor/observer/logger"
+require "processor/observer/null_observer"
+
+require "processor/process_runner/successive"
+require "processor/process_runner/threads"
+
+require "processor/runner"
+require "processor/thread"
+
 module Processor
 end

data/processor.gemspec

@@ -8,8 +8,9 @@ Gem::Specification.new do |gem|
   gem.version       = Processor::VERSION
   gem.authors       = ["Alexander Paramonov"]
   gem.email         = ["alexander.n.paramonov@gmail.com"]
-  gem.summary       = %q{Process records one by one}
-  gem.description   = %q{Universal processor for data migration}
+  gem.summary       = %q{Universal processor for data migration and reports generation.}
+  gem.description   = %q{Processor could execute any DataProcessor you specify and log entire process.
+  You may add own observers for monitoring background tasks on even send email to bussiness with generated report.}
   gem.homepage      = "http://github.com/AlexParamonov/processor"
   gem.license       = "MIT"
 

data/spec/processor/data/array_processor_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper_lite'
+require 'processor/data/array_processor'
+
+describe Processor::Data::ArrayProcessor do
+  it "should have total records count equals to count of records" do
+    records = *1..5
+    subject.stub(records: records)
+    subject.total_records.should eq 5
+  end
+end
+

data/spec/processor/data/batch_processor_spec.rb

@@ -0,0 +1,38 @@
+require "spec_helper_lite"
+
+require "processor/data/batch_processor"
+
+describe Processor::Data::BatchProcessor do
+  it "should create and process records by batch" do
+    processor = Processor::Data::BatchProcessor.new 2
+
+    watcher = mock
+    5.times do
+      watcher.should_receive(:created).ordered
+      watcher.should_receive(:created).ordered
+      watcher.should_receive(:processed).ordered
+      watcher.should_receive(:processed).ordered
+    end
+
+    query = Enumerator.new do |y|
+      a = 1
+      loop do
+        break if a > 10
+        watcher.created
+        y << a
+        a += 1
+      end
+    end
+
+    processor.stub(query: query)
+    processor.records.each do |record|
+      watcher.processed
+    end
+  end
+
+  it "should have total records count equals to count of query" do
+    query = 1..5
+    subject.stub(query: query)
+    subject.total_records.should eq 5
+  end
+end

data/spec/processor/data/null_processor_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper_lite'
+require 'processor/data/null_processor'
+
+describe Processor::Data::NullProcessor do
+  it "should have a name equals to underscored class name" do
+    subject.name.should eq "processor_data_null_processor"
+  end
+end
+

data/spec/processor/process_runner/specs.rb

@@ -0,0 +1,50 @@
+shared_examples_for "a records processor" do
+  let(:process_runner) { described_class.new }
+  let(:records) { 1..2 }
+  let(:processor) { stub.tap { |p| p.stub(records: records) } }
+
+  let(:no_recursion_preventer) { Proc.new{} }
+  let(:no_events) { stub.as_null_object }
+
+  it "should fetch records from processor" do
+    processor.should_receive(:records).and_return([])
+    process_runner.call processor, no_events, no_recursion_preventer
+  end
+
+  it "should send each found record to processor" do
+    records.each { |record| processor.should_receive(:process).with(record) }
+    process_runner.call processor, no_events, no_recursion_preventer
+  end
+
+  describe "exception handling" do
+    describe "processing a record raised StandardError" do
+      it "should continue processing" do
+        processor.should_receive(:process).twice.and_raise(StandardError)
+        expect { process_runner.call processor, no_events, no_recursion_preventer }.to_not raise_error
+      end
+
+      it "should register a record_processing_error event" do
+        event = mock.tap { |event| event.should_receive(:trigger).any_number_of_times }
+
+        events_registrator = stub
+        events_registrator.should_receive(:register) do |event_name, failed_record, exception|
+          next if event_name != :record_processing_error
+          event_name.should eq :record_processing_error
+          exception.should be_a StandardError
+          event.trigger
+        end.any_number_of_times
+
+        processor.stub(:process).and_raise(StandardError)
+
+        process_runner.call processor, events_registrator, no_recursion_preventer rescue nil
+      end
+    end
+
+    describe "processing a record raised Exception" do
+      it "should break processing" do
+        processor.should_receive(:process).once.and_raise(Exception)
+        expect { process_runner.call processor, no_events, no_recursion_preventer }.to raise_error(Exception)
+      end
+    end
+  end
+end

data/spec/processor/process_runner/successive_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper_lite'
+require_relative 'specs'
+require 'processor/process_runner/successive'
+
+describe Processor::ProcessRunner::Successive do
+  it_behaves_like "a records processor"
+end

data/spec/processor/process_runner/threads_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper_lite'
+require_relative 'specs'
+require 'processor/process_runner/threads'
+
+describe Processor::ProcessRunner::Threads do
+  let(:no_recursion_preventer) { Proc.new{} }
+  let(:no_events) { stub.as_null_object }
+  it_behaves_like "a records processor"
+
+  it "should run in defined number of threads" do
+    process_runner = Processor::ProcessRunner::Threads.new 5
+
+    process_runner.should_receive(:join_threads).once.ordered.and_call_original
+    process_runner.should_receive(:new_thread).exactly(5).times.ordered.and_call_original
+    process_runner.should_receive(:join_threads).once.ordered.and_call_original
+    process_runner.should_receive(:new_thread).exactly(4).times.ordered.and_call_original
+    process_runner.should_receive(:join_threads).once.ordered.and_call_original
+
+    processor = mock
+    processor.stub(records: 1..9)
+    processor.should_receive(:process).exactly(9).times
+
+    process_runner.call(processor, no_events, no_recursion_preventer)
+  end
+end

data/spec/processor/runner_spec.rb

@@ -5,25 +5,10 @@ describe Processor::Runner do
   let(:runner) { Processor::Runner.new(processor, events_registrator) }
   let(:processor) { stub }
   let(:events_registrator) { stub.as_null_object }
-  let(:no_records_processor) { Proc.new{} }
-
-  it "should fetch records from processor till it'll be done" do
-    processor.stub(:done?).and_return(false, false, true)
-    processor.should_receive(:fetch_records).exactly(3).times.and_return([])
-    runner.run no_records_processor
-  end
+  let(:no_process_runner) { Proc.new{} }
 
   describe "exception handling" do
-    let(:record) { stub }
-    before(:each) do
-      processor.stub(:fetch_records).and_return([record], [record], [])
-    end
-
     describe "processing records raised" do
-      before(:each) do
-        processor.stub(done?: false)
-      end
-
       it "should break processing and rerise" do
         expect do
           runner.run Proc.new { raise RuntimeError }
@@ -36,21 +21,6 @@ describe Processor::Runner do
       end
     end
 
-    describe "fetching records raised" do
-      it "should break processing and rerise Exception" do
-        processor.stub(:fetch_records).and_raise(RuntimeError)
-        processor.should_not_receive(:process)
-        expect { runner.run no_records_processor }.to raise_error(RuntimeError)
-      end
-
-      it "should register a processing_error" do
-        register_processing_error_event mock.tap { |event| event.should_receive :trigger }
-
-        processor.stub(:fetch_records).and_raise(RuntimeError)
-        runner.run no_records_processor rescue nil
-      end
-    end
-
     private
     def register_processing_error_event(event)
       # Check that processing_error event was register
@@ -66,22 +36,21 @@ describe Processor::Runner do
 
   describe "recursion" do
     before(:each) do
-      processor.stub(done?: false)
       processor.stub(total_records: 100)
-      processor.stub(fetch_records: 1..3)
+      processor.stub(records: 1..Float::INFINITY)
     end
 
     it "should not fall into recursion" do
       processor.should_receive(:process).at_most(1000).times
 
       expect do
-        records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
-          records.each do |record|
+        process_runner = Proc.new do |data_processor, events, recursion_preventer|
+          data_processor.records.each do |record|
             recursion_preventer.call
-            records_processor.process record
+            data_processor.process record
           end
         end
-        runner.run records_processor
+        runner.run process_runner
       end.to raise_error(Exception, /Processing fall into recursion/)
     end
 
@@ -89,13 +58,13 @@ describe Processor::Runner do
       processor.should_receive(:process).exactly(120).times
 
       expect do
-        records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
-          records.each do |record|
+        process_runner = Proc.new do |data_processor, events, recursion_preventer|
+          data_processor.records.each do |record|
             recursion_preventer.call
-            processor.process record
+            data_processor.process record
           end
         end
-        runner.run records_processor
+        runner.run process_runner
       end.to raise_error(Exception, /Processing fall into recursion/)
     end
   end

data/spec/processor/thread_spec.rb

@@ -13,8 +13,8 @@ describe Processor::Thread do
 
   it "should run a migration using provided block" do
     thread = Processor::Thread.new @migration
-    thread.run_as do |records, processor, *|
-      records.each do |record|
+    thread.run_as do |processor, *|
+      processor.records.each do |record|
         processor.process record
       end
     end
@@ -30,7 +30,7 @@ describe Processor::Thread do
     thread.run_in_threads
   end
 
-  pending "should run a migration in specifien number of threads" do
+  it "should run a migration in specifien number of threads" do
     thread = Processor::Thread.new @migration
     thread.run_in_threads 3
   end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: processor
 version: !ruby/object:Gem::Version
-  version: 0.0.1
-  prerelease: 
+  version: 1.0.0.alpha
+  prerelease: 6
 platform: ruby
 authors:
 - Alexander Paramonov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-11 00:00:00.000000000 Z
+date: 2013-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -59,7 +59,9 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Universal processor for data migration
+description: ! "Processor could execute any DataProcessor you specify and log entire
+  process.\n  You may add own observers for monitoring background tasks on even send
+  email to bussiness with generated report."
 email:
 - alexander.n.paramonov@gmail.com
 executables: []
@@ -78,26 +80,31 @@ files:
 - example/migration.rb
 - example/migrator.rb
 - example/observer/progress_bar.rb
-- example/solr_migration.rb
-- example/solr_pages_migration.rb
 - lib/processor.rb
-- lib/processor/data_processor.rb
+- lib/processor/data/array_processor.rb
+- lib/processor/data/batch_processor.rb
+- lib/processor/data/csv_processor.rb
+- lib/processor/data/null_processor.rb
+- lib/processor/data/solr_pages_processor.rb
+- lib/processor/data/solr_processor.rb
 - lib/processor/events_registrator.rb
 - lib/processor/observer/logger.rb
 - lib/processor/observer/null_observer.rb
-- lib/processor/records_processor/successive.rb
-- lib/processor/records_processor/threads.rb
+- lib/processor/process_runner/successive.rb
+- lib/processor/process_runner/threads.rb
 - lib/processor/runner.rb
 - lib/processor/thread.rb
 - lib/processor/version.rb
 - processor.gemspec
 - spec/example_spec.rb
-- spec/processor/data_processor_spec.rb
+- spec/processor/data/array_processor_spec.rb
+- spec/processor/data/batch_processor_spec.rb
+- spec/processor/data/null_processor_spec.rb
 - spec/processor/events_registrator_spec.rb
 - spec/processor/observer/logger_spec.rb
-- spec/processor/records_processor/specs.rb
-- spec/processor/records_processor/successive_spec.rb
-- spec/processor/records_processor/threads_spec.rb
+- spec/processor/process_runner/specs.rb
+- spec/processor/process_runner/successive_spec.rb
+- spec/processor/process_runner/threads_spec.rb
 - spec/processor/runner_spec.rb
 - spec/processor/thread_spec.rb
 - spec/spec_helper_lite.rb
@@ -116,30 +123,29 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4470760754938067473
+      hash: -1752503773457190967
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
-      segments:
-      - 0
-      hash: 4470760754938067473
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: Process records one by one
+summary: Universal processor for data migration and reports generation.
 test_files:
 - spec/example_spec.rb
-- spec/processor/data_processor_spec.rb
+- spec/processor/data/array_processor_spec.rb
+- spec/processor/data/batch_processor_spec.rb
+- spec/processor/data/null_processor_spec.rb
 - spec/processor/events_registrator_spec.rb
 - spec/processor/observer/logger_spec.rb
-- spec/processor/records_processor/specs.rb
-- spec/processor/records_processor/successive_spec.rb
-- spec/processor/records_processor/threads_spec.rb
+- spec/processor/process_runner/specs.rb
+- spec/processor/process_runner/successive_spec.rb
+- spec/processor/process_runner/threads_spec.rb
 - spec/processor/runner_spec.rb
 - spec/processor/thread_spec.rb
 - spec/spec_helper_lite.rb

data/example/solr_migration.rb

@@ -1,31 +0,0 @@
-require 'processor/data_processor'
-
-module Processor
-  module Example
-    class SolrMigration < DataProcessor
-      def process(user)
-        user.set_contact_method "Address book"
-        user.save!
-      end
-
-      def fetch_records
-        query.results
-      end
-
-      def total_records
-        @total_records ||= query.total
-      end
-
-      private
-      # query will return 0 records when all users'll be processed successfully
-      def query
-        User.search {
-          fulltext "My company"
-          with :title, "Manager"
-          with :contact_method, "Direct contact"
-          paginate page: 1, per_page: 10
-        }
-      end
-    end
-  end
-end

data/example/solr_pages_migration.rb

@@ -1,48 +0,0 @@
-require 'processor/data_processor'
-
-module Processor
-  module Example
-    class SolrPagesMigration < DataProcessor
-      def done?(records)
-        # Optional custom check for migration to be done.
-
-        # your custom check
-
-        # use a default check if you like
-        super
-      end
-
-      def process(user)
-        user.set_contact_method "Address book"
-        user.save!
-      end
-
-      def fetch_records
-        query(next_page).results
-      end
-
-      def total_records
-        @total_records ||= query(1).total
-      end
-
-      # optional name to use in observers.
-      def name
-        "my_company_users_contact_method_migration"
-      end
-
-      private
-      def query(requested_page)
-        User.search {
-          fulltext "My company"
-          paginate page: requested_page, per_page: 10
-        }
-      end
-
-      def next_page
-        @page ||= 0
-        @page += 1
-      end
-    end
-  end
-end
-

data/lib/processor/data_processor.rb

@@ -1,28 +0,0 @@
-module Processor
-  class DataProcessor
-    def done?(records)
-      records.count < 1
-    end
-
-    def process(record)
-      raise NotImplementedError
-    end
-
-    def fetch_records
-      raise NotImplementedError
-    end
-
-    def total_records
-      raise NotImplementedError
-    end
-
-    def name
-      # underscore a class name
-      self.class.name.to_s.
-        gsub(/::/, '_').
-        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-        gsub(/([a-z\d])([A-Z])/,'\1_\2').
-        downcase
-    end
-  end
-end

data/lib/processor/records_processor/successive.rb

@@ -1,20 +0,0 @@
-module Processor
-  module RecordsProcessor
-    class Successive
-      def call(records, processor, events, recursion_preventer)
-        records.each do |record|
-          recursion_preventer.call
-          begin
-            events.register :before_record_processing, record
-
-            result = processor.process(record)
-
-            events.register :after_record_processing, record, result
-          rescue RuntimeError => exception
-            events.register :record_processing_error, record, exception
-          end
-        end
-      end
-    end
-  end
-end

data/lib/processor/records_processor/threads.rb

@@ -1,27 +0,0 @@
-module Processor
-  module RecordsProcessor
-    class Threads
-      def call(records, processor, events, recursion_preventer)
-        threads = []
-        begin
-          records.each do |record|
-            recursion_preventer.call
-            threads << ::Thread.new(processor, record) do |thread_data_processor, thread_record|
-              begin
-                events.register :before_record_processing, thread_record
-
-                result = thread_data_processor.process(thread_record)
-
-                events.register :after_record_processing, thread_record, result
-              rescue RuntimeError => exception
-                events.register :record_processing_error, thread_record, exception
-              end
-            end
-          end
-        ensure # join already created threads even if recursion was detected
-          threads.each(&:join)
-        end
-      end
-    end
-  end
-end

data/spec/processor/data_processor_spec.rb

@@ -1,20 +0,0 @@
-require 'spec_helper_lite'
-require 'processor/data_processor'
-
-describe Processor::DataProcessor do
-  it "should have a name equals to underscored class name" do
-    subject.name.should eq "processor_data_processor"
-  end
-
-  it "should be done when there are 0 records to process" do
-    records = []
-    subject.done?(records).should be true
-  end
-
-  %w[done? fetch_records total_records process].each do |method_name|
-    it "should respond to #{method_name}" do
-      subject.should respond_to method_name
-    end
-  end
-end
-

data/spec/processor/records_processor/specs.rb

@@ -1,38 +0,0 @@
-shared_examples_for "a records processor" do
-  let(:records_processor) { described_class.new }
-  let(:records) { 1..2 }
-  let(:processor) { stub }
-
-  let(:no_recursion_preventer) { Proc.new{} }
-  let(:no_events) { stub.as_null_object }
-
-  it "should send each found record to processor" do
-    records.each { |record| processor.should_receive(:process).with(record) }
-    records_processor.call records, processor, no_events, no_recursion_preventer
-  end
-
-  describe "exception handling" do
-    describe "processing a record raised RuntimeError" do
-      it "should continue processing" do
-        processor.should_receive(:process).twice.and_raise(RuntimeError)
-        expect { records_processor.call records, processor, no_events, no_recursion_preventer }.to_not raise_error
-      end
-
-      it "should register a record_processing_error event" do
-        event = mock.tap { |event| event.should_receive(:trigger).any_number_of_times }
-
-        events_registrator = stub
-        events_registrator.should_receive(:register) do |event_name, failed_record, exception|
-          next if event_name != :record_processing_error
-          event_name.should eq :record_processing_error
-          exception.should be_a RuntimeError
-          event.trigger
-        end.any_number_of_times
-
-        processor.stub(:process).and_raise(RuntimeError)
-
-        records_processor.call records, processor, events_registrator, no_recursion_preventer rescue nil
-      end
-    end
-  end
-end

data/spec/processor/records_processor/successive_spec.rb

@@ -1,7 +0,0 @@
-require 'spec_helper_lite'
-require_relative 'specs'
-require 'processor/records_processor/successive'
-
-describe Processor::RecordsProcessor::Successive do
-  it_behaves_like "a records processor"
-end

data/spec/processor/records_processor/threads_spec.rb

@@ -1,8 +0,0 @@
-require 'spec_helper_lite'
-require_relative 'specs'
-require 'processor/records_processor/threads'
-
-describe Processor::RecordsProcessor::Threads do
-  it_behaves_like "a records processor"
-  it "should run in defined number of threads"
-end