processor 2.1.0 → 2.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzUxNDQ3NmM4YzZhOTk5Njc0MjVmNThiNDJhYTlhZWQ0NjhkNjJlOQ==
+  data.tar.gz: !binary |-
+    MWExYTAxNjk0MDU3MGRiMDc3NmE3NzA0MzNkZjMzMTYxZTEwY2I3OA==
+SHA512:
+  metadata.gz: !binary |-
+    NzcxYTA0Y2UyYTU4OGNiYTA0YjkxMDdlODE5YTQ2ZjRhZDA3ZGY0ODM0MWY5
+    N2Y5NzY3MzI4MTZiN2UyMDY0NDgwYzQ1N2E1NDIxNmU2NmM4Y2I0ZmE5YjI1
+    NGU2ZjIwZTU1OWVmNjRmM2FhODY1ODFiOTBmODI5ZWI0MWExNGU=
+  data.tar.gz: !binary |-
+    MWQ3MWIyZGIzZGVhYzA1ZWJkYjlhOWM5NzE0MDIzZDRhNjE2NmI0MGE5OTk5
+    N2FhZTBiMzViNDY0MTM1NWMyNGNkZDI1ZTNjZDcxYjA5OTgzMzBjZThkMTNk
+    NzNmOWE2YjdhNDMwZmE1YmM5MjU3N2Q4ZmZiZWRlNjAzM2MwZjI=

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 rvm:
   - 1.9.3
+  - 2.1.4
   - ruby-head
   - rbx-19mode
   - jruby-19mode

data/README.md

@@ -5,18 +5,18 @@ Processor
 [![Coverage Status](https://coveralls.io/repos/AlexParamonov/processor/badge.png?branch=master)](https://coveralls.io/r/AlexParamonov/processor?branch=master)
 
 
-Processor is a tool that helps to iterate over collection and
+Processor is a tool that helps to iterate over a collection and
 perform complex actions on a result. It is extremely useful in data
 migrations, report generation, etc.
 
 Collection could be iteratively fetched by parts but for processor
 it will looks like an endless collection. There are a lot such tiny
 goodness that makes usage of processor pleasant. Need logging,
-exception processing, post/pre processing a result - no problems, all
+exception processing, post/pre processing a result? No problem - all
 included and easily extended.
 
-Use the processor to DRY your migrations, report and stop mess with
-logging and post processing.
+Use the processor to DRY your migrations, reports and to stop mess
+with logging and post processing.
 
 Did I mentioned you can run in threads as easy as say
 `processor.run_in_threads 10`?
@@ -36,8 +36,8 @@ Contents
     1. Data processors
     1. Subroutines
     1. Run modes
-    1. Processor Thread
     1. Observers
+    1. Processor Thread
 1. Contacts
 1. Compatibility
 1. Contributing
@@ -45,17 +45,20 @@ Contents
 
 Installation
 ------------
-Add this line to your application's Gemfile:
+If on Rails, add this line to your application's Gemfile:
+
 ``` ruby
 gem 'processor'
 ```
 
 And then execute:
+
 ``` sh
 bundle
 ```
 
-Or install it yourself as:
+Or install it yourself:
+
 ``` sh
 gem install processor
 ```
@@ -148,6 +151,7 @@ It runs `process` one by one for each found record returned by
 `records` method.
 
 Call it using a `Processor::Thread`:
+
 ``` ruby
 Processor::Thread.new(migration).run_successive
 ```
@@ -158,15 +162,20 @@ not waiting for previous `process` to finish.
 
 Possible to specify number of threads used by passing a number to
 constructor:
+
 ``` ruby
 Processor::ProcessRunner::Threads.new 5
 ```
 
 Call it using a `Processor::Thread`:
+
 ``` ruby
 Processor::Thread.new(migration).run_in_threads 5
 ```
 
+Note: on MRI ruby you could expect perfomance gain using threads if
+your application has resource consuming IO operations
+
 
 ### Observers
 Processor support unlimited number of observers that are watching
@@ -191,16 +200,19 @@ Read below section Processor Thread to see how to use observers in runner.
 Processor classes and provides __stable__ interface.
 
 Creating a new Thread:
+
 ``` ruby
 Processor::Thread.new data_processor
 ```
 
 You may provide optional observers:
+
 ``` ruby
 Processor::Thread.new data_processor, observer1, observer2, ...
 ```
 
 Instance have a `run_as` method that accepts a block:
+
 ``` ruby
 thread = Processor::Thread.new @migration
 thread.run_as do |processor|
@@ -211,6 +223,7 @@ end
 ```
 
 Instance have a `run_successive` method: 
+
 ``` ruby
 data_processor = UserLocationMigration.new
 thread = Processor::Thread.new data_processor
@@ -218,6 +231,7 @@ thread.run_successive
 ```
 
 And `run_in_threads` method:
+
 ``` ruby
 data_processor = UserCsvImport.new csv_file
 thread = Processor::Thread.new data_processor
@@ -228,6 +242,7 @@ See `spec/processor/thread_spec.rb` and `spec/example_spec.rb` and
 `example` directory for other usage examples.
 
 It is recommended to wrap Processor::Thread by classes named like:
+
 ``` ruby
 WeeklyReport
 TaxonomyMigration
@@ -236,6 +251,7 @@ UserDataImport
 
 The point is to hide configuration of observers and use (if you wish)
 your own API to run reports or migrations:
+
 ``` ruby
 weekly_report.create_and_deliver
 user_data_import.from_csv(file)
@@ -244,6 +260,7 @@ etc.
 
 It is possible to use it raw, but please don't fear to add a wrapper
 class like `CsvUserImport` for this:
+
 ``` ruby
 csv_data_processor = Processor::Data::CsvProcessor.new file
 stdout_notifier = Processor::Observer::Logger.new(Logger.new(STDOUT))
@@ -257,6 +274,7 @@ Processor::Thread.new(
 ```
 
 More documentation could be found by running
+
 ``` sh
 rspec
 ```
@@ -274,6 +292,7 @@ Compatibility
 tested with Ruby
 
 * 1.9.3
+* 2.1.4
 * rbx-19mode
 * ruby-head
 

data/lib/processor/data/batch_processor.rb

@@ -2,6 +2,8 @@ require_relative 'null_processor'
 
 module Processor
   module Data
+    # TODO: Change it to be useful.
+    # Usually the external iterator is provided and #results are mapped to it
     class BatchProcessor < NullProcessor
       def initialize(batch_size = 10)
         @batch_size = batch_size
@@ -9,9 +11,7 @@ module Processor
 
       def records
         Enumerator.new do |result|
-          loop do
-            batch = fetch_batch
-            break if batch.count < 1
+          while (batch = fetch_batch).any?
             batch.each do |record|
               result << record
             end
@@ -21,7 +21,10 @@ module Processor
 
       def fetch_batch
         @fetcher ||= query.each_slice(batch_size)
+        # TODO get rid of .next enumeration here
         @fetcher.next
+      rescue StopIteration
+        []
       end
 
       def total_records

data/lib/processor/data/solr_pages_processor.rb

@@ -7,7 +7,7 @@ module Processor
         raise NotImplementedError
       end
 
-      def query(requested_page)
+      def query(requested_page, per_page = batch_size)
         raise NotImplementedError
       end
 

data/lib/processor/observer/logger.rb

@@ -8,10 +8,16 @@ module Processor
   module Observer
     class Logger < NullObserver
       def initialize(logger = nil, options = {})
-        @logger_source = logger
+        if logger.is_a? Hash
+          @log_level = logger.fetch :level, ::Logger::INFO
+        else
+          @logger_source = logger
+        end
+
         @messages = options.fetch :messages, nil
         @messages = OpenStruct.new @messages if @messages.is_a? Hash
 
+
         super options
       end
 
@@ -33,11 +39,11 @@ module Processor
         messenger.message messages.finished
       end
 
-      def after_record_error(result, record, exception)
-        logger.error "Error processing #{id_for record}: #{exception}"
+      def before_record_error(record, exception)
+        logger.error "Error processing #{id_for record}: #{exception.inspect}"
       end
 
-      def after_error(result, exception)
+      def before_error(exception)
         logger.fatal "Processing #{processor_name} FAILED: #{exception.backtrace}"
       end
 
@@ -47,7 +53,7 @@ module Processor
             @logger_source.call processor_name
           else
             @logger_source or ::Logger.new(create_log_filename(processor_name)).tap do |logger|
-              logger.level = ::Logger::INFO
+              logger.level = log_level
             end
           end
         end
@@ -55,6 +61,10 @@ module Processor
 
       private
 
+      def log_level
+        @log_level ||= ::Logger::INFO
+      end
+
       def messages
         @messages ||= LoggerMessages.new logger
       end
@@ -85,15 +95,7 @@ module Processor
       end
 
       def id_for record
-        [:uid, :id, :to_token, :token, :to_sym].each do |method|
-          return record.public_send method if record.respond_to? method
-        end
-
-        [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
-          return record[method] if record.key? method
-          return record[method.to_s] if record.key? method.to_s
-        end if record.respond_to?(:key?) && record.respond_to?(:[])
-
+        return processor.record_id record if processor.respond_to? :record_id
         record.to_s.strip
       end
     end

data/lib/processor/observer/null_observer.rb

@@ -7,7 +7,7 @@ module Processor
       attr_reader :processor
 
       def initialize(options = {})
-        @messenger = options.fetch :messenger, Processor::Messenger.new(:info, STDOUT, self.class.name)
+        @messenger = options.fetch :messenger, Processor::Messenger.new(:info, STDERR, self.class.name)
         @processor = options.fetch :processor, nil
       end
 

data/lib/processor/process_runner/threads.rb

@@ -17,7 +17,13 @@ module Processor
               begin
                 thread_data_processor.process(thread_record)
               rescue StandardError => exception
-                thread_data_processor.record_error thread_record, exception
+                command = catch(:command) do
+                  thread_data_processor.record_error thread_record, exception
+                end
+
+                # NOTE: redo can not be moved into a method or block
+                # to break from records loop, use Exception
+                redo if command == :redo
               end
             end
 

data/lib/processor/thread.rb

@@ -2,7 +2,6 @@ require 'processor/runner'
 require 'processor/event_processor'
 require 'processor/process_runner/successive'
 require 'processor/process_runner/threads'
-require 'processor/subroutine/recursion'
 
 module Processor
   class Thread

data/lib/processor/version.rb

@@ -1,3 +1,3 @@
 module Processor
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/lib/processor.rb

@@ -13,6 +13,10 @@ require "processor/observer/null_observer"
 require "processor/process_runner/successive"
 require "processor/process_runner/threads"
 
+require "processor/subroutine/counter"
+require "processor/subroutine/recursion"
+require "processor/subroutine/name"
+
 require "processor/runner"
 require "processor/thread"
 

data/processor.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |gem|
   gem.version       = Processor::VERSION
   gem.authors       = ["Alexander Paramonov"]
   gem.email         = ["alexander.n.paramonov@gmail.com"]
-  gem.summary       = %q{Universal processor for iteration over a collection with threads, logging and post processing}
+  gem.summary       = %q{Universal processor for threaded collection iteration, logging and post processing}
   gem.description   = %q{Processor is a tool that helps to iterate over collection and perform complex actions on a result. It is extremely useful in data migrations, report generation, etc. }
   gem.homepage      = "http://github.com/AlexParamonov/processor"
   gem.license       = "MIT"
@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
-  gem.add_development_dependency "rspec", ">= 2.6"
+  gem.add_development_dependency "rspec", "~> 2.6"
   gem.add_development_dependency "pry" unless Processor::RUNNING_ON_CI
   gem.add_development_dependency "pry-plus" if "ruby" == RUBY_ENGINE && false == Processor::RUNNING_ON_CI
   gem.add_development_dependency "coveralls" if Processor::RUNNING_ON_CI

data/spec/processor/data/batch_processor_spec.rb

@@ -6,26 +6,30 @@ describe Processor::Data::BatchProcessor do
   it "should create and process records by batch" do
     processor = Processor::Data::BatchProcessor.new 2
 
-    watcher = mock
-    5.times do
-      watcher.should_receive(:created).ordered
-      watcher.should_receive(:created).ordered
-      watcher.should_receive(:processed).ordered
-      watcher.should_receive(:processed).ordered
-    end
+    records = (1..10).each_slice(2).map do |first_record, second_record|
+      record1 = mock("record_#{first_record}")
+      record2 = mock("record_#{second_record}")
+
+      record1.should_receive(:created).ordered
+      record2.should_receive(:created).ordered
+
+      record1.should_receive(:processed).ordered
+      record2.should_receive(:processed).ordered
+
+      [ record1, record2 ]
+    end.flatten
 
     query = Enumerator.new do |y|
-      a = 1
-      until a > 10
-        watcher.created
-        y << a
-        a += 1
+      while records.any?
+        record = records.shift
+        record.created
+        y << record
       end
     end
 
     processor.stub(query: query)
     processor.records.each do |record|
-      watcher.processed
+      record.processed
     end
   end
 

data/spec/processor/observer/logger_spec.rb

@@ -9,6 +9,15 @@ describe Processor::Observer::Logger do
   let(:messenger) { ::Logger.new("/dev/null") }
   let(:logger) { ::Logger.new("/dev/null") }
 
+  describe "record_id" do
+    it "uses processor.record_id if possible" do
+      record = double :record
+
+      expect(processor).to receive(:record_id).with(record).and_return 1
+      subject.send(:id_for, record).should eq 1
+    end
+  end
+
   describe "logger" do
     describe "as proc" do
       let(:external_logger) { stub }
@@ -37,6 +46,28 @@ describe Processor::Observer::Logger do
         ::Logger.should_receive(:new).and_return(ruby_logger)
         observer.logger.should eq ruby_logger
       end
+
+      it "sets log level to info by default" do
+        observer = subject
+        ::Logger.any_instance.should_receive(:level=).with(::Logger::INFO)
+        observer.logger
+      end
+
+      it "sets log level to user specified value" do
+        observer = described_class.new level: ::Logger::DEBUG
+        ::Logger.any_instance.should_receive(:level=).with(::Logger::DEBUG)
+        observer.logger
+      end
+    end
+
+    describe "hash" do
+      let(:logger) { { level: ::Logger::DEBUG } }
+
+      it "applies options to default logger" do
+        observer = subject
+        ::Logger.any_instance.should_receive(:level=).with(::Logger::DEBUG)
+        observer.logger
+      end
     end
   end
 

data/spec/processor/process_runner/specs.rb

@@ -14,9 +14,9 @@ shared_examples_for "a records processor" do
   end
 
   describe "exception handling" do
-    describe "processing a record raised StandardError" do
-      let(:records) { 1..3 }
+    let(:records) { 1..3 }
 
+    describe "processing a record raised StandardError" do
       it "should continue processing" do
         processor.should_receive(:process).exactly(3).times.and_raise(StandardError)
         expect { process_runner.call processor }.to_not raise_error
@@ -35,5 +35,16 @@ shared_examples_for "a records processor" do
         expect { process_runner.call processor }.to raise_error(Exception)
       end
     end
+
+    describe "processing a record have requested a command run" do
+      it "should run redo command" do
+        processor.should_receive(:process).with(1).once.and_raise(StandardError)
+        processor.should_receive(:process).with(1).once
+        processor.should_receive(:process).exactly(2).times
+
+        processor.should_receive(:record_error).once.and_throw(:command, :redo)
+        process_runner.call processor
+      end
+    end
   end
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: processor
 version: !ruby/object:Gem::Version
-  version: 2.1.0
-  prerelease: 
+  version: 2.2.0
 platform: ruby
 authors:
 - Alexander Paramonov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-08 00:00:00.000000000 Z
+date: 2014-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,23 +27,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.6'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: pry-plus
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -140,35 +131,27 @@ files:
 homepage: http://github.com/AlexParamonov/processor
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 148591874566411659
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 148591874566411659
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
-summary: Universal processor for iteration over a collection with threads, logging
-  and post processing
+specification_version: 4
+summary: Universal processor for threaded collection iteration, logging and post processing
 test_files:
 - spec/example_spec.rb
 - spec/processor/data/array_processor_spec.rb