processor 0.0.0.beta → 0.0.0.initial

data/lib/processor/version.rb

@@ -1,3 +1,3 @@
 module Processor
-  VERSION = "0.0.0.beta"
+  VERSION = "0.0.0.initial"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: processor
 version: !ruby/object:Gem::Version
-  version: 0.0.0.beta
+  version: 0.0.0.initial
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-02 00:00:00.000000000 Z
+date: 2013-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -67,29 +67,15 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
-- .rspec
 - .rvmrc
 - .travis.yml
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
-- example/example_runner.rb
-- example/migration.rb
-- example/observer/progress_bar.rb
-- example/simple_runner.rb
 - lib/processor.rb
-- lib/processor/data_processor.rb
-- lib/processor/events_registrator.rb
-- lib/processor/observer/logger.rb
-- lib/processor/observer/null_observer.rb
-- lib/processor/thread_runner.rb
 - lib/processor/version.rb
 - processor.gemspec
-- spec/example_spec.rb
-- spec/processor/events_registrator_spec.rb
-- spec/processor/observer/logger_spec.rb
-- spec/processor/thread_runner_spec.rb
 - spec/spec_helper_lite.rb
 homepage: http://github.com/AlexParamonov/processor
 licenses:
@@ -117,8 +103,4 @@ signing_key:
 specification_version: 3
 summary: Process records one by one
 test_files:
-- spec/example_spec.rb
-- spec/processor/events_registrator_spec.rb
-- spec/processor/observer/logger_spec.rb
-- spec/processor/thread_runner_spec.rb
 - spec/spec_helper_lite.rb

data/.rspec

@@ -1,2 +0,0 @@
---colour
---format documentation

data/example/example_runner.rb

@@ -1,43 +0,0 @@
-require 'processor/thread_runner'
-require 'processor/observer/logger'
-
-module Processor
-  module Example
-    class ExampleRunner < ThreadRunner
-      def initialize
-        # Logger could be a lambda
-        # logger = -> name do
-        #   ::Logger.new("log/debug_#{name}_daily.log", "daily").tap do |logger|
-        #     logger.datetime_format = "%H:%M:%S"
-        #     logger.level = ::Logger::DEBUG
-        #   end
-        # end
-
-        # logger could be an instance of Ruby Logger
-        logger = ::Logger.new(STDOUT).tap do |logger|
-          logger.level = ::Logger::DEBUG
-          logger.formatter =  -> _, _, _, msg do
-            "log < #{msg}\n"
-          end
-        end
-
-        # logger could be an instance of Rails Logger
-        # logger = Rails.logger
-        # Or be nil
-        # logger = nil
-        # in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
-
-        # messenger could be an instance of Ruby Logger
-        messenger = ::Logger.new(STDOUT).tap do |logger|
-          logger.formatter =  -> _, _, _, msg do
-            "message > #{msg}\n"
-          end
-          logger.level = ::Logger::INFO
-        end
-
-        logger_observer = Processor::Observer::Logger.new(logger, messenger: messenger)
-        super logger_observer
-      end
-    end
-  end
-end

data/example/migration.rb

@@ -1,29 +0,0 @@
-require 'processor/data_processor'
-
-module Processor
-  module Example
-    class Migration < DataProcessor
-      attr_reader :records
-      def initialize(records)
-        @records = records
-      end
-
-      def done?(records)
-        records.count < 1
-      end
-
-      def process(record)
-        record.do_something
-        "OK"
-      end
-
-      def fetch_records
-        records.shift(2)
-      end
-
-      def total_records
-        records.count
-      end
-    end
-  end
-end

data/example/observer/progress_bar.rb

@@ -1,26 +0,0 @@
-require 'processor/observer/null_observer'
-require 'progressbar'
-
-module Processor
-  module Example
-    module Observer
-      class ProgressBar < Processor::Observer::NullObserver
-        def processing_started
-          @progress_bar = ::ProgressBar.new("Records", processor.total_records)
-          messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
-        end
-
-        def before_record_processing(record)
-          progress_bar.inc
-        end
-
-        def processing_finished
-          progress_bar.finish
-        end
-
-        private
-        attr_reader :progress_bar
-      end
-    end
-  end
-end

data/example/simple_runner.rb

@@ -1,12 +0,0 @@
-require 'processor/thread_runner'
-require 'processor/observer/logger'
-
-module Processor
-  module Example
-    class SimpleRunner < ThreadRunner
-      def initialize
-        super Processor::Observer::Logger.new
-      end
-    end
-  end
-end

data/lib/processor/data_processor.rb

@@ -1,28 +0,0 @@
-module Processor
-  class DataProcessor
-    def done?(records)
-      records.count < 1
-    end
-
-    def process(record)
-      raise NotImplementedError
-    end
-
-    def fetch_records
-      raise NotImplementedError
-    end
-
-    def total_records
-      raise NotImplementedError
-    end
-
-    def name
-      # underscore a class name
-      self.class.name.to_s.
-        gsub(/::/, '_').
-        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-        gsub(/([a-z\d])([A-Z])/,'\1_\2').
-        downcase
-    end
-  end
-end

data/lib/processor/events_registrator.rb

@@ -1,16 +0,0 @@
-module Processor
-  class EventsRegistrator
-    def initialize(observers)
-      @observers = observers
-    end
-
-    def register(event, *data)
-      observers.each do |observer|
-        observer.update event, *data
-      end
-    end
-
-    private
-    attr_reader :observers
-  end
-end

data/lib/processor/observer/logger.rb

@@ -1,98 +0,0 @@
-require_relative 'null_observer'
-require 'logger'
-
-module Processor
-  module Observer
-    class Logger < NullObserver
-      def initialize(logger = nil, options = {})
-        @logger_source = logger
-        super options
-      end
-
-      def processing_started(processor)
-        initialize_logger(processor)
-
-        message = "Processing of #{processor.name} started."
-        logger.info message
-        messenger.info message
-
-        message = <<-MESSAGE.gsub(/^\s+/, '')
-          Proggress will be saved to the log file. Run
-          tail -f #{log_file_name}
-          to see log in realtime
-          MESSAGE
-          messenger.info message if use_log_file?
-      end
-
-      def before_record_processing(record)
-        message = "Record #{id_for record} is going to be processed"
-        logger.debug message
-        messenger.debug message
-      end
-
-      def after_record_processing(record, result)
-        message = "Successfully processed #{id_for record}: #{result}"
-        logger.info message
-        messenger.debug message
-      end
-
-      def processing_finished(processor)
-        message = "Processing of #{processor.name} finished."
-        logger.info message
-        messenger.info message
-        messenger.info "Log file saved to #{log_file_name}" if use_log_file?
-      end
-
-      def record_processing_error(record, exception)
-        message = "Error processing #{id_for record}: #{exception}"
-        logger.error message
-        messenger.error message
-      end
-
-      def processing_error(processor, exception)
-        message = "Processing #{processor.name} failed: #{exception}"
-        logger.fatal message
-        messenger.fatal message
-      end
-
-      private
-      attr_reader :logger, :log_file_name
-
-      def initialize_logger(processor)
-        @logger =
-          if @logger_source.is_a? Proc
-            @logger_source.call processor.name
-          else
-            @logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
-              logger.level = ::Logger::INFO
-            end
-          end
-      end
-
-      def create_log_filename(processor_name)
-        @log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
-      end
-
-      def use_log_file?
-        not log_file_name.nil?
-      end
-
-      def current_time_string
-        Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
-      end
-
-      def id_for record
-        [:uid, :id, :to_token, :token, :to_sym].each do |method|
-          return record.public_send method if record.respond_to? method
-        end
-
-        [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
-          return record[method] if record.key? method
-          return record[method.to_s] if record.key? method.to_s
-        end if record.respond_to?(:key?) && record.respond_to?(:[])
-
-        record.to_s
-      end
-    end
-  end
-end

data/lib/processor/observer/null_observer.rb

@@ -1,23 +0,0 @@
-require 'logger'
-
-module Processor
-  module Observer
-    class NullObserver
-      def initialize(options = {})
-        @messenger = options.fetch :messenger do
-          ::Logger.new(STDOUT).tap do |logger|
-            logger.formatter =  -> _, _, _, msg do
-              "> #{msg}\n"
-            end
-          end
-        end
-      end
-
-      def method_missing(*); end
-      alias_method :update, :send
-
-      private
-      attr_reader :messenger
-    end
-  end
-end

data/lib/processor/thread_runner.rb

@@ -1,59 +0,0 @@
-require_relative "events_registrator"
-
-module Processor
-  class ThreadRunner
-    def initialize(*observers)
-      @observers = observers
-    end
-
-    # This method is thread-safe. But not observers.
-    # Consider creating new runner for a thread or use thread safe observers
-    def run(processor)
-      events = events_registrator
-      events.register :processing_started, processor
-
-      records_ran = 0
-      until processor.done?(records = processor.fetch_records)
-        threads = []
-        begin
-          records.each do |record|
-            recursion_preventer processor do
-              records_ran += 1
-            end
-
-            threads << Thread.new(processor, record) do |thread_data_processor, thread_record|
-              begin
-                events.register :before_record_processing, thread_record
-
-                result = thread_data_processor.process(thread_record)
-
-                events.register :after_record_processing, thread_record, result
-              rescue RuntimeError => exception
-                events.register :record_processing_error, thread_record, exception
-              end
-            end
-          end
-        ensure # join already created threads even if recursion was detected
-          threads.each(&:join)
-        end
-      end
-
-      events.register :processing_finished, processor
-    rescue Exception => exception
-      events.register :processing_error, processor, exception
-      raise exception
-    end
-
-    private
-    attr_reader :observers
-
-    def recursion_preventer(processor)
-      counter = yield
-      raise Exception, "Processing fall into recursion. Check logs." if counter > (processor.total_records * 1.1).round + 10
-    end
-
-    def events_registrator
-      EventsRegistrator.new observers
-    end
-  end
-end

data/spec/example_spec.rb

@@ -1,24 +0,0 @@
-require 'spec_helper_lite'
-require_relative '../example/example_runner'
-require_relative '../example/migration'
-require 'fileutils'
-
-describe "Example" do
-  before(:each) do
-    records = %w[item1 item2 item3 item4 item5]
-    records.each do |record|
-      record.should_receive(:do_something).once
-    end
-    @migration = Processor::Example::Migration.new records
-  end
-
-  it "should use logger and messenger" do
-    migration_runner = Processor::Example::ExampleRunner.new
-    migration_runner.run @migration
-  end
-
-  it "should run without configuration" do
-    runner = Processor::ThreadRunner.new
-    runner.run @migration
-  end
-end

data/spec/processor/events_registrator_spec.rb

@@ -1,15 +0,0 @@
-require 'spec_helper_lite'
-require 'processor/events_registrator'
-
-describe Processor::EventsRegistrator do
-  subject { Processor::EventsRegistrator }
-
-  it "should broadcast events to all observers" do
-    observers = 3.times.map do
-      stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
-    end
-
-    events = subject.new(observers)
-    events.register :test_event
-  end
-end

data/spec/processor/observer/logger_spec.rb

@@ -1,32 +0,0 @@
-require 'spec_helper_lite'
-require 'processor/observer/logger'
-
-describe Processor::Observer::Logger do
-  let(:processor) { stub.as_null_object }
-  let(:no_messages) { ::Logger.new("/dev/null") }
-
-  subject { Processor::Observer::Logger }
-    it "accepts logger builder as parameter" do
-      external_logger = mock
-      logger_observer = subject.new -> name { external_logger }, messenger: no_messages
-
-      external_logger.should_receive(:info)
-      logger_observer.processing_started processor
-    end
-
-    it "accepts logger as parameter" do
-      external_logger = mock
-      logger_observer = subject.new external_logger, messenger: no_messages
-
-      external_logger.should_receive(:info)
-      logger_observer.processing_started processor
-    end
-
-    it "use ruby Logger if no external logger provided" do
-      logger_observer = subject.new nil, messenger: no_messages
-
-      Logger.should_receive(:new).and_return(stub.as_null_object)
-      logger_observer.processing_started processor
-    end
-end
-

data/spec/processor/thread_runner_spec.rb

@@ -1,134 +0,0 @@
-require 'spec_helper_lite'
-require 'processor/thread_runner'
-
-processor = Class.new do
-  def done?(records)
-    records.count < 1
-  end
-
-  def total_records
-    10
-  end
-end.new
-
-describe Processor::ThreadRunner do
-  let(:runner) { Processor::ThreadRunner.new }
-  let(:events_registrator) { stub.as_null_object }
-  before(:each) do
-    runner.stub(events_registrator: events_registrator)
-  end
-
-  it "should fetch records from processor till it'll be done" do
-    processor.stub(:done?).and_return(false, false, true)
-    processor.should_receive(:fetch_records).exactly(3).times.and_return([])
-    runner.run processor
-  end
-
-  it "should send each found record to processor" do
-    records = [mock(:one), mock(:two), mock(:three)]
-    records.each { |record| processor.should_receive(:process).with(record) }
-
-    processor.stub(:fetch_records).and_return(records, [])
-    runner.run processor
-  end
-
-  describe "exception handling" do
-    let(:record) { stub }
-    before(:each) do
-      processor.stub(:fetch_records).and_return([record], [record], [])
-    end
-
-    describe "processing a record raised RuntimeError" do
-      it "should continue processing" do
-        processor.should_receive(:process).twice.and_raise(RuntimeError)
-        expect { runner.run processor }.to_not raise_error
-      end
-
-      it "should register a record_processing_error event" do
-        event_registered = false
-        events_registrator.should_receive(:register) do |event_name, failed_record, exception|
-          next if event_name != :record_processing_error
-          event_name.should eq :record_processing_error
-          failed_record.should eq record
-          exception.should be_a RuntimeError
-          event_registered = true
-        end.any_number_of_times
-
-        processor.stub(:process).and_raise(RuntimeError)
-
-        begin
-          runner.run processor
-        rescue Exception; end
-        event_registered.should be_true
-      end
-    end
-
-    describe "processing a record raised Exception" do
-      it "should break processing and rerise Exception" do
-        custom_exception = Class.new Exception
-        processor.should_receive(:process).once.and_raise(custom_exception)
-        expect { runner.run processor }.to raise_error(custom_exception)
-      end
-
-      it "should register a processing_error event" do
-        event_registered = false
-        events_registrator.should_receive(:register) do |event_name, current_processor, exception|
-          next if event_name != :processing_error
-          event_name.should eq :processing_error
-          current_processor.should eq processor
-          exception.should be_a Exception
-          event_registered = true
-        end.any_number_of_times
-
-        processor.stub(:process).and_raise(Exception)
-
-        begin
-          runner.run processor
-        rescue Exception; end
-        event_registered.should be_true
-      end
-    end
-
-    describe "fetching records raised" do
-      it "should break processing and rerise Exception" do
-        custom_exception = Class.new RuntimeError
-        processor.stub(:fetch_records).and_raise(custom_exception)
-        processor.should_not_receive(:process)
-        expect { runner.run processor }.to raise_error(custom_exception)
-      end
-
-      it "should register a processing_error" do
-        event_registered = false
-        events_registrator.should_receive(:register) do |event_name, processor, exception|
-          next if event_name != :processing_error
-          event_name.should eq :processing_error
-          exception.should be_a RuntimeError
-          event_registered = true
-        end.any_number_of_times
-
-        processor.stub(:fetch_records).and_raise(RuntimeError)
-
-        begin
-          runner.run processor
-        rescue Exception; end
-        event_registered.should be_true
-      end
-    end
-  end
-
-  describe "recursion" do
-    it "should not allow infinit recursion" do
-      processor.stub(:fetch_records).and_return([:one, :two])
-      processor.should_receive(:process).at_most(100).times
-      expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
-    end
-
-    it "should have 10% + 10 rerurns window" do
-      processor.stub(total_records: 100)
-      processor.stub(:fetch_records).and_return([:one, :two])
-      processor.should_receive(:process).exactly(120).times
-      expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
-    end
-  end
-end
-