processor 0.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,16 @@
1
+ *.gem
2
+ *.rbc
3
+ /.bundle
4
+
5
+ Gemfile.lock
6
+ /pkg
7
+ /tmp
8
+ /tags
9
+
10
+ ## generic files to ignore
11
+ *~
12
+ *.lock
13
+ *.DS_Store
14
+ *.swp
15
+ *.out
16
+ *.swo
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --format documentation
data/.rvmrc ADDED
@@ -0,0 +1,38 @@
1
+ #!/usr/bin/env bash
2
+
3
+ # This is an RVM Project .rvmrc file, used to automatically load the ruby
4
+ # development environment upon cd'ing into the directory
5
+
6
+ # First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
7
+ # Only full ruby name is supported here, for short names use:
8
+ # echo "rvm use 1.9.3" > .rvmrc
9
+ environment_id="ruby-1.9.3@processor"
10
+
11
+ # Uncomment the following lines if you want to verify rvm version per project
12
+ # rvmrc_rvm_version="1.15.8 (stable)" # 1.10.1 seams as a safe start
13
+ # eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
14
+ # echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
15
+ # return 1
16
+ # }
17
+
18
+ # First we attempt to load the desired environment directly from the environment
19
+ # file. This is very fast and efficient compared to running through the entire
20
+ # CLI and selector. If you want feedback on which environment was used then
21
+ # insert the word 'use' after --create as this triggers verbose mode.
22
+ if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
23
+ && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
24
+ then
25
+ \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
26
+ [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
27
+ \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
28
+ if [[ $- == *i* ]] # check for interactive shells
29
+ then echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green
30
+ else echo "Using: $GEM_HOME" # don't use colors in non-interactive shells
31
+ fi
32
+ else
33
+ # If the environment file has not yet been created, use the RVM CLI to select.
34
+ rvm --create use "$environment_id" || {
35
+ echo "Failed to create RVM environment '${environment_id}'."
36
+ return 1
37
+ }
38
+ fi
data/.travis.yml ADDED
@@ -0,0 +1,15 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - ruby-head
5
+ - rbx-19mode
6
+
7
+ matrix:
8
+ allow_failures:
9
+ - rvm: ruby-head
10
+ - rvm: rbx-19mode
11
+
12
+ before_install: gem install bundler --pre
13
+
14
+ script:
15
+ - "bundle exec rake ci:all"
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in extentions.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Alexander Paramonov
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,64 @@
1
+ Processor
2
+ ==========
3
+ [![Build Status](https://travis-ci.org/AlexParamonov/processor.png?branch=master)](http://travis-ci.org/AlexParamonov/processor)
4
+ [![Gemnasium Build Status](https://gemnasium.com/AlexParamonov/processor.png)](http://gemnasium.com/AlexParamonov/processor)
5
+
6
+ Universal processor for data migration
7
+
8
+ Contents
9
+ ---------
10
+ 1. Installation
11
+ 1. Contributing
12
+ 1. Requirements
13
+ 1. Compatibility
14
+ 1. Copyright
15
+
16
+ Installation
17
+ ------------
18
+ Add this line to your application's Gemfile:
19
+
20
+ ``` ruby
21
+ gem 'processor'
22
+ ```
23
+
24
+ And then execute:
25
+
26
+ ``` sh
27
+ bundle
28
+ ```
29
+
30
+ Or install it yourself as:
31
+
32
+ ``` sh
33
+ gem install processor
34
+ ```
35
+
36
+ Requirements
37
+ ------------
38
+ none
39
+
40
+ rspec2 for testing
41
+
42
+ Compatibility
43
+ -------------
44
+ tested with Ruby
45
+
46
+ * 1.9.3
47
+ * jruby-19mode
48
+ * rbx-19mode
49
+ * ruby-head
50
+
51
+ see [build history](http://travis-ci.org/#!/AlexParamonov/processor/builds)
52
+
53
+ Contributing
54
+ -------------
55
+ 1. Fork repository [AlexParamonov/processor](https://github.com/AlexParamonov/processor)
56
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
57
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
58
+ 4. Push to the branch (`git push origin my-new-feature`)
59
+ 5. Create new Pull Request
60
+
61
+ Copyright
62
+ ---------
63
+ Copyright © 2013 Alexander Paramonov.
64
+ Released under the MIT License. See the LICENSE file for further details.
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rspec/core/rake_task'
4
+ namespace :ci do
5
+ desc "Run tests"
6
+ RSpec::Core::RakeTask.new('all') do |t|
7
+ t.rspec_opts = '-fprogress'
8
+ t.verbose = true
9
+ end
10
+ end
@@ -0,0 +1,43 @@
1
+ require 'processor/thread_runner'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class ExampleRunner < ThreadRunner
7
+ def initialize
8
+ # Logger could be a lambda
9
+ # logger = -> name do
10
+ # ::Logger.new("log/debug_#{name}_daily.log", "daily").tap do |logger|
11
+ # logger.datetime_format = "%H:%M:%S"
12
+ # logger.level = ::Logger::DEBUG
13
+ # end
14
+ # end
15
+
16
+ # logger could be an instance of Ruby Logger
17
+ logger = ::Logger.new(STDOUT).tap do |logger|
18
+ logger.level = ::Logger::DEBUG
19
+ logger.formatter = -> _, _, _, msg do
20
+ "log < #{msg}\n"
21
+ end
22
+ end
23
+
24
+ # logger could be an instance of Rails Logger
25
+ # logger = Rails.logger
26
+ # Or be nil
27
+ # logger = nil
28
+ # in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
29
+
30
+ # messenger could be an instance of Ruby Logger
31
+ messenger = ::Logger.new(STDOUT).tap do |logger|
32
+ logger.formatter = -> _, _, _, msg do
33
+ "message > #{msg}\n"
34
+ end
35
+ logger.level = ::Logger::INFO
36
+ end
37
+
38
+ logger_observer = Processor::Observer::Logger.new(logger, messenger: messenger)
39
+ super logger_observer
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,29 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class Migration < DataProcessor
6
+ attr_reader :records
7
+ def initialize(records)
8
+ @records = records
9
+ end
10
+
11
+ def done?(records)
12
+ records.count < 1
13
+ end
14
+
15
+ def process(record)
16
+ record.do_something
17
+ "OK"
18
+ end
19
+
20
+ def fetch_records
21
+ records.shift(2)
22
+ end
23
+
24
+ def total_records
25
+ records.count
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,26 @@
1
+ require 'processor/observer/null_observer'
2
+ require 'progressbar'
3
+
4
+ module Processor
5
+ module Example
6
+ module Observer
7
+ class ProgressBar < Processor::Observer::NullObserver
8
+ def processing_started
9
+ @progress_bar = ::ProgressBar.new("Records", processor.total_records)
10
+ messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
11
+ end
12
+
13
+ def before_record_processing(record)
14
+ progress_bar.inc
15
+ end
16
+
17
+ def processing_finished
18
+ progress_bar.finish
19
+ end
20
+
21
+ private
22
+ attr_reader :progress_bar
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,12 @@
1
+ require 'processor/thread_runner'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class SimpleRunner < ThreadRunner
7
+ def initialize
8
+ super Processor::Observer::Logger.new
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,28 @@
1
+ module Processor
2
+ class DataProcessor
3
+ def done?(records)
4
+ records.count < 1
5
+ end
6
+
7
+ def process(record)
8
+ raise NotImplementedError
9
+ end
10
+
11
+ def fetch_records
12
+ raise NotImplementedError
13
+ end
14
+
15
+ def total_records
16
+ raise NotImplementedError
17
+ end
18
+
19
+ def name
20
+ # underscore a class name
21
+ self.class.name.to_s.
22
+ gsub(/::/, '_').
23
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
24
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
25
+ downcase
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,16 @@
1
+ module Processor
2
+ class EventsRegistrator
3
+ def initialize(observers)
4
+ @observers = observers
5
+ end
6
+
7
+ def register(event, *data)
8
+ observers.each do |observer|
9
+ observer.update event, *data
10
+ end
11
+ end
12
+
13
+ private
14
+ attr_reader :observers
15
+ end
16
+ end
@@ -0,0 +1,98 @@
1
+ require_relative 'null_observer'
2
+ require 'logger'
3
+
4
+ module Processor
5
+ module Observer
6
+ class Logger < NullObserver
7
+ def initialize(logger = nil, options = {})
8
+ @logger_source = logger
9
+ super options
10
+ end
11
+
12
+ def processing_started(processor)
13
+ initialize_logger(processor)
14
+
15
+ message = "Processing of #{processor.name} started."
16
+ logger.info message
17
+ messenger.info message
18
+
19
+ message = <<-MESSAGE.gsub(/^\s+/, '')
20
+ Proggress will be saved to the log file. Run
21
+ tail -f #{log_file_name}
22
+ to see log in realtime
23
+ MESSAGE
24
+ messenger.info message if use_log_file?
25
+ end
26
+
27
+ def before_record_processing(record)
28
+ message = "Record #{id_for record} is going to be processed"
29
+ logger.debug message
30
+ messenger.debug message
31
+ end
32
+
33
+ def after_record_processing(record, result)
34
+ message = "Successfully processed #{id_for record}: #{result}"
35
+ logger.info message
36
+ messenger.debug message
37
+ end
38
+
39
+ def processing_finished(processor)
40
+ message = "Processing of #{processor.name} finished."
41
+ logger.info message
42
+ messenger.info message
43
+ messenger.info "Log file saved to #{log_file_name}" if use_log_file?
44
+ end
45
+
46
+ def record_processing_error(record, exception)
47
+ message = "Error processing #{id_for record}: #{exception}"
48
+ logger.error message
49
+ messenger.error message
50
+ end
51
+
52
+ def processing_error(processor, exception)
53
+ message = "Processing #{processor.name} failed: #{exception}"
54
+ logger.fatal message
55
+ messenger.fatal message
56
+ end
57
+
58
+ private
59
+ attr_reader :logger, :log_file_name
60
+
61
+ def initialize_logger(processor)
62
+ @logger =
63
+ if @logger_source.is_a? Proc
64
+ @logger_source.call processor.name
65
+ else
66
+ @logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
67
+ logger.level = ::Logger::INFO
68
+ end
69
+ end
70
+ end
71
+
72
+ def create_log_filename(processor_name)
73
+ @log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
74
+ end
75
+
76
+ def use_log_file?
77
+ not log_file_name.nil?
78
+ end
79
+
80
+ def current_time_string
81
+ Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
82
+ end
83
+
84
+ def id_for record
85
+ [:uid, :id, :to_token, :token, :to_sym].each do |method|
86
+ return record.public_send method if record.respond_to? method
87
+ end
88
+
89
+ [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
90
+ return record[method] if record.key? method
91
+ return record[method.to_s] if record.key? method.to_s
92
+ end if record.respond_to?(:key?) && record.respond_to?(:[])
93
+
94
+ record.to_s
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,23 @@
1
+ require 'logger'
2
+
3
+ module Processor
4
+ module Observer
5
+ class NullObserver
6
+ def initialize(options = {})
7
+ @messenger = options.fetch :messenger do
8
+ ::Logger.new(STDOUT).tap do |logger|
9
+ logger.formatter = -> _, _, _, msg do
10
+ "> #{msg}\n"
11
+ end
12
+ end
13
+ end
14
+ end
15
+
16
+ def method_missing(*); end
17
+ alias_method :update, :send
18
+
19
+ private
20
+ attr_reader :messenger
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,59 @@
1
+ require_relative "events_registrator"
2
+
3
+ module Processor
4
+ class ThreadRunner
5
+ def initialize(*observers)
6
+ @observers = observers
7
+ end
8
+
9
+ # This method is thread-safe. But not observers.
10
+ # Consider creating new runner for a thread or use thread safe observers
11
+ def run(processor)
12
+ events = events_registrator
13
+ events.register :processing_started, processor
14
+
15
+ records_ran = 0
16
+ until processor.done?(records = processor.fetch_records)
17
+ threads = []
18
+ begin
19
+ records.each do |record|
20
+ recursion_preventer processor do
21
+ records_ran += 1
22
+ end
23
+
24
+ threads << Thread.new(processor, record) do |thread_data_processor, thread_record|
25
+ begin
26
+ events.register :before_record_processing, thread_record
27
+
28
+ result = thread_data_processor.process(thread_record)
29
+
30
+ events.register :after_record_processing, thread_record, result
31
+ rescue RuntimeError => exception
32
+ events.register :record_processing_error, thread_record, exception
33
+ end
34
+ end
35
+ end
36
+ ensure # join already created threads even if recursion was detected
37
+ threads.each(&:join)
38
+ end
39
+ end
40
+
41
+ events.register :processing_finished, processor
42
+ rescue Exception => exception
43
+ events.register :processing_error, processor, exception
44
+ raise exception
45
+ end
46
+
47
+ private
48
+ attr_reader :observers
49
+
50
+ def recursion_preventer(processor)
51
+ counter = yield
52
+ raise Exception, "Processing fall into recursion. Check logs." if counter > (processor.total_records * 1.1).round + 10
53
+ end
54
+
55
+ def events_registrator
56
+ EventsRegistrator.new observers
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,3 @@
1
+ module Processor
2
+ VERSION = "0.0.0.beta"
3
+ end
data/lib/processor.rb ADDED
@@ -0,0 +1,4 @@
1
+ require "processor/version"
2
+
3
+ module Processor
4
+ end
data/processor.gemspec ADDED
@@ -0,0 +1,25 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'processor/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = "processor"
8
+ gem.version = Processor::VERSION
9
+ gem.authors = ["Alexander Paramonov"]
10
+ gem.email = ["alexander.n.paramonov@gmail.com"]
11
+ gem.summary = %q{Process records one by one}
12
+ gem.description = %q{Universal processor for data migration}
13
+ gem.homepage = "http://github.com/AlexParamonov/processor"
14
+ gem.license = "MIT"
15
+
16
+ gem.files = `git ls-files`.split($/)
17
+ gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
19
+ gem.require_paths = ["lib"]
20
+
21
+ gem.add_development_dependency "rake"
22
+ gem.add_development_dependency "rspec", ">= 2.6"
23
+ gem.add_development_dependency "pry"
24
+ end
25
+
@@ -0,0 +1,24 @@
1
+ require 'spec_helper_lite'
2
+ require_relative '../example/example_runner'
3
+ require_relative '../example/migration'
4
+ require 'fileutils'
5
+
6
+ describe "Example" do
7
+ before(:each) do
8
+ records = %w[item1 item2 item3 item4 item5]
9
+ records.each do |record|
10
+ record.should_receive(:do_something).once
11
+ end
12
+ @migration = Processor::Example::Migration.new records
13
+ end
14
+
15
+ it "should use logger and messenger" do
16
+ migration_runner = Processor::Example::ExampleRunner.new
17
+ migration_runner.run @migration
18
+ end
19
+
20
+ it "should run without configuration" do
21
+ runner = Processor::ThreadRunner.new
22
+ runner.run @migration
23
+ end
24
+ end
@@ -0,0 +1,15 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/events_registrator'
3
+
4
+ describe Processor::EventsRegistrator do
5
+ subject { Processor::EventsRegistrator }
6
+
7
+ it "should broadcast events to all observers" do
8
+ observers = 3.times.map do
9
+ stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
10
+ end
11
+
12
+ events = subject.new(observers)
13
+ events.register :test_event
14
+ end
15
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/observer/logger'
3
+
4
+ describe Processor::Observer::Logger do
5
+ let(:processor) { stub.as_null_object }
6
+ let(:no_messages) { ::Logger.new("/dev/null") }
7
+
8
+ subject { Processor::Observer::Logger }
9
+ it "accepts logger builder as parameter" do
10
+ external_logger = mock
11
+ logger_observer = subject.new -> name { external_logger }, messenger: no_messages
12
+
13
+ external_logger.should_receive(:info)
14
+ logger_observer.processing_started processor
15
+ end
16
+
17
+ it "accepts logger as parameter" do
18
+ external_logger = mock
19
+ logger_observer = subject.new external_logger, messenger: no_messages
20
+
21
+ external_logger.should_receive(:info)
22
+ logger_observer.processing_started processor
23
+ end
24
+
25
+ it "use ruby Logger if no external logger provided" do
26
+ logger_observer = subject.new nil, messenger: no_messages
27
+
28
+ Logger.should_receive(:new).and_return(stub.as_null_object)
29
+ logger_observer.processing_started processor
30
+ end
31
+ end
32
+
@@ -0,0 +1,134 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/thread_runner'
3
+
4
+ processor = Class.new do
5
+ def done?(records)
6
+ records.count < 1
7
+ end
8
+
9
+ def total_records
10
+ 10
11
+ end
12
+ end.new
13
+
14
+ describe Processor::ThreadRunner do
15
+ let(:runner) { Processor::ThreadRunner.new }
16
+ let(:events_registrator) { stub.as_null_object }
17
+ before(:each) do
18
+ runner.stub(events_registrator: events_registrator)
19
+ end
20
+
21
+ it "should fetch records from processor till it'll be done" do
22
+ processor.stub(:done?).and_return(false, false, true)
23
+ processor.should_receive(:fetch_records).exactly(3).times.and_return([])
24
+ runner.run processor
25
+ end
26
+
27
+ it "should send each found record to processor" do
28
+ records = [mock(:one), mock(:two), mock(:three)]
29
+ records.each { |record| processor.should_receive(:process).with(record) }
30
+
31
+ processor.stub(:fetch_records).and_return(records, [])
32
+ runner.run processor
33
+ end
34
+
35
+ describe "exception handling" do
36
+ let(:record) { stub }
37
+ before(:each) do
38
+ processor.stub(:fetch_records).and_return([record], [record], [])
39
+ end
40
+
41
+ describe "processing a record raised RuntimeError" do
42
+ it "should continue processing" do
43
+ processor.should_receive(:process).twice.and_raise(RuntimeError)
44
+ expect { runner.run processor }.to_not raise_error
45
+ end
46
+
47
+ it "should register a record_processing_error event" do
48
+ event_registered = false
49
+ events_registrator.should_receive(:register) do |event_name, failed_record, exception|
50
+ next if event_name != :record_processing_error
51
+ event_name.should eq :record_processing_error
52
+ failed_record.should eq record
53
+ exception.should be_a RuntimeError
54
+ event_registered = true
55
+ end.any_number_of_times
56
+
57
+ processor.stub(:process).and_raise(RuntimeError)
58
+
59
+ begin
60
+ runner.run processor
61
+ rescue Exception; end
62
+ event_registered.should be_true
63
+ end
64
+ end
65
+
66
+ describe "processing a record raised Exception" do
67
+ it "should break processing and rerise Exception" do
68
+ custom_exception = Class.new Exception
69
+ processor.should_receive(:process).once.and_raise(custom_exception)
70
+ expect { runner.run processor }.to raise_error(custom_exception)
71
+ end
72
+
73
+ it "should register a processing_error event" do
74
+ event_registered = false
75
+ events_registrator.should_receive(:register) do |event_name, current_processor, exception|
76
+ next if event_name != :processing_error
77
+ event_name.should eq :processing_error
78
+ current_processor.should eq processor
79
+ exception.should be_a Exception
80
+ event_registered = true
81
+ end.any_number_of_times
82
+
83
+ processor.stub(:process).and_raise(Exception)
84
+
85
+ begin
86
+ runner.run processor
87
+ rescue Exception; end
88
+ event_registered.should be_true
89
+ end
90
+ end
91
+
92
+ describe "fetching records raised" do
93
+ it "should break processing and rerise Exception" do
94
+ custom_exception = Class.new RuntimeError
95
+ processor.stub(:fetch_records).and_raise(custom_exception)
96
+ processor.should_not_receive(:process)
97
+ expect { runner.run processor }.to raise_error(custom_exception)
98
+ end
99
+
100
+ it "should register a processing_error" do
101
+ event_registered = false
102
+ events_registrator.should_receive(:register) do |event_name, processor, exception|
103
+ next if event_name != :processing_error
104
+ event_name.should eq :processing_error
105
+ exception.should be_a RuntimeError
106
+ event_registered = true
107
+ end.any_number_of_times
108
+
109
+ processor.stub(:fetch_records).and_raise(RuntimeError)
110
+
111
+ begin
112
+ runner.run processor
113
+ rescue Exception; end
114
+ event_registered.should be_true
115
+ end
116
+ end
117
+ end
118
+
119
+ describe "recursion" do
120
+ it "should not allow infinit recursion" do
121
+ processor.stub(:fetch_records).and_return([:one, :two])
122
+ processor.should_receive(:process).at_most(100).times
123
+ expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
124
+ end
125
+
126
+ it "should have 10% + 10 rerurns window" do
127
+ processor.stub(total_records: 100)
128
+ processor.stub(:fetch_records).and_return([:one, :two])
129
+ processor.should_receive(:process).exactly(120).times
130
+ expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
131
+ end
132
+ end
133
+ end
134
+
@@ -0,0 +1,6 @@
1
+ require 'pry'
2
+
3
+ ENV['RAILS_ENV'] ||= 'test'
4
+ require "rspec"
5
+
6
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
metadata ADDED
@@ -0,0 +1,124 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: processor
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.0.beta
5
+ prerelease: 6
6
+ platform: ruby
7
+ authors:
8
+ - Alexander Paramonov
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-05-02 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: rake
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: rspec
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '2.6'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '2.6'
46
+ - !ruby/object:Gem::Dependency
47
+ name: pry
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ description: Universal processor for data migration
63
+ email:
64
+ - alexander.n.paramonov@gmail.com
65
+ executables: []
66
+ extensions: []
67
+ extra_rdoc_files: []
68
+ files:
69
+ - .gitignore
70
+ - .rspec
71
+ - .rvmrc
72
+ - .travis.yml
73
+ - Gemfile
74
+ - LICENSE
75
+ - README.md
76
+ - Rakefile
77
+ - example/example_runner.rb
78
+ - example/migration.rb
79
+ - example/observer/progress_bar.rb
80
+ - example/simple_runner.rb
81
+ - lib/processor.rb
82
+ - lib/processor/data_processor.rb
83
+ - lib/processor/events_registrator.rb
84
+ - lib/processor/observer/logger.rb
85
+ - lib/processor/observer/null_observer.rb
86
+ - lib/processor/thread_runner.rb
87
+ - lib/processor/version.rb
88
+ - processor.gemspec
89
+ - spec/example_spec.rb
90
+ - spec/processor/events_registrator_spec.rb
91
+ - spec/processor/observer/logger_spec.rb
92
+ - spec/processor/thread_runner_spec.rb
93
+ - spec/spec_helper_lite.rb
94
+ homepage: http://github.com/AlexParamonov/processor
95
+ licenses:
96
+ - MIT
97
+ post_install_message:
98
+ rdoc_options: []
99
+ require_paths:
100
+ - lib
101
+ required_ruby_version: !ruby/object:Gem::Requirement
102
+ none: false
103
+ requirements:
104
+ - - ! '>='
105
+ - !ruby/object:Gem::Version
106
+ version: '0'
107
+ required_rubygems_version: !ruby/object:Gem::Requirement
108
+ none: false
109
+ requirements:
110
+ - - ! '>'
111
+ - !ruby/object:Gem::Version
112
+ version: 1.3.1
113
+ requirements: []
114
+ rubyforge_project:
115
+ rubygems_version: 1.8.25
116
+ signing_key:
117
+ specification_version: 3
118
+ summary: Process records one by one
119
+ test_files:
120
+ - spec/example_spec.rb
121
+ - spec/processor/events_registrator_spec.rb
122
+ - spec/processor/observer/logger_spec.rb
123
+ - spec/processor/thread_runner_spec.rb
124
+ - spec/spec_helper_lite.rb