processor 0.0.0.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,16 @@
1
+ *.gem
2
+ *.rbc
3
+ /.bundle
4
+
5
+ Gemfile.lock
6
+ /pkg
7
+ /tmp
8
+ /tags
9
+
10
+ ## generic files to ignore
11
+ *~
12
+ *.lock
13
+ *.DS_Store
14
+ *.swp
15
+ *.out
16
+ *.swo
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --format documentation
data/.rvmrc ADDED
@@ -0,0 +1,38 @@
1
+ #!/usr/bin/env bash
2
+
3
+ # This is an RVM Project .rvmrc file, used to automatically load the ruby
4
+ # development environment upon cd'ing into the directory
5
+
6
+ # First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
7
+ # Only full ruby name is supported here, for short names use:
8
+ # echo "rvm use 1.9.3" > .rvmrc
9
+ environment_id="ruby-1.9.3@processor"
10
+
11
+ # Uncomment the following lines if you want to verify rvm version per project
12
+ # rvmrc_rvm_version="1.15.8 (stable)" # 1.10.1 seams as a safe start
13
+ # eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
14
+ # echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
15
+ # return 1
16
+ # }
17
+
18
+ # First we attempt to load the desired environment directly from the environment
19
+ # file. This is very fast and efficient compared to running through the entire
20
+ # CLI and selector. If you want feedback on which environment was used then
21
+ # insert the word 'use' after --create as this triggers verbose mode.
22
+ if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
23
+ && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
24
+ then
25
+ \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
26
+ [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
27
+ \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
28
+ if [[ $- == *i* ]] # check for interactive shells
29
+ then echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green
30
+ else echo "Using: $GEM_HOME" # don't use colors in non-interactive shells
31
+ fi
32
+ else
33
+ # If the environment file has not yet been created, use the RVM CLI to select.
34
+ rvm --create use "$environment_id" || {
35
+ echo "Failed to create RVM environment '${environment_id}'."
36
+ return 1
37
+ }
38
+ fi
data/.travis.yml ADDED
@@ -0,0 +1,15 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - ruby-head
5
+ - rbx-19mode
6
+
7
+ matrix:
8
+ allow_failures:
9
+ - rvm: ruby-head
10
+ - rvm: rbx-19mode
11
+
12
+ before_install: gem install bundler --pre
13
+
14
+ script:
15
+ - "bundle exec rake ci:all"
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in extentions.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Alexander Paramonov
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,64 @@
1
+ Processor
2
+ ==========
3
+ [![Build Status](https://travis-ci.org/AlexParamonov/processor.png?branch=master)](http://travis-ci.org/AlexParamonov/processor)
4
+ [![Gemnasium Build Status](https://gemnasium.com/AlexParamonov/processor.png)](http://gemnasium.com/AlexParamonov/processor)
5
+
6
+ Universal processor for data migration
7
+
8
+ Contents
9
+ ---------
10
+ 1. Installation
11
+ 1. Contributing
12
+ 1. Requirements
13
+ 1. Compatibility
14
+ 1. Copyright
15
+
16
+ Installation
17
+ ------------
18
+ Add this line to your application's Gemfile:
19
+
20
+ ``` ruby
21
+ gem 'processor'
22
+ ```
23
+
24
+ And then execute:
25
+
26
+ ``` sh
27
+ bundle
28
+ ```
29
+
30
+ Or install it yourself as:
31
+
32
+ ``` sh
33
+ gem install processor
34
+ ```
35
+
36
+ Requirements
37
+ ------------
38
+ none
39
+
40
+ rspec2 for testing
41
+
42
+ Compatibility
43
+ -------------
44
+ tested with Ruby
45
+
46
+ * 1.9.3
47
+ * jruby-19mode
48
+ * rbx-19mode
49
+ * ruby-head
50
+
51
+ see [build history](http://travis-ci.org/#!/AlexParamonov/processor/builds)
52
+
53
+ Contributing
54
+ -------------
55
+ 1. Fork repository [AlexParamonov/processor](https://github.com/AlexParamonov/processor)
56
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
57
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
58
+ 4. Push to the branch (`git push origin my-new-feature`)
59
+ 5. Create new Pull Request
60
+
61
+ Copyright
62
+ ---------
63
+ Copyright © 2013 Alexander Paramonov.
64
+ Released under the MIT License. See the LICENSE file for further details.
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rspec/core/rake_task'
4
+ namespace :ci do
5
+ desc "Run tests"
6
+ RSpec::Core::RakeTask.new('all') do |t|
7
+ t.rspec_opts = '-fprogress'
8
+ t.verbose = true
9
+ end
10
+ end
@@ -0,0 +1,43 @@
1
+ require 'processor/thread_runner'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class ExampleRunner < ThreadRunner
7
+ def initialize
8
+ # Logger could be a lambda
9
+ # logger = -> name do
10
+ # ::Logger.new("log/debug_#{name}_daily.log", "daily").tap do |logger|
11
+ # logger.datetime_format = "%H:%M:%S"
12
+ # logger.level = ::Logger::DEBUG
13
+ # end
14
+ # end
15
+
16
+ # logger could be an instance of Ruby Logger
17
+ logger = ::Logger.new(STDOUT).tap do |logger|
18
+ logger.level = ::Logger::DEBUG
19
+ logger.formatter = -> _, _, _, msg do
20
+ "log < #{msg}\n"
21
+ end
22
+ end
23
+
24
+ # logger could be an instance of Rails Logger
25
+ # logger = Rails.logger
26
+ # Or be nil
27
+ # logger = nil
28
+ # in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
29
+
30
+ # messenger could be an instance of Ruby Logger
31
+ messenger = ::Logger.new(STDOUT).tap do |logger|
32
+ logger.formatter = -> _, _, _, msg do
33
+ "message > #{msg}\n"
34
+ end
35
+ logger.level = ::Logger::INFO
36
+ end
37
+
38
+ logger_observer = Processor::Observer::Logger.new(logger, messenger: messenger)
39
+ super logger_observer
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,29 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class Migration < DataProcessor
6
+ attr_reader :records
7
+ def initialize(records)
8
+ @records = records
9
+ end
10
+
11
+ def done?(records)
12
+ records.count < 1
13
+ end
14
+
15
+ def process(record)
16
+ record.do_something
17
+ "OK"
18
+ end
19
+
20
+ def fetch_records
21
+ records.shift(2)
22
+ end
23
+
24
+ def total_records
25
+ records.count
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,26 @@
1
+ require 'processor/observer/null_observer'
2
+ require 'progressbar'
3
+
4
+ module Processor
5
+ module Example
6
+ module Observer
7
+ class ProgressBar < Processor::Observer::NullObserver
8
+ def processing_started
9
+ @progress_bar = ::ProgressBar.new("Records", processor.total_records)
10
+ messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
11
+ end
12
+
13
+ def before_record_processing(record)
14
+ progress_bar.inc
15
+ end
16
+
17
+ def processing_finished
18
+ progress_bar.finish
19
+ end
20
+
21
+ private
22
+ attr_reader :progress_bar
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,12 @@
1
+ require 'processor/thread_runner'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class SimpleRunner < ThreadRunner
7
+ def initialize
8
+ super Processor::Observer::Logger.new
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,28 @@
1
+ module Processor
2
+ class DataProcessor
3
+ def done?(records)
4
+ records.count < 1
5
+ end
6
+
7
+ def process(record)
8
+ raise NotImplementedError
9
+ end
10
+
11
+ def fetch_records
12
+ raise NotImplementedError
13
+ end
14
+
15
+ def total_records
16
+ raise NotImplementedError
17
+ end
18
+
19
+ def name
20
+ # underscore a class name
21
+ self.class.name.to_s.
22
+ gsub(/::/, '_').
23
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
24
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
25
+ downcase
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,16 @@
1
+ module Processor
2
+ class EventsRegistrator
3
+ def initialize(observers)
4
+ @observers = observers
5
+ end
6
+
7
+ def register(event, *data)
8
+ observers.each do |observer|
9
+ observer.update event, *data
10
+ end
11
+ end
12
+
13
+ private
14
+ attr_reader :observers
15
+ end
16
+ end
@@ -0,0 +1,98 @@
1
+ require_relative 'null_observer'
2
+ require 'logger'
3
+
4
+ module Processor
5
+ module Observer
6
+ class Logger < NullObserver
7
+ def initialize(logger = nil, options = {})
8
+ @logger_source = logger
9
+ super options
10
+ end
11
+
12
+ def processing_started(processor)
13
+ initialize_logger(processor)
14
+
15
+ message = "Processing of #{processor.name} started."
16
+ logger.info message
17
+ messenger.info message
18
+
19
+ message = <<-MESSAGE.gsub(/^\s+/, '')
20
+ Proggress will be saved to the log file. Run
21
+ tail -f #{log_file_name}
22
+ to see log in realtime
23
+ MESSAGE
24
+ messenger.info message if use_log_file?
25
+ end
26
+
27
+ def before_record_processing(record)
28
+ message = "Record #{id_for record} is going to be processed"
29
+ logger.debug message
30
+ messenger.debug message
31
+ end
32
+
33
+ def after_record_processing(record, result)
34
+ message = "Successfully processed #{id_for record}: #{result}"
35
+ logger.info message
36
+ messenger.debug message
37
+ end
38
+
39
+ def processing_finished(processor)
40
+ message = "Processing of #{processor.name} finished."
41
+ logger.info message
42
+ messenger.info message
43
+ messenger.info "Log file saved to #{log_file_name}" if use_log_file?
44
+ end
45
+
46
+ def record_processing_error(record, exception)
47
+ message = "Error processing #{id_for record}: #{exception}"
48
+ logger.error message
49
+ messenger.error message
50
+ end
51
+
52
+ def processing_error(processor, exception)
53
+ message = "Processing #{processor.name} failed: #{exception}"
54
+ logger.fatal message
55
+ messenger.fatal message
56
+ end
57
+
58
+ private
59
+ attr_reader :logger, :log_file_name
60
+
61
+ def initialize_logger(processor)
62
+ @logger =
63
+ if @logger_source.is_a? Proc
64
+ @logger_source.call processor.name
65
+ else
66
+ @logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
67
+ logger.level = ::Logger::INFO
68
+ end
69
+ end
70
+ end
71
+
72
+ def create_log_filename(processor_name)
73
+ @log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
74
+ end
75
+
76
+ def use_log_file?
77
+ not log_file_name.nil?
78
+ end
79
+
80
+ def current_time_string
81
+ Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
82
+ end
83
+
84
+ def id_for record
85
+ [:uid, :id, :to_token, :token, :to_sym].each do |method|
86
+ return record.public_send method if record.respond_to? method
87
+ end
88
+
89
+ [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
90
+ return record[method] if record.key? method
91
+ return record[method.to_s] if record.key? method.to_s
92
+ end if record.respond_to?(:key?) && record.respond_to?(:[])
93
+
94
+ record.to_s
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,23 @@
1
+ require 'logger'
2
+
3
+ module Processor
4
+ module Observer
5
+ class NullObserver
6
+ def initialize(options = {})
7
+ @messenger = options.fetch :messenger do
8
+ ::Logger.new(STDOUT).tap do |logger|
9
+ logger.formatter = -> _, _, _, msg do
10
+ "> #{msg}\n"
11
+ end
12
+ end
13
+ end
14
+ end
15
+
16
+ def method_missing(*); end
17
+ alias_method :update, :send
18
+
19
+ private
20
+ attr_reader :messenger
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,59 @@
1
+ require_relative "events_registrator"
2
+
3
+ module Processor
4
+ class ThreadRunner
5
+ def initialize(*observers)
6
+ @observers = observers
7
+ end
8
+
9
+ # This method is thread-safe. But not observers.
10
+ # Consider creating new runner for a thread or use thread safe observers
11
+ def run(processor)
12
+ events = events_registrator
13
+ events.register :processing_started, processor
14
+
15
+ records_ran = 0
16
+ until processor.done?(records = processor.fetch_records)
17
+ threads = []
18
+ begin
19
+ records.each do |record|
20
+ recursion_preventer processor do
21
+ records_ran += 1
22
+ end
23
+
24
+ threads << Thread.new(processor, record) do |thread_data_processor, thread_record|
25
+ begin
26
+ events.register :before_record_processing, thread_record
27
+
28
+ result = thread_data_processor.process(thread_record)
29
+
30
+ events.register :after_record_processing, thread_record, result
31
+ rescue RuntimeError => exception
32
+ events.register :record_processing_error, thread_record, exception
33
+ end
34
+ end
35
+ end
36
+ ensure # join already created threads even if recursion was detected
37
+ threads.each(&:join)
38
+ end
39
+ end
40
+
41
+ events.register :processing_finished, processor
42
+ rescue Exception => exception
43
+ events.register :processing_error, processor, exception
44
+ raise exception
45
+ end
46
+
47
+ private
48
+ attr_reader :observers
49
+
50
+ def recursion_preventer(processor)
51
+ counter = yield
52
+ raise Exception, "Processing fall into recursion. Check logs." if counter > (processor.total_records * 1.1).round + 10
53
+ end
54
+
55
+ def events_registrator
56
+ EventsRegistrator.new observers
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,3 @@
1
+ module Processor
2
+ VERSION = "0.0.0.beta"
3
+ end
data/lib/processor.rb ADDED
@@ -0,0 +1,4 @@
1
+ require "processor/version"
2
+
3
+ module Processor
4
+ end
data/processor.gemspec ADDED
@@ -0,0 +1,25 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'processor/version'
5
+
6
+ Gem::Specification.new do |gem|
7
+ gem.name = "processor"
8
+ gem.version = Processor::VERSION
9
+ gem.authors = ["Alexander Paramonov"]
10
+ gem.email = ["alexander.n.paramonov@gmail.com"]
11
+ gem.summary = %q{Process records one by one}
12
+ gem.description = %q{Universal processor for data migration}
13
+ gem.homepage = "http://github.com/AlexParamonov/processor"
14
+ gem.license = "MIT"
15
+
16
+ gem.files = `git ls-files`.split($/)
17
+ gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
19
+ gem.require_paths = ["lib"]
20
+
21
+ gem.add_development_dependency "rake"
22
+ gem.add_development_dependency "rspec", ">= 2.6"
23
+ gem.add_development_dependency "pry"
24
+ end
25
+
@@ -0,0 +1,24 @@
1
+ require 'spec_helper_lite'
2
+ require_relative '../example/example_runner'
3
+ require_relative '../example/migration'
4
+ require 'fileutils'
5
+
6
+ describe "Example" do
7
+ before(:each) do
8
+ records = %w[item1 item2 item3 item4 item5]
9
+ records.each do |record|
10
+ record.should_receive(:do_something).once
11
+ end
12
+ @migration = Processor::Example::Migration.new records
13
+ end
14
+
15
+ it "should use logger and messenger" do
16
+ migration_runner = Processor::Example::ExampleRunner.new
17
+ migration_runner.run @migration
18
+ end
19
+
20
+ it "should run without configuration" do
21
+ runner = Processor::ThreadRunner.new
22
+ runner.run @migration
23
+ end
24
+ end
@@ -0,0 +1,15 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/events_registrator'
3
+
4
+ describe Processor::EventsRegistrator do
5
+ subject { Processor::EventsRegistrator }
6
+
7
+ it "should broadcast events to all observers" do
8
+ observers = 3.times.map do
9
+ stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
10
+ end
11
+
12
+ events = subject.new(observers)
13
+ events.register :test_event
14
+ end
15
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/observer/logger'
3
+
4
+ describe Processor::Observer::Logger do
5
+ let(:processor) { stub.as_null_object }
6
+ let(:no_messages) { ::Logger.new("/dev/null") }
7
+
8
+ subject { Processor::Observer::Logger }
9
+ it "accepts logger builder as parameter" do
10
+ external_logger = mock
11
+ logger_observer = subject.new -> name { external_logger }, messenger: no_messages
12
+
13
+ external_logger.should_receive(:info)
14
+ logger_observer.processing_started processor
15
+ end
16
+
17
+ it "accepts logger as parameter" do
18
+ external_logger = mock
19
+ logger_observer = subject.new external_logger, messenger: no_messages
20
+
21
+ external_logger.should_receive(:info)
22
+ logger_observer.processing_started processor
23
+ end
24
+
25
+ it "use ruby Logger if no external logger provided" do
26
+ logger_observer = subject.new nil, messenger: no_messages
27
+
28
+ Logger.should_receive(:new).and_return(stub.as_null_object)
29
+ logger_observer.processing_started processor
30
+ end
31
+ end
32
+
@@ -0,0 +1,134 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/thread_runner'
3
+
4
+ processor = Class.new do
5
+ def done?(records)
6
+ records.count < 1
7
+ end
8
+
9
+ def total_records
10
+ 10
11
+ end
12
+ end.new
13
+
14
+ describe Processor::ThreadRunner do
15
+ let(:runner) { Processor::ThreadRunner.new }
16
+ let(:events_registrator) { stub.as_null_object }
17
+ before(:each) do
18
+ runner.stub(events_registrator: events_registrator)
19
+ end
20
+
21
+ it "should fetch records from processor till it'll be done" do
22
+ processor.stub(:done?).and_return(false, false, true)
23
+ processor.should_receive(:fetch_records).exactly(3).times.and_return([])
24
+ runner.run processor
25
+ end
26
+
27
+ it "should send each found record to processor" do
28
+ records = [mock(:one), mock(:two), mock(:three)]
29
+ records.each { |record| processor.should_receive(:process).with(record) }
30
+
31
+ processor.stub(:fetch_records).and_return(records, [])
32
+ runner.run processor
33
+ end
34
+
35
+ describe "exception handling" do
36
+ let(:record) { stub }
37
+ before(:each) do
38
+ processor.stub(:fetch_records).and_return([record], [record], [])
39
+ end
40
+
41
+ describe "processing a record raised RuntimeError" do
42
+ it "should continue processing" do
43
+ processor.should_receive(:process).twice.and_raise(RuntimeError)
44
+ expect { runner.run processor }.to_not raise_error
45
+ end
46
+
47
+ it "should register a record_processing_error event" do
48
+ event_registered = false
49
+ events_registrator.should_receive(:register) do |event_name, failed_record, exception|
50
+ next if event_name != :record_processing_error
51
+ event_name.should eq :record_processing_error
52
+ failed_record.should eq record
53
+ exception.should be_a RuntimeError
54
+ event_registered = true
55
+ end.any_number_of_times
56
+
57
+ processor.stub(:process).and_raise(RuntimeError)
58
+
59
+ begin
60
+ runner.run processor
61
+ rescue Exception; end
62
+ event_registered.should be_true
63
+ end
64
+ end
65
+
66
+ describe "processing a record raised Exception" do
67
+ it "should break processing and rerise Exception" do
68
+ custom_exception = Class.new Exception
69
+ processor.should_receive(:process).once.and_raise(custom_exception)
70
+ expect { runner.run processor }.to raise_error(custom_exception)
71
+ end
72
+
73
+ it "should register a processing_error event" do
74
+ event_registered = false
75
+ events_registrator.should_receive(:register) do |event_name, current_processor, exception|
76
+ next if event_name != :processing_error
77
+ event_name.should eq :processing_error
78
+ current_processor.should eq processor
79
+ exception.should be_a Exception
80
+ event_registered = true
81
+ end.any_number_of_times
82
+
83
+ processor.stub(:process).and_raise(Exception)
84
+
85
+ begin
86
+ runner.run processor
87
+ rescue Exception; end
88
+ event_registered.should be_true
89
+ end
90
+ end
91
+
92
+ describe "fetching records raised" do
93
+ it "should break processing and rerise Exception" do
94
+ custom_exception = Class.new RuntimeError
95
+ processor.stub(:fetch_records).and_raise(custom_exception)
96
+ processor.should_not_receive(:process)
97
+ expect { runner.run processor }.to raise_error(custom_exception)
98
+ end
99
+
100
+ it "should register a processing_error" do
101
+ event_registered = false
102
+ events_registrator.should_receive(:register) do |event_name, processor, exception|
103
+ next if event_name != :processing_error
104
+ event_name.should eq :processing_error
105
+ exception.should be_a RuntimeError
106
+ event_registered = true
107
+ end.any_number_of_times
108
+
109
+ processor.stub(:fetch_records).and_raise(RuntimeError)
110
+
111
+ begin
112
+ runner.run processor
113
+ rescue Exception; end
114
+ event_registered.should be_true
115
+ end
116
+ end
117
+ end
118
+
119
+ describe "recursion" do
120
+ it "should not allow infinit recursion" do
121
+ processor.stub(:fetch_records).and_return([:one, :two])
122
+ processor.should_receive(:process).at_most(100).times
123
+ expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
124
+ end
125
+
126
+ it "should have 10% + 10 rerurns window" do
127
+ processor.stub(total_records: 100)
128
+ processor.stub(:fetch_records).and_return([:one, :two])
129
+ processor.should_receive(:process).exactly(120).times
130
+ expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
131
+ end
132
+ end
133
+ end
134
+
@@ -0,0 +1,6 @@
1
+ require 'pry'
2
+
3
+ ENV['RAILS_ENV'] ||= 'test'
4
+ require "rspec"
5
+
6
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
metadata ADDED
@@ -0,0 +1,124 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: processor
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.0.beta
5
+ prerelease: 6
6
+ platform: ruby
7
+ authors:
8
+ - Alexander Paramonov
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-05-02 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: rake
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: rspec
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '2.6'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '2.6'
46
+ - !ruby/object:Gem::Dependency
47
+ name: pry
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ description: Universal processor for data migration
63
+ email:
64
+ - alexander.n.paramonov@gmail.com
65
+ executables: []
66
+ extensions: []
67
+ extra_rdoc_files: []
68
+ files:
69
+ - .gitignore
70
+ - .rspec
71
+ - .rvmrc
72
+ - .travis.yml
73
+ - Gemfile
74
+ - LICENSE
75
+ - README.md
76
+ - Rakefile
77
+ - example/example_runner.rb
78
+ - example/migration.rb
79
+ - example/observer/progress_bar.rb
80
+ - example/simple_runner.rb
81
+ - lib/processor.rb
82
+ - lib/processor/data_processor.rb
83
+ - lib/processor/events_registrator.rb
84
+ - lib/processor/observer/logger.rb
85
+ - lib/processor/observer/null_observer.rb
86
+ - lib/processor/thread_runner.rb
87
+ - lib/processor/version.rb
88
+ - processor.gemspec
89
+ - spec/example_spec.rb
90
+ - spec/processor/events_registrator_spec.rb
91
+ - spec/processor/observer/logger_spec.rb
92
+ - spec/processor/thread_runner_spec.rb
93
+ - spec/spec_helper_lite.rb
94
+ homepage: http://github.com/AlexParamonov/processor
95
+ licenses:
96
+ - MIT
97
+ post_install_message:
98
+ rdoc_options: []
99
+ require_paths:
100
+ - lib
101
+ required_ruby_version: !ruby/object:Gem::Requirement
102
+ none: false
103
+ requirements:
104
+ - - ! '>='
105
+ - !ruby/object:Gem::Version
106
+ version: '0'
107
+ required_rubygems_version: !ruby/object:Gem::Requirement
108
+ none: false
109
+ requirements:
110
+ - - ! '>'
111
+ - !ruby/object:Gem::Version
112
+ version: 1.3.1
113
+ requirements: []
114
+ rubyforge_project:
115
+ rubygems_version: 1.8.25
116
+ signing_key:
117
+ specification_version: 3
118
+ summary: Process records one by one
119
+ test_files:
120
+ - spec/example_spec.rb
121
+ - spec/processor/events_registrator_spec.rb
122
+ - spec/processor/observer/logger_spec.rb
123
+ - spec/processor/thread_runner_spec.rb
124
+ - spec/spec_helper_lite.rb