processor 0.0.0.beta
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +16 -0
- data/.rspec +2 -0
- data/.rvmrc +38 -0
- data/.travis.yml +15 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +64 -0
- data/Rakefile +10 -0
- data/example/example_runner.rb +43 -0
- data/example/migration.rb +29 -0
- data/example/observer/progress_bar.rb +26 -0
- data/example/simple_runner.rb +12 -0
- data/lib/processor/data_processor.rb +28 -0
- data/lib/processor/events_registrator.rb +16 -0
- data/lib/processor/observer/logger.rb +98 -0
- data/lib/processor/observer/null_observer.rb +23 -0
- data/lib/processor/thread_runner.rb +59 -0
- data/lib/processor/version.rb +3 -0
- data/lib/processor.rb +4 -0
- data/processor.gemspec +25 -0
- data/spec/example_spec.rb +24 -0
- data/spec/processor/events_registrator_spec.rb +15 -0
- data/spec/processor/observer/logger_spec.rb +32 -0
- data/spec/processor/thread_runner_spec.rb +134 -0
- data/spec/spec_helper_lite.rb +6 -0
- metadata +124 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
# This is an RVM Project .rvmrc file, used to automatically load the ruby
|
4
|
+
# development environment upon cd'ing into the directory
|
5
|
+
|
6
|
+
# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
|
7
|
+
# Only full ruby name is supported here, for short names use:
|
8
|
+
# echo "rvm use 1.9.3" > .rvmrc
|
9
|
+
environment_id="ruby-1.9.3@processor"
|
10
|
+
|
11
|
+
# Uncomment the following lines if you want to verify rvm version per project
|
12
|
+
# rvmrc_rvm_version="1.15.8 (stable)" # 1.10.1 seams as a safe start
|
13
|
+
# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
|
14
|
+
# echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
|
15
|
+
# return 1
|
16
|
+
# }
|
17
|
+
|
18
|
+
# First we attempt to load the desired environment directly from the environment
|
19
|
+
# file. This is very fast and efficient compared to running through the entire
|
20
|
+
# CLI and selector. If you want feedback on which environment was used then
|
21
|
+
# insert the word 'use' after --create as this triggers verbose mode.
|
22
|
+
if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
|
23
|
+
&& -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
|
24
|
+
then
|
25
|
+
\. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
|
26
|
+
[[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
|
27
|
+
\. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
|
28
|
+
if [[ $- == *i* ]] # check for interactive shells
|
29
|
+
then echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green
|
30
|
+
else echo "Using: $GEM_HOME" # don't use colors in non-interactive shells
|
31
|
+
fi
|
32
|
+
else
|
33
|
+
# If the environment file has not yet been created, use the RVM CLI to select.
|
34
|
+
rvm --create use "$environment_id" || {
|
35
|
+
echo "Failed to create RVM environment '${environment_id}'."
|
36
|
+
return 1
|
37
|
+
}
|
38
|
+
fi
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Alexander Paramonov
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
Processor
|
2
|
+
==========
|
3
|
+
[![Build Status](https://travis-ci.org/AlexParamonov/processor.png?branch=master)](http://travis-ci.org/AlexParamonov/processor)
|
4
|
+
[![Gemnasium Build Status](https://gemnasium.com/AlexParamonov/processor.png)](http://gemnasium.com/AlexParamonov/processor)
|
5
|
+
|
6
|
+
Universal processor for data migration
|
7
|
+
|
8
|
+
Contents
|
9
|
+
---------
|
10
|
+
1. Installation
|
11
|
+
1. Contributing
|
12
|
+
1. Requirements
|
13
|
+
1. Compatibility
|
14
|
+
1. Copyright
|
15
|
+
|
16
|
+
Installation
|
17
|
+
------------
|
18
|
+
Add this line to your application's Gemfile:
|
19
|
+
|
20
|
+
``` ruby
|
21
|
+
gem 'processor'
|
22
|
+
```
|
23
|
+
|
24
|
+
And then execute:
|
25
|
+
|
26
|
+
``` sh
|
27
|
+
bundle
|
28
|
+
```
|
29
|
+
|
30
|
+
Or install it yourself as:
|
31
|
+
|
32
|
+
``` sh
|
33
|
+
gem install processor
|
34
|
+
```
|
35
|
+
|
36
|
+
Requirements
|
37
|
+
------------
|
38
|
+
none
|
39
|
+
|
40
|
+
rspec2 for testing
|
41
|
+
|
42
|
+
Compatibility
|
43
|
+
-------------
|
44
|
+
tested with Ruby
|
45
|
+
|
46
|
+
* 1.9.3
|
47
|
+
* jruby-19mode
|
48
|
+
* rbx-19mode
|
49
|
+
* ruby-head
|
50
|
+
|
51
|
+
see [build history](http://travis-ci.org/#!/AlexParamonov/processor/builds)
|
52
|
+
|
53
|
+
Contributing
|
54
|
+
-------------
|
55
|
+
1. Fork repository [AlexParamonov/processor](https://github.com/AlexParamonov/processor)
|
56
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
57
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
58
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
59
|
+
5. Create new Pull Request
|
60
|
+
|
61
|
+
Copyright
|
62
|
+
---------
|
63
|
+
Copyright © 2013 Alexander Paramonov.
|
64
|
+
Released under the MIT License. See the LICENSE file for further details.
|
data/Rakefile
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'processor/thread_runner'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
class ExampleRunner < ThreadRunner
|
7
|
+
def initialize
|
8
|
+
# Logger could be a lambda
|
9
|
+
# logger = -> name do
|
10
|
+
# ::Logger.new("log/debug_#{name}_daily.log", "daily").tap do |logger|
|
11
|
+
# logger.datetime_format = "%H:%M:%S"
|
12
|
+
# logger.level = ::Logger::DEBUG
|
13
|
+
# end
|
14
|
+
# end
|
15
|
+
|
16
|
+
# logger could be an instance of Ruby Logger
|
17
|
+
logger = ::Logger.new(STDOUT).tap do |logger|
|
18
|
+
logger.level = ::Logger::DEBUG
|
19
|
+
logger.formatter = -> _, _, _, msg do
|
20
|
+
"log < #{msg}\n"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# logger could be an instance of Rails Logger
|
25
|
+
# logger = Rails.logger
|
26
|
+
# Or be nil
|
27
|
+
# logger = nil
|
28
|
+
# in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
|
29
|
+
|
30
|
+
# messenger could be an instance of Ruby Logger
|
31
|
+
messenger = ::Logger.new(STDOUT).tap do |logger|
|
32
|
+
logger.formatter = -> _, _, _, msg do
|
33
|
+
"message > #{msg}\n"
|
34
|
+
end
|
35
|
+
logger.level = ::Logger::INFO
|
36
|
+
end
|
37
|
+
|
38
|
+
logger_observer = Processor::Observer::Logger.new(logger, messenger: messenger)
|
39
|
+
super logger_observer
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class Migration < DataProcessor
|
6
|
+
attr_reader :records
|
7
|
+
def initialize(records)
|
8
|
+
@records = records
|
9
|
+
end
|
10
|
+
|
11
|
+
def done?(records)
|
12
|
+
records.count < 1
|
13
|
+
end
|
14
|
+
|
15
|
+
def process(record)
|
16
|
+
record.do_something
|
17
|
+
"OK"
|
18
|
+
end
|
19
|
+
|
20
|
+
def fetch_records
|
21
|
+
records.shift(2)
|
22
|
+
end
|
23
|
+
|
24
|
+
def total_records
|
25
|
+
records.count
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'processor/observer/null_observer'
|
2
|
+
require 'progressbar'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
module Observer
|
7
|
+
class ProgressBar < Processor::Observer::NullObserver
|
8
|
+
def processing_started
|
9
|
+
@progress_bar = ::ProgressBar.new("Records", processor.total_records)
|
10
|
+
messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
|
11
|
+
end
|
12
|
+
|
13
|
+
def before_record_processing(record)
|
14
|
+
progress_bar.inc
|
15
|
+
end
|
16
|
+
|
17
|
+
def processing_finished
|
18
|
+
progress_bar.finish
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
attr_reader :progress_bar
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Processor
|
2
|
+
class DataProcessor
|
3
|
+
def done?(records)
|
4
|
+
records.count < 1
|
5
|
+
end
|
6
|
+
|
7
|
+
def process(record)
|
8
|
+
raise NotImplementedError
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_records
|
12
|
+
raise NotImplementedError
|
13
|
+
end
|
14
|
+
|
15
|
+
def total_records
|
16
|
+
raise NotImplementedError
|
17
|
+
end
|
18
|
+
|
19
|
+
def name
|
20
|
+
# underscore a class name
|
21
|
+
self.class.name.to_s.
|
22
|
+
gsub(/::/, '_').
|
23
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
24
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
25
|
+
downcase
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Processor
|
2
|
+
class EventsRegistrator
|
3
|
+
def initialize(observers)
|
4
|
+
@observers = observers
|
5
|
+
end
|
6
|
+
|
7
|
+
def register(event, *data)
|
8
|
+
observers.each do |observer|
|
9
|
+
observer.update event, *data
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
attr_reader :observers
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require_relative 'null_observer'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Observer
|
6
|
+
class Logger < NullObserver
|
7
|
+
def initialize(logger = nil, options = {})
|
8
|
+
@logger_source = logger
|
9
|
+
super options
|
10
|
+
end
|
11
|
+
|
12
|
+
def processing_started(processor)
|
13
|
+
initialize_logger(processor)
|
14
|
+
|
15
|
+
message = "Processing of #{processor.name} started."
|
16
|
+
logger.info message
|
17
|
+
messenger.info message
|
18
|
+
|
19
|
+
message = <<-MESSAGE.gsub(/^\s+/, '')
|
20
|
+
Proggress will be saved to the log file. Run
|
21
|
+
tail -f #{log_file_name}
|
22
|
+
to see log in realtime
|
23
|
+
MESSAGE
|
24
|
+
messenger.info message if use_log_file?
|
25
|
+
end
|
26
|
+
|
27
|
+
def before_record_processing(record)
|
28
|
+
message = "Record #{id_for record} is going to be processed"
|
29
|
+
logger.debug message
|
30
|
+
messenger.debug message
|
31
|
+
end
|
32
|
+
|
33
|
+
def after_record_processing(record, result)
|
34
|
+
message = "Successfully processed #{id_for record}: #{result}"
|
35
|
+
logger.info message
|
36
|
+
messenger.debug message
|
37
|
+
end
|
38
|
+
|
39
|
+
def processing_finished(processor)
|
40
|
+
message = "Processing of #{processor.name} finished."
|
41
|
+
logger.info message
|
42
|
+
messenger.info message
|
43
|
+
messenger.info "Log file saved to #{log_file_name}" if use_log_file?
|
44
|
+
end
|
45
|
+
|
46
|
+
def record_processing_error(record, exception)
|
47
|
+
message = "Error processing #{id_for record}: #{exception}"
|
48
|
+
logger.error message
|
49
|
+
messenger.error message
|
50
|
+
end
|
51
|
+
|
52
|
+
def processing_error(processor, exception)
|
53
|
+
message = "Processing #{processor.name} failed: #{exception}"
|
54
|
+
logger.fatal message
|
55
|
+
messenger.fatal message
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
attr_reader :logger, :log_file_name
|
60
|
+
|
61
|
+
def initialize_logger(processor)
|
62
|
+
@logger =
|
63
|
+
if @logger_source.is_a? Proc
|
64
|
+
@logger_source.call processor.name
|
65
|
+
else
|
66
|
+
@logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
|
67
|
+
logger.level = ::Logger::INFO
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def create_log_filename(processor_name)
|
73
|
+
@log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
|
74
|
+
end
|
75
|
+
|
76
|
+
def use_log_file?
|
77
|
+
not log_file_name.nil?
|
78
|
+
end
|
79
|
+
|
80
|
+
def current_time_string
|
81
|
+
Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
|
82
|
+
end
|
83
|
+
|
84
|
+
def id_for record
|
85
|
+
[:uid, :id, :to_token, :token, :to_sym].each do |method|
|
86
|
+
return record.public_send method if record.respond_to? method
|
87
|
+
end
|
88
|
+
|
89
|
+
[:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
|
90
|
+
return record[method] if record.key? method
|
91
|
+
return record[method.to_s] if record.key? method.to_s
|
92
|
+
end if record.respond_to?(:key?) && record.respond_to?(:[])
|
93
|
+
|
94
|
+
record.to_s
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Observer
|
5
|
+
class NullObserver
|
6
|
+
def initialize(options = {})
|
7
|
+
@messenger = options.fetch :messenger do
|
8
|
+
::Logger.new(STDOUT).tap do |logger|
|
9
|
+
logger.formatter = -> _, _, _, msg do
|
10
|
+
"> #{msg}\n"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def method_missing(*); end
|
17
|
+
alias_method :update, :send
|
18
|
+
|
19
|
+
private
|
20
|
+
attr_reader :messenger
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require_relative "events_registrator"
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
class ThreadRunner
|
5
|
+
def initialize(*observers)
|
6
|
+
@observers = observers
|
7
|
+
end
|
8
|
+
|
9
|
+
# This method is thread-safe. But not observers.
|
10
|
+
# Consider creating new runner for a thread or use thread safe observers
|
11
|
+
def run(processor)
|
12
|
+
events = events_registrator
|
13
|
+
events.register :processing_started, processor
|
14
|
+
|
15
|
+
records_ran = 0
|
16
|
+
until processor.done?(records = processor.fetch_records)
|
17
|
+
threads = []
|
18
|
+
begin
|
19
|
+
records.each do |record|
|
20
|
+
recursion_preventer processor do
|
21
|
+
records_ran += 1
|
22
|
+
end
|
23
|
+
|
24
|
+
threads << Thread.new(processor, record) do |thread_data_processor, thread_record|
|
25
|
+
begin
|
26
|
+
events.register :before_record_processing, thread_record
|
27
|
+
|
28
|
+
result = thread_data_processor.process(thread_record)
|
29
|
+
|
30
|
+
events.register :after_record_processing, thread_record, result
|
31
|
+
rescue RuntimeError => exception
|
32
|
+
events.register :record_processing_error, thread_record, exception
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
ensure # join already created threads even if recursion was detected
|
37
|
+
threads.each(&:join)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
events.register :processing_finished, processor
|
42
|
+
rescue Exception => exception
|
43
|
+
events.register :processing_error, processor, exception
|
44
|
+
raise exception
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
attr_reader :observers
|
49
|
+
|
50
|
+
def recursion_preventer(processor)
|
51
|
+
counter = yield
|
52
|
+
raise Exception, "Processing fall into recursion. Check logs." if counter > (processor.total_records * 1.1).round + 10
|
53
|
+
end
|
54
|
+
|
55
|
+
def events_registrator
|
56
|
+
EventsRegistrator.new observers
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/processor.rb
ADDED
data/processor.gemspec
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'processor/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "processor"
|
8
|
+
gem.version = Processor::VERSION
|
9
|
+
gem.authors = ["Alexander Paramonov"]
|
10
|
+
gem.email = ["alexander.n.paramonov@gmail.com"]
|
11
|
+
gem.summary = %q{Process records one by one}
|
12
|
+
gem.description = %q{Universal processor for data migration}
|
13
|
+
gem.homepage = "http://github.com/AlexParamonov/processor"
|
14
|
+
gem.license = "MIT"
|
15
|
+
|
16
|
+
gem.files = `git ls-files`.split($/)
|
17
|
+
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
19
|
+
gem.require_paths = ["lib"]
|
20
|
+
|
21
|
+
gem.add_development_dependency "rake"
|
22
|
+
gem.add_development_dependency "rspec", ">= 2.6"
|
23
|
+
gem.add_development_dependency "pry"
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require_relative '../example/example_runner'
|
3
|
+
require_relative '../example/migration'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
describe "Example" do
|
7
|
+
before(:each) do
|
8
|
+
records = %w[item1 item2 item3 item4 item5]
|
9
|
+
records.each do |record|
|
10
|
+
record.should_receive(:do_something).once
|
11
|
+
end
|
12
|
+
@migration = Processor::Example::Migration.new records
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should use logger and messenger" do
|
16
|
+
migration_runner = Processor::Example::ExampleRunner.new
|
17
|
+
migration_runner.run @migration
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should run without configuration" do
|
21
|
+
runner = Processor::ThreadRunner.new
|
22
|
+
runner.run @migration
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/events_registrator'
|
3
|
+
|
4
|
+
describe Processor::EventsRegistrator do
|
5
|
+
subject { Processor::EventsRegistrator }
|
6
|
+
|
7
|
+
it "should broadcast events to all observers" do
|
8
|
+
observers = 3.times.map do
|
9
|
+
stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
|
10
|
+
end
|
11
|
+
|
12
|
+
events = subject.new(observers)
|
13
|
+
events.register :test_event
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
describe Processor::Observer::Logger do
|
5
|
+
let(:processor) { stub.as_null_object }
|
6
|
+
let(:no_messages) { ::Logger.new("/dev/null") }
|
7
|
+
|
8
|
+
subject { Processor::Observer::Logger }
|
9
|
+
it "accepts logger builder as parameter" do
|
10
|
+
external_logger = mock
|
11
|
+
logger_observer = subject.new -> name { external_logger }, messenger: no_messages
|
12
|
+
|
13
|
+
external_logger.should_receive(:info)
|
14
|
+
logger_observer.processing_started processor
|
15
|
+
end
|
16
|
+
|
17
|
+
it "accepts logger as parameter" do
|
18
|
+
external_logger = mock
|
19
|
+
logger_observer = subject.new external_logger, messenger: no_messages
|
20
|
+
|
21
|
+
external_logger.should_receive(:info)
|
22
|
+
logger_observer.processing_started processor
|
23
|
+
end
|
24
|
+
|
25
|
+
it "use ruby Logger if no external logger provided" do
|
26
|
+
logger_observer = subject.new nil, messenger: no_messages
|
27
|
+
|
28
|
+
Logger.should_receive(:new).and_return(stub.as_null_object)
|
29
|
+
logger_observer.processing_started processor
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
@@ -0,0 +1,134 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/thread_runner'
|
3
|
+
|
4
|
+
processor = Class.new do
|
5
|
+
def done?(records)
|
6
|
+
records.count < 1
|
7
|
+
end
|
8
|
+
|
9
|
+
def total_records
|
10
|
+
10
|
11
|
+
end
|
12
|
+
end.new
|
13
|
+
|
14
|
+
describe Processor::ThreadRunner do
|
15
|
+
let(:runner) { Processor::ThreadRunner.new }
|
16
|
+
let(:events_registrator) { stub.as_null_object }
|
17
|
+
before(:each) do
|
18
|
+
runner.stub(events_registrator: events_registrator)
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should fetch records from processor till it'll be done" do
|
22
|
+
processor.stub(:done?).and_return(false, false, true)
|
23
|
+
processor.should_receive(:fetch_records).exactly(3).times.and_return([])
|
24
|
+
runner.run processor
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should send each found record to processor" do
|
28
|
+
records = [mock(:one), mock(:two), mock(:three)]
|
29
|
+
records.each { |record| processor.should_receive(:process).with(record) }
|
30
|
+
|
31
|
+
processor.stub(:fetch_records).and_return(records, [])
|
32
|
+
runner.run processor
|
33
|
+
end
|
34
|
+
|
35
|
+
describe "exception handling" do
|
36
|
+
let(:record) { stub }
|
37
|
+
before(:each) do
|
38
|
+
processor.stub(:fetch_records).and_return([record], [record], [])
|
39
|
+
end
|
40
|
+
|
41
|
+
describe "processing a record raised RuntimeError" do
|
42
|
+
it "should continue processing" do
|
43
|
+
processor.should_receive(:process).twice.and_raise(RuntimeError)
|
44
|
+
expect { runner.run processor }.to_not raise_error
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should register a record_processing_error event" do
|
48
|
+
event_registered = false
|
49
|
+
events_registrator.should_receive(:register) do |event_name, failed_record, exception|
|
50
|
+
next if event_name != :record_processing_error
|
51
|
+
event_name.should eq :record_processing_error
|
52
|
+
failed_record.should eq record
|
53
|
+
exception.should be_a RuntimeError
|
54
|
+
event_registered = true
|
55
|
+
end.any_number_of_times
|
56
|
+
|
57
|
+
processor.stub(:process).and_raise(RuntimeError)
|
58
|
+
|
59
|
+
begin
|
60
|
+
runner.run processor
|
61
|
+
rescue Exception; end
|
62
|
+
event_registered.should be_true
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
describe "processing a record raised Exception" do
|
67
|
+
it "should break processing and rerise Exception" do
|
68
|
+
custom_exception = Class.new Exception
|
69
|
+
processor.should_receive(:process).once.and_raise(custom_exception)
|
70
|
+
expect { runner.run processor }.to raise_error(custom_exception)
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should register a processing_error event" do
|
74
|
+
event_registered = false
|
75
|
+
events_registrator.should_receive(:register) do |event_name, current_processor, exception|
|
76
|
+
next if event_name != :processing_error
|
77
|
+
event_name.should eq :processing_error
|
78
|
+
current_processor.should eq processor
|
79
|
+
exception.should be_a Exception
|
80
|
+
event_registered = true
|
81
|
+
end.any_number_of_times
|
82
|
+
|
83
|
+
processor.stub(:process).and_raise(Exception)
|
84
|
+
|
85
|
+
begin
|
86
|
+
runner.run processor
|
87
|
+
rescue Exception; end
|
88
|
+
event_registered.should be_true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe "fetching records raised" do
|
93
|
+
it "should break processing and rerise Exception" do
|
94
|
+
custom_exception = Class.new RuntimeError
|
95
|
+
processor.stub(:fetch_records).and_raise(custom_exception)
|
96
|
+
processor.should_not_receive(:process)
|
97
|
+
expect { runner.run processor }.to raise_error(custom_exception)
|
98
|
+
end
|
99
|
+
|
100
|
+
it "should register a processing_error" do
|
101
|
+
event_registered = false
|
102
|
+
events_registrator.should_receive(:register) do |event_name, processor, exception|
|
103
|
+
next if event_name != :processing_error
|
104
|
+
event_name.should eq :processing_error
|
105
|
+
exception.should be_a RuntimeError
|
106
|
+
event_registered = true
|
107
|
+
end.any_number_of_times
|
108
|
+
|
109
|
+
processor.stub(:fetch_records).and_raise(RuntimeError)
|
110
|
+
|
111
|
+
begin
|
112
|
+
runner.run processor
|
113
|
+
rescue Exception; end
|
114
|
+
event_registered.should be_true
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
describe "recursion" do
|
120
|
+
it "should not allow infinit recursion" do
|
121
|
+
processor.stub(:fetch_records).and_return([:one, :two])
|
122
|
+
processor.should_receive(:process).at_most(100).times
|
123
|
+
expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
|
124
|
+
end
|
125
|
+
|
126
|
+
it "should have 10% + 10 rerurns window" do
|
127
|
+
processor.stub(total_records: 100)
|
128
|
+
processor.stub(:fetch_records).and_return([:one, :two])
|
129
|
+
processor.should_receive(:process).exactly(120).times
|
130
|
+
expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
metadata
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: processor
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0.beta
|
5
|
+
prerelease: 6
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Alexander Paramonov
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-05-02 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rake
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rspec
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '2.6'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '2.6'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: pry
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
description: Universal processor for data migration
|
63
|
+
email:
|
64
|
+
- alexander.n.paramonov@gmail.com
|
65
|
+
executables: []
|
66
|
+
extensions: []
|
67
|
+
extra_rdoc_files: []
|
68
|
+
files:
|
69
|
+
- .gitignore
|
70
|
+
- .rspec
|
71
|
+
- .rvmrc
|
72
|
+
- .travis.yml
|
73
|
+
- Gemfile
|
74
|
+
- LICENSE
|
75
|
+
- README.md
|
76
|
+
- Rakefile
|
77
|
+
- example/example_runner.rb
|
78
|
+
- example/migration.rb
|
79
|
+
- example/observer/progress_bar.rb
|
80
|
+
- example/simple_runner.rb
|
81
|
+
- lib/processor.rb
|
82
|
+
- lib/processor/data_processor.rb
|
83
|
+
- lib/processor/events_registrator.rb
|
84
|
+
- lib/processor/observer/logger.rb
|
85
|
+
- lib/processor/observer/null_observer.rb
|
86
|
+
- lib/processor/thread_runner.rb
|
87
|
+
- lib/processor/version.rb
|
88
|
+
- processor.gemspec
|
89
|
+
- spec/example_spec.rb
|
90
|
+
- spec/processor/events_registrator_spec.rb
|
91
|
+
- spec/processor/observer/logger_spec.rb
|
92
|
+
- spec/processor/thread_runner_spec.rb
|
93
|
+
- spec/spec_helper_lite.rb
|
94
|
+
homepage: http://github.com/AlexParamonov/processor
|
95
|
+
licenses:
|
96
|
+
- MIT
|
97
|
+
post_install_message:
|
98
|
+
rdoc_options: []
|
99
|
+
require_paths:
|
100
|
+
- lib
|
101
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
102
|
+
none: false
|
103
|
+
requirements:
|
104
|
+
- - ! '>='
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
+
none: false
|
109
|
+
requirements:
|
110
|
+
- - ! '>'
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: 1.3.1
|
113
|
+
requirements: []
|
114
|
+
rubyforge_project:
|
115
|
+
rubygems_version: 1.8.25
|
116
|
+
signing_key:
|
117
|
+
specification_version: 3
|
118
|
+
summary: Process records one by one
|
119
|
+
test_files:
|
120
|
+
- spec/example_spec.rb
|
121
|
+
- spec/processor/events_registrator_spec.rb
|
122
|
+
- spec/processor/observer/logger_spec.rb
|
123
|
+
- spec/processor/thread_runner_spec.rb
|
124
|
+
- spec/spec_helper_lite.rb
|