processor 0.0.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +16 -0
- data/.rspec +2 -0
- data/.rvmrc +38 -0
- data/.travis.yml +15 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +64 -0
- data/Rakefile +10 -0
- data/example/example_runner.rb +43 -0
- data/example/migration.rb +29 -0
- data/example/observer/progress_bar.rb +26 -0
- data/example/simple_runner.rb +12 -0
- data/lib/processor/data_processor.rb +28 -0
- data/lib/processor/events_registrator.rb +16 -0
- data/lib/processor/observer/logger.rb +98 -0
- data/lib/processor/observer/null_observer.rb +23 -0
- data/lib/processor/thread_runner.rb +59 -0
- data/lib/processor/version.rb +3 -0
- data/lib/processor.rb +4 -0
- data/processor.gemspec +25 -0
- data/spec/example_spec.rb +24 -0
- data/spec/processor/events_registrator_spec.rb +15 -0
- data/spec/processor/observer/logger_spec.rb +32 -0
- data/spec/processor/thread_runner_spec.rb +134 -0
- data/spec/spec_helper_lite.rb +6 -0
- metadata +124 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
# This is an RVM Project .rvmrc file, used to automatically load the ruby
|
4
|
+
# development environment upon cd'ing into the directory
|
5
|
+
|
6
|
+
# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
|
7
|
+
# Only full ruby name is supported here, for short names use:
|
8
|
+
# echo "rvm use 1.9.3" > .rvmrc
|
9
|
+
environment_id="ruby-1.9.3@processor"
|
10
|
+
|
11
|
+
# Uncomment the following lines if you want to verify rvm version per project
|
12
|
+
# rvmrc_rvm_version="1.15.8 (stable)" # 1.10.1 seams as a safe start
|
13
|
+
# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
|
14
|
+
# echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
|
15
|
+
# return 1
|
16
|
+
# }
|
17
|
+
|
18
|
+
# First we attempt to load the desired environment directly from the environment
|
19
|
+
# file. This is very fast and efficient compared to running through the entire
|
20
|
+
# CLI and selector. If you want feedback on which environment was used then
|
21
|
+
# insert the word 'use' after --create as this triggers verbose mode.
|
22
|
+
if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
|
23
|
+
&& -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
|
24
|
+
then
|
25
|
+
\. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
|
26
|
+
[[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
|
27
|
+
\. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
|
28
|
+
if [[ $- == *i* ]] # check for interactive shells
|
29
|
+
then echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green
|
30
|
+
else echo "Using: $GEM_HOME" # don't use colors in non-interactive shells
|
31
|
+
fi
|
32
|
+
else
|
33
|
+
# If the environment file has not yet been created, use the RVM CLI to select.
|
34
|
+
rvm --create use "$environment_id" || {
|
35
|
+
echo "Failed to create RVM environment '${environment_id}'."
|
36
|
+
return 1
|
37
|
+
}
|
38
|
+
fi
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Alexander Paramonov
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
Processor
|
2
|
+
==========
|
3
|
+
[](http://travis-ci.org/AlexParamonov/processor)
|
4
|
+
[](http://gemnasium.com/AlexParamonov/processor)
|
5
|
+
|
6
|
+
Universal processor for data migration
|
7
|
+
|
8
|
+
Contents
|
9
|
+
---------
|
10
|
+
1. Installation
|
11
|
+
1. Contributing
|
12
|
+
1. Requirements
|
13
|
+
1. Compatibility
|
14
|
+
1. Copyright
|
15
|
+
|
16
|
+
Installation
|
17
|
+
------------
|
18
|
+
Add this line to your application's Gemfile:
|
19
|
+
|
20
|
+
``` ruby
|
21
|
+
gem 'processor'
|
22
|
+
```
|
23
|
+
|
24
|
+
And then execute:
|
25
|
+
|
26
|
+
``` sh
|
27
|
+
bundle
|
28
|
+
```
|
29
|
+
|
30
|
+
Or install it yourself as:
|
31
|
+
|
32
|
+
``` sh
|
33
|
+
gem install processor
|
34
|
+
```
|
35
|
+
|
36
|
+
Requirements
|
37
|
+
------------
|
38
|
+
none
|
39
|
+
|
40
|
+
rspec2 for testing
|
41
|
+
|
42
|
+
Compatibility
|
43
|
+
-------------
|
44
|
+
tested with Ruby
|
45
|
+
|
46
|
+
* 1.9.3
|
47
|
+
* jruby-19mode
|
48
|
+
* rbx-19mode
|
49
|
+
* ruby-head
|
50
|
+
|
51
|
+
see [build history](http://travis-ci.org/#!/AlexParamonov/processor/builds)
|
52
|
+
|
53
|
+
Contributing
|
54
|
+
-------------
|
55
|
+
1. Fork repository [AlexParamonov/processor](https://github.com/AlexParamonov/processor)
|
56
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
57
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
58
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
59
|
+
5. Create new Pull Request
|
60
|
+
|
61
|
+
Copyright
|
62
|
+
---------
|
63
|
+
Copyright © 2013 Alexander Paramonov.
|
64
|
+
Released under the MIT License. See the LICENSE file for further details.
|
data/Rakefile
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'processor/thread_runner'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
class ExampleRunner < ThreadRunner
|
7
|
+
def initialize
|
8
|
+
# Logger could be a lambda
|
9
|
+
# logger = -> name do
|
10
|
+
# ::Logger.new("log/debug_#{name}_daily.log", "daily").tap do |logger|
|
11
|
+
# logger.datetime_format = "%H:%M:%S"
|
12
|
+
# logger.level = ::Logger::DEBUG
|
13
|
+
# end
|
14
|
+
# end
|
15
|
+
|
16
|
+
# logger could be an instance of Ruby Logger
|
17
|
+
logger = ::Logger.new(STDOUT).tap do |logger|
|
18
|
+
logger.level = ::Logger::DEBUG
|
19
|
+
logger.formatter = -> _, _, _, msg do
|
20
|
+
"log < #{msg}\n"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# logger could be an instance of Rails Logger
|
25
|
+
# logger = Rails.logger
|
26
|
+
# Or be nil
|
27
|
+
# logger = nil
|
28
|
+
# in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
|
29
|
+
|
30
|
+
# messenger could be an instance of Ruby Logger
|
31
|
+
messenger = ::Logger.new(STDOUT).tap do |logger|
|
32
|
+
logger.formatter = -> _, _, _, msg do
|
33
|
+
"message > #{msg}\n"
|
34
|
+
end
|
35
|
+
logger.level = ::Logger::INFO
|
36
|
+
end
|
37
|
+
|
38
|
+
logger_observer = Processor::Observer::Logger.new(logger, messenger: messenger)
|
39
|
+
super logger_observer
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class Migration < DataProcessor
|
6
|
+
attr_reader :records
|
7
|
+
def initialize(records)
|
8
|
+
@records = records
|
9
|
+
end
|
10
|
+
|
11
|
+
def done?(records)
|
12
|
+
records.count < 1
|
13
|
+
end
|
14
|
+
|
15
|
+
def process(record)
|
16
|
+
record.do_something
|
17
|
+
"OK"
|
18
|
+
end
|
19
|
+
|
20
|
+
def fetch_records
|
21
|
+
records.shift(2)
|
22
|
+
end
|
23
|
+
|
24
|
+
def total_records
|
25
|
+
records.count
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'processor/observer/null_observer'
|
2
|
+
require 'progressbar'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
module Observer
|
7
|
+
class ProgressBar < Processor::Observer::NullObserver
|
8
|
+
def processing_started
|
9
|
+
@progress_bar = ::ProgressBar.new("Records", processor.total_records)
|
10
|
+
messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
|
11
|
+
end
|
12
|
+
|
13
|
+
def before_record_processing(record)
|
14
|
+
progress_bar.inc
|
15
|
+
end
|
16
|
+
|
17
|
+
def processing_finished
|
18
|
+
progress_bar.finish
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
attr_reader :progress_bar
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Processor
|
2
|
+
class DataProcessor
|
3
|
+
def done?(records)
|
4
|
+
records.count < 1
|
5
|
+
end
|
6
|
+
|
7
|
+
def process(record)
|
8
|
+
raise NotImplementedError
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_records
|
12
|
+
raise NotImplementedError
|
13
|
+
end
|
14
|
+
|
15
|
+
def total_records
|
16
|
+
raise NotImplementedError
|
17
|
+
end
|
18
|
+
|
19
|
+
def name
|
20
|
+
# underscore a class name
|
21
|
+
self.class.name.to_s.
|
22
|
+
gsub(/::/, '_').
|
23
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
24
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
25
|
+
downcase
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Processor
|
2
|
+
class EventsRegistrator
|
3
|
+
def initialize(observers)
|
4
|
+
@observers = observers
|
5
|
+
end
|
6
|
+
|
7
|
+
def register(event, *data)
|
8
|
+
observers.each do |observer|
|
9
|
+
observer.update event, *data
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
attr_reader :observers
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require_relative 'null_observer'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Observer
|
6
|
+
class Logger < NullObserver
|
7
|
+
def initialize(logger = nil, options = {})
|
8
|
+
@logger_source = logger
|
9
|
+
super options
|
10
|
+
end
|
11
|
+
|
12
|
+
def processing_started(processor)
|
13
|
+
initialize_logger(processor)
|
14
|
+
|
15
|
+
message = "Processing of #{processor.name} started."
|
16
|
+
logger.info message
|
17
|
+
messenger.info message
|
18
|
+
|
19
|
+
message = <<-MESSAGE.gsub(/^\s+/, '')
|
20
|
+
Proggress will be saved to the log file. Run
|
21
|
+
tail -f #{log_file_name}
|
22
|
+
to see log in realtime
|
23
|
+
MESSAGE
|
24
|
+
messenger.info message if use_log_file?
|
25
|
+
end
|
26
|
+
|
27
|
+
def before_record_processing(record)
|
28
|
+
message = "Record #{id_for record} is going to be processed"
|
29
|
+
logger.debug message
|
30
|
+
messenger.debug message
|
31
|
+
end
|
32
|
+
|
33
|
+
def after_record_processing(record, result)
|
34
|
+
message = "Successfully processed #{id_for record}: #{result}"
|
35
|
+
logger.info message
|
36
|
+
messenger.debug message
|
37
|
+
end
|
38
|
+
|
39
|
+
def processing_finished(processor)
|
40
|
+
message = "Processing of #{processor.name} finished."
|
41
|
+
logger.info message
|
42
|
+
messenger.info message
|
43
|
+
messenger.info "Log file saved to #{log_file_name}" if use_log_file?
|
44
|
+
end
|
45
|
+
|
46
|
+
def record_processing_error(record, exception)
|
47
|
+
message = "Error processing #{id_for record}: #{exception}"
|
48
|
+
logger.error message
|
49
|
+
messenger.error message
|
50
|
+
end
|
51
|
+
|
52
|
+
def processing_error(processor, exception)
|
53
|
+
message = "Processing #{processor.name} failed: #{exception}"
|
54
|
+
logger.fatal message
|
55
|
+
messenger.fatal message
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
attr_reader :logger, :log_file_name
|
60
|
+
|
61
|
+
def initialize_logger(processor)
|
62
|
+
@logger =
|
63
|
+
if @logger_source.is_a? Proc
|
64
|
+
@logger_source.call processor.name
|
65
|
+
else
|
66
|
+
@logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
|
67
|
+
logger.level = ::Logger::INFO
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def create_log_filename(processor_name)
|
73
|
+
@log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
|
74
|
+
end
|
75
|
+
|
76
|
+
def use_log_file?
|
77
|
+
not log_file_name.nil?
|
78
|
+
end
|
79
|
+
|
80
|
+
def current_time_string
|
81
|
+
Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
|
82
|
+
end
|
83
|
+
|
84
|
+
def id_for record
|
85
|
+
[:uid, :id, :to_token, :token, :to_sym].each do |method|
|
86
|
+
return record.public_send method if record.respond_to? method
|
87
|
+
end
|
88
|
+
|
89
|
+
[:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
|
90
|
+
return record[method] if record.key? method
|
91
|
+
return record[method.to_s] if record.key? method.to_s
|
92
|
+
end if record.respond_to?(:key?) && record.respond_to?(:[])
|
93
|
+
|
94
|
+
record.to_s
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Observer
|
5
|
+
class NullObserver
|
6
|
+
def initialize(options = {})
|
7
|
+
@messenger = options.fetch :messenger do
|
8
|
+
::Logger.new(STDOUT).tap do |logger|
|
9
|
+
logger.formatter = -> _, _, _, msg do
|
10
|
+
"> #{msg}\n"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def method_missing(*); end
|
17
|
+
alias_method :update, :send
|
18
|
+
|
19
|
+
private
|
20
|
+
attr_reader :messenger
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require_relative "events_registrator"
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
class ThreadRunner
|
5
|
+
def initialize(*observers)
|
6
|
+
@observers = observers
|
7
|
+
end
|
8
|
+
|
9
|
+
# This method is thread-safe. But not observers.
|
10
|
+
# Consider creating new runner for a thread or use thread safe observers
|
11
|
+
def run(processor)
|
12
|
+
events = events_registrator
|
13
|
+
events.register :processing_started, processor
|
14
|
+
|
15
|
+
records_ran = 0
|
16
|
+
until processor.done?(records = processor.fetch_records)
|
17
|
+
threads = []
|
18
|
+
begin
|
19
|
+
records.each do |record|
|
20
|
+
recursion_preventer processor do
|
21
|
+
records_ran += 1
|
22
|
+
end
|
23
|
+
|
24
|
+
threads << Thread.new(processor, record) do |thread_data_processor, thread_record|
|
25
|
+
begin
|
26
|
+
events.register :before_record_processing, thread_record
|
27
|
+
|
28
|
+
result = thread_data_processor.process(thread_record)
|
29
|
+
|
30
|
+
events.register :after_record_processing, thread_record, result
|
31
|
+
rescue RuntimeError => exception
|
32
|
+
events.register :record_processing_error, thread_record, exception
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
ensure # join already created threads even if recursion was detected
|
37
|
+
threads.each(&:join)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
events.register :processing_finished, processor
|
42
|
+
rescue Exception => exception
|
43
|
+
events.register :processing_error, processor, exception
|
44
|
+
raise exception
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
attr_reader :observers
|
49
|
+
|
50
|
+
def recursion_preventer(processor)
|
51
|
+
counter = yield
|
52
|
+
raise Exception, "Processing fall into recursion. Check logs." if counter > (processor.total_records * 1.1).round + 10
|
53
|
+
end
|
54
|
+
|
55
|
+
def events_registrator
|
56
|
+
EventsRegistrator.new observers
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/processor.rb
ADDED
data/processor.gemspec
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'processor/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "processor"
|
8
|
+
gem.version = Processor::VERSION
|
9
|
+
gem.authors = ["Alexander Paramonov"]
|
10
|
+
gem.email = ["alexander.n.paramonov@gmail.com"]
|
11
|
+
gem.summary = %q{Process records one by one}
|
12
|
+
gem.description = %q{Universal processor for data migration}
|
13
|
+
gem.homepage = "http://github.com/AlexParamonov/processor"
|
14
|
+
gem.license = "MIT"
|
15
|
+
|
16
|
+
gem.files = `git ls-files`.split($/)
|
17
|
+
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
19
|
+
gem.require_paths = ["lib"]
|
20
|
+
|
21
|
+
gem.add_development_dependency "rake"
|
22
|
+
gem.add_development_dependency "rspec", ">= 2.6"
|
23
|
+
gem.add_development_dependency "pry"
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require_relative '../example/example_runner'
|
3
|
+
require_relative '../example/migration'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
describe "Example" do
|
7
|
+
before(:each) do
|
8
|
+
records = %w[item1 item2 item3 item4 item5]
|
9
|
+
records.each do |record|
|
10
|
+
record.should_receive(:do_something).once
|
11
|
+
end
|
12
|
+
@migration = Processor::Example::Migration.new records
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should use logger and messenger" do
|
16
|
+
migration_runner = Processor::Example::ExampleRunner.new
|
17
|
+
migration_runner.run @migration
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should run without configuration" do
|
21
|
+
runner = Processor::ThreadRunner.new
|
22
|
+
runner.run @migration
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/events_registrator'
|
3
|
+
|
4
|
+
describe Processor::EventsRegistrator do
|
5
|
+
subject { Processor::EventsRegistrator }
|
6
|
+
|
7
|
+
it "should broadcast events to all observers" do
|
8
|
+
observers = 3.times.map do
|
9
|
+
stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
|
10
|
+
end
|
11
|
+
|
12
|
+
events = subject.new(observers)
|
13
|
+
events.register :test_event
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
describe Processor::Observer::Logger do
|
5
|
+
let(:processor) { stub.as_null_object }
|
6
|
+
let(:no_messages) { ::Logger.new("/dev/null") }
|
7
|
+
|
8
|
+
subject { Processor::Observer::Logger }
|
9
|
+
it "accepts logger builder as parameter" do
|
10
|
+
external_logger = mock
|
11
|
+
logger_observer = subject.new -> name { external_logger }, messenger: no_messages
|
12
|
+
|
13
|
+
external_logger.should_receive(:info)
|
14
|
+
logger_observer.processing_started processor
|
15
|
+
end
|
16
|
+
|
17
|
+
it "accepts logger as parameter" do
|
18
|
+
external_logger = mock
|
19
|
+
logger_observer = subject.new external_logger, messenger: no_messages
|
20
|
+
|
21
|
+
external_logger.should_receive(:info)
|
22
|
+
logger_observer.processing_started processor
|
23
|
+
end
|
24
|
+
|
25
|
+
it "use ruby Logger if no external logger provided" do
|
26
|
+
logger_observer = subject.new nil, messenger: no_messages
|
27
|
+
|
28
|
+
Logger.should_receive(:new).and_return(stub.as_null_object)
|
29
|
+
logger_observer.processing_started processor
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
@@ -0,0 +1,134 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/thread_runner'
|
3
|
+
|
4
|
+
processor = Class.new do
|
5
|
+
def done?(records)
|
6
|
+
records.count < 1
|
7
|
+
end
|
8
|
+
|
9
|
+
def total_records
|
10
|
+
10
|
11
|
+
end
|
12
|
+
end.new
|
13
|
+
|
14
|
+
describe Processor::ThreadRunner do
|
15
|
+
let(:runner) { Processor::ThreadRunner.new }
|
16
|
+
let(:events_registrator) { stub.as_null_object }
|
17
|
+
before(:each) do
|
18
|
+
runner.stub(events_registrator: events_registrator)
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should fetch records from processor till it'll be done" do
|
22
|
+
processor.stub(:done?).and_return(false, false, true)
|
23
|
+
processor.should_receive(:fetch_records).exactly(3).times.and_return([])
|
24
|
+
runner.run processor
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should send each found record to processor" do
|
28
|
+
records = [mock(:one), mock(:two), mock(:three)]
|
29
|
+
records.each { |record| processor.should_receive(:process).with(record) }
|
30
|
+
|
31
|
+
processor.stub(:fetch_records).and_return(records, [])
|
32
|
+
runner.run processor
|
33
|
+
end
|
34
|
+
|
35
|
+
describe "exception handling" do
|
36
|
+
let(:record) { stub }
|
37
|
+
before(:each) do
|
38
|
+
processor.stub(:fetch_records).and_return([record], [record], [])
|
39
|
+
end
|
40
|
+
|
41
|
+
describe "processing a record raised RuntimeError" do
|
42
|
+
it "should continue processing" do
|
43
|
+
processor.should_receive(:process).twice.and_raise(RuntimeError)
|
44
|
+
expect { runner.run processor }.to_not raise_error
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should register a record_processing_error event" do
|
48
|
+
event_registered = false
|
49
|
+
events_registrator.should_receive(:register) do |event_name, failed_record, exception|
|
50
|
+
next if event_name != :record_processing_error
|
51
|
+
event_name.should eq :record_processing_error
|
52
|
+
failed_record.should eq record
|
53
|
+
exception.should be_a RuntimeError
|
54
|
+
event_registered = true
|
55
|
+
end.any_number_of_times
|
56
|
+
|
57
|
+
processor.stub(:process).and_raise(RuntimeError)
|
58
|
+
|
59
|
+
begin
|
60
|
+
runner.run processor
|
61
|
+
rescue Exception; end
|
62
|
+
event_registered.should be_true
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
describe "processing a record raised Exception" do
|
67
|
+
it "should break processing and rerise Exception" do
|
68
|
+
custom_exception = Class.new Exception
|
69
|
+
processor.should_receive(:process).once.and_raise(custom_exception)
|
70
|
+
expect { runner.run processor }.to raise_error(custom_exception)
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should register a processing_error event" do
|
74
|
+
event_registered = false
|
75
|
+
events_registrator.should_receive(:register) do |event_name, current_processor, exception|
|
76
|
+
next if event_name != :processing_error
|
77
|
+
event_name.should eq :processing_error
|
78
|
+
current_processor.should eq processor
|
79
|
+
exception.should be_a Exception
|
80
|
+
event_registered = true
|
81
|
+
end.any_number_of_times
|
82
|
+
|
83
|
+
processor.stub(:process).and_raise(Exception)
|
84
|
+
|
85
|
+
begin
|
86
|
+
runner.run processor
|
87
|
+
rescue Exception; end
|
88
|
+
event_registered.should be_true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe "fetching records raised" do
|
93
|
+
it "should break processing and rerise Exception" do
|
94
|
+
custom_exception = Class.new RuntimeError
|
95
|
+
processor.stub(:fetch_records).and_raise(custom_exception)
|
96
|
+
processor.should_not_receive(:process)
|
97
|
+
expect { runner.run processor }.to raise_error(custom_exception)
|
98
|
+
end
|
99
|
+
|
100
|
+
it "should register a processing_error" do
|
101
|
+
event_registered = false
|
102
|
+
events_registrator.should_receive(:register) do |event_name, processor, exception|
|
103
|
+
next if event_name != :processing_error
|
104
|
+
event_name.should eq :processing_error
|
105
|
+
exception.should be_a RuntimeError
|
106
|
+
event_registered = true
|
107
|
+
end.any_number_of_times
|
108
|
+
|
109
|
+
processor.stub(:fetch_records).and_raise(RuntimeError)
|
110
|
+
|
111
|
+
begin
|
112
|
+
runner.run processor
|
113
|
+
rescue Exception; end
|
114
|
+
event_registered.should be_true
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
describe "recursion" do
|
120
|
+
it "should not allow infinit recursion" do
|
121
|
+
processor.stub(:fetch_records).and_return([:one, :two])
|
122
|
+
processor.should_receive(:process).at_most(100).times
|
123
|
+
expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
|
124
|
+
end
|
125
|
+
|
126
|
+
it "should have 10% + 10 rerurns window" do
|
127
|
+
processor.stub(total_records: 100)
|
128
|
+
processor.stub(:fetch_records).and_return([:one, :two])
|
129
|
+
processor.should_receive(:process).exactly(120).times
|
130
|
+
expect { runner.run processor }.to raise_error(Exception, /Processing fall into recursion/)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
metadata
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: processor
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0.beta
|
5
|
+
prerelease: 6
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Alexander Paramonov
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-05-02 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rake
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rspec
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '2.6'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '2.6'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: pry
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
description: Universal processor for data migration
|
63
|
+
email:
|
64
|
+
- alexander.n.paramonov@gmail.com
|
65
|
+
executables: []
|
66
|
+
extensions: []
|
67
|
+
extra_rdoc_files: []
|
68
|
+
files:
|
69
|
+
- .gitignore
|
70
|
+
- .rspec
|
71
|
+
- .rvmrc
|
72
|
+
- .travis.yml
|
73
|
+
- Gemfile
|
74
|
+
- LICENSE
|
75
|
+
- README.md
|
76
|
+
- Rakefile
|
77
|
+
- example/example_runner.rb
|
78
|
+
- example/migration.rb
|
79
|
+
- example/observer/progress_bar.rb
|
80
|
+
- example/simple_runner.rb
|
81
|
+
- lib/processor.rb
|
82
|
+
- lib/processor/data_processor.rb
|
83
|
+
- lib/processor/events_registrator.rb
|
84
|
+
- lib/processor/observer/logger.rb
|
85
|
+
- lib/processor/observer/null_observer.rb
|
86
|
+
- lib/processor/thread_runner.rb
|
87
|
+
- lib/processor/version.rb
|
88
|
+
- processor.gemspec
|
89
|
+
- spec/example_spec.rb
|
90
|
+
- spec/processor/events_registrator_spec.rb
|
91
|
+
- spec/processor/observer/logger_spec.rb
|
92
|
+
- spec/processor/thread_runner_spec.rb
|
93
|
+
- spec/spec_helper_lite.rb
|
94
|
+
homepage: http://github.com/AlexParamonov/processor
|
95
|
+
licenses:
|
96
|
+
- MIT
|
97
|
+
post_install_message:
|
98
|
+
rdoc_options: []
|
99
|
+
require_paths:
|
100
|
+
- lib
|
101
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
102
|
+
none: false
|
103
|
+
requirements:
|
104
|
+
- - ! '>='
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
+
none: false
|
109
|
+
requirements:
|
110
|
+
- - ! '>'
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: 1.3.1
|
113
|
+
requirements: []
|
114
|
+
rubyforge_project:
|
115
|
+
rubygems_version: 1.8.25
|
116
|
+
signing_key:
|
117
|
+
specification_version: 3
|
118
|
+
summary: Process records one by one
|
119
|
+
test_files:
|
120
|
+
- spec/example_spec.rb
|
121
|
+
- spec/processor/events_registrator_spec.rb
|
122
|
+
- spec/processor/observer/logger_spec.rb
|
123
|
+
- spec/processor/thread_runner_spec.rb
|
124
|
+
- spec/spec_helper_lite.rb
|