processor 0.0.0.initial → 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.rspec +2 -0
- data/LICENSE +1 -1
- data/README.md +35 -3
- data/example/logging_thread.rb +15 -0
- data/example/migration.rb +29 -0
- data/example/migrator.rb +66 -0
- data/example/observer/progress_bar.rb +26 -0
- data/example/solr_migration.rb +31 -0
- data/example/solr_pages_migration.rb +48 -0
- data/lib/processor/data_processor.rb +28 -0
- data/lib/processor/events_registrator.rb +16 -0
- data/lib/processor/observer/logger.rb +86 -0
- data/lib/processor/observer/null_observer.rb +24 -0
- data/lib/processor/records_processor/successive.rb +20 -0
- data/lib/processor/records_processor/threads.rb +27 -0
- data/lib/processor/runner.rb +39 -0
- data/lib/processor/thread.rb +26 -0
- data/lib/processor/version.rb +1 -1
- data/spec/example_spec.rb +18 -0
- data/spec/processor/data_processor_spec.rb +20 -0
- data/spec/processor/events_registrator_spec.rb +15 -0
- data/spec/processor/observer/logger_spec.rb +32 -0
- data/spec/processor/records_processor/specs.rb +38 -0
- data/spec/processor/records_processor/successive_spec.rb +7 -0
- data/spec/processor/records_processor/threads_spec.rb +8 -0
- data/spec/processor/runner_spec.rb +103 -0
- data/spec/processor/thread_spec.rb +37 -0
- metadata +44 -5
data/.rspec
ADDED
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -8,9 +8,10 @@ Universal processor for data migration
|
|
8
8
|
Contents
|
9
9
|
---------
|
10
10
|
1. Installation
|
11
|
-
1. Contributing
|
12
11
|
1. Requirements
|
12
|
+
1. Usage
|
13
13
|
1. Compatibility
|
14
|
+
1. Contributing
|
14
15
|
1. Copyright
|
15
16
|
|
16
17
|
Installation
|
@@ -35,9 +36,40 @@ gem install processor
|
|
35
36
|
|
36
37
|
Requirements
|
37
38
|
------------
|
38
|
-
|
39
|
+
1. Ruby 1.9
|
40
|
+
1. Rspec2 for testing
|
41
|
+
|
42
|
+
Usage
|
43
|
+
------------
|
44
|
+
1. Implement a `DataProcessor`. See `Processor::Example::Migration`, `Processor::Example::SolrMigration`, `Processor::Example::SolrPagesMigration`
|
45
|
+
1. Run your `DataProcessor`:
|
46
|
+
|
47
|
+
``` ruby
|
48
|
+
thread = Processor::Thread.new data_processor
|
49
|
+
thread.run_successive
|
50
|
+
```
|
51
|
+
See `spec/processor/thread_spec.rb` and `spec/example_spec.rb` and `example` directory for other usage examples.
|
52
|
+
Below are specs for most valuable parts of the gem.
|
53
|
+
|
54
|
+
``` rspec
|
55
|
+
Processor::Thread
|
56
|
+
should run a migration using provided block
|
57
|
+
should run a migration successive
|
58
|
+
should run a migration in threads
|
39
59
|
|
40
|
-
|
60
|
+
Processor::DataProcessor
|
61
|
+
should have a name equals to underscored class name
|
62
|
+
should be done when there are 0 records to process
|
63
|
+
should respond to done?
|
64
|
+
should respond to fetch_records
|
65
|
+
should respond to total_records
|
66
|
+
should respond to process
|
67
|
+
|
68
|
+
Processor::Observer::Logger
|
69
|
+
accepts logger builder as parameter
|
70
|
+
accepts logger as parameter
|
71
|
+
use ruby Logger if no external logger provided
|
72
|
+
```
|
41
73
|
|
42
74
|
Compatibility
|
43
75
|
-------------
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'processor/thread'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
class LoggingThread < Processor::Thread
|
7
|
+
def initialize(processor)
|
8
|
+
super(
|
9
|
+
processor,
|
10
|
+
Processor::Observer::Logger.new
|
11
|
+
)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class Migration < DataProcessor
|
6
|
+
attr_reader :records
|
7
|
+
def initialize(records)
|
8
|
+
@records = records
|
9
|
+
end
|
10
|
+
|
11
|
+
def done?(records)
|
12
|
+
records.count < 1
|
13
|
+
end
|
14
|
+
|
15
|
+
def process(record)
|
16
|
+
record.do_something
|
17
|
+
"OK"
|
18
|
+
end
|
19
|
+
|
20
|
+
def fetch_records
|
21
|
+
records.shift(2)
|
22
|
+
end
|
23
|
+
|
24
|
+
def total_records
|
25
|
+
records.count
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/example/migrator.rb
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'processor/thread'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
class Migrator < Processor::Thread
|
7
|
+
def migrate
|
8
|
+
messenger.info "Migrator start migrating a records one by one at #{Time.now}"
|
9
|
+
run_successive
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(migration)
|
13
|
+
# logger could be an instance of Ruby Logger
|
14
|
+
logger1 = ::Logger.new(STDOUT).tap do |logger|
|
15
|
+
logger.formatter = -> _, _, _, msg do
|
16
|
+
"debug\t< #{msg}\n"
|
17
|
+
end
|
18
|
+
logger.level = ::Logger::DEBUG
|
19
|
+
end
|
20
|
+
|
21
|
+
# Logger could be a lambda
|
22
|
+
# Where "name" is a processor.name
|
23
|
+
logger2 = -> name do
|
24
|
+
::Logger.new(STDOUT).tap do |logger|
|
25
|
+
logger.formatter = -> _, _, _, msg do
|
26
|
+
"info\t< #{msg}\n"
|
27
|
+
end
|
28
|
+
logger.level = ::Logger::INFO
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# logger could be an instance of Rails Logger
|
33
|
+
# logger = Rails.logger
|
34
|
+
# Or be nil
|
35
|
+
# logger = nil
|
36
|
+
# in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
|
37
|
+
|
38
|
+
# You may customize a messenger:
|
39
|
+
messenger = ::Logger.new(STDOUT).tap do |logger|
|
40
|
+
logger.formatter = -> _, _, _, msg do
|
41
|
+
"message\t> #{msg}\n"
|
42
|
+
end
|
43
|
+
logger.level = ::Logger::DEBUG
|
44
|
+
end
|
45
|
+
|
46
|
+
stdout_logger_debug = Processor::Observer::Logger.new(logger1, messenger: messenger)
|
47
|
+
stdout_logger_info = Processor::Observer::Logger.new(logger2)
|
48
|
+
your_custom_observer1 = Observer::NullObserver.new
|
49
|
+
your_custom_observer2 = Observer::NullObserver.new
|
50
|
+
|
51
|
+
super(
|
52
|
+
migration,
|
53
|
+
stdout_logger_debug,
|
54
|
+
stdout_logger_info,
|
55
|
+
your_custom_observer1,
|
56
|
+
your_custom_observer2,
|
57
|
+
)
|
58
|
+
|
59
|
+
@messenger = messenger
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
attr_reader :messenger
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'processor/observer/null_observer'
|
2
|
+
require 'progressbar'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
module Observer
|
7
|
+
class ProgressBar < Processor::Observer::NullObserver
|
8
|
+
def processing_started
|
9
|
+
@progress_bar = ::ProgressBar.new("Records", processor.total_records)
|
10
|
+
messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
|
11
|
+
end
|
12
|
+
|
13
|
+
def before_record_processing(record)
|
14
|
+
progress_bar.inc
|
15
|
+
end
|
16
|
+
|
17
|
+
def processing_finished
|
18
|
+
progress_bar.finish
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
attr_reader :progress_bar
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class SolrMigration < DataProcessor
|
6
|
+
def process(user)
|
7
|
+
user.set_contact_method "Address book"
|
8
|
+
user.save!
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_records
|
12
|
+
query.results
|
13
|
+
end
|
14
|
+
|
15
|
+
def total_records
|
16
|
+
@total_records ||= query.total
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
# query will return 0 records when all users'll be processed successfully
|
21
|
+
def query
|
22
|
+
User.search {
|
23
|
+
fulltext "My company"
|
24
|
+
with :title, "Manager"
|
25
|
+
with :contact_method, "Direct contact"
|
26
|
+
paginate page: 1, per_page: 10
|
27
|
+
}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class SolrPagesMigration < DataProcessor
|
6
|
+
def done?(records)
|
7
|
+
# Optional custom check for migration to be done.
|
8
|
+
|
9
|
+
# your custom check
|
10
|
+
|
11
|
+
# use a default check if you like
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def process(user)
|
16
|
+
user.set_contact_method "Address book"
|
17
|
+
user.save!
|
18
|
+
end
|
19
|
+
|
20
|
+
def fetch_records
|
21
|
+
query(next_page).results
|
22
|
+
end
|
23
|
+
|
24
|
+
def total_records
|
25
|
+
@total_records ||= query(1).total
|
26
|
+
end
|
27
|
+
|
28
|
+
# optional name to use in observers.
|
29
|
+
def name
|
30
|
+
"my_company_users_contact_method_migration"
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
def query(requested_page)
|
35
|
+
User.search {
|
36
|
+
fulltext "My company"
|
37
|
+
paginate page: requested_page, per_page: 10
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
def next_page
|
42
|
+
@page ||= 0
|
43
|
+
@page += 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Processor
|
2
|
+
class DataProcessor
|
3
|
+
def done?(records)
|
4
|
+
records.count < 1
|
5
|
+
end
|
6
|
+
|
7
|
+
def process(record)
|
8
|
+
raise NotImplementedError
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_records
|
12
|
+
raise NotImplementedError
|
13
|
+
end
|
14
|
+
|
15
|
+
def total_records
|
16
|
+
raise NotImplementedError
|
17
|
+
end
|
18
|
+
|
19
|
+
def name
|
20
|
+
# underscore a class name
|
21
|
+
self.class.name.to_s.
|
22
|
+
gsub(/::/, '_').
|
23
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
24
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
25
|
+
downcase
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Processor
|
2
|
+
class EventsRegistrator
|
3
|
+
def initialize(observers)
|
4
|
+
@observers = observers
|
5
|
+
end
|
6
|
+
|
7
|
+
def register(event, *data)
|
8
|
+
observers.each do |observer|
|
9
|
+
observer.update event, *data
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
attr_reader :observers
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require_relative 'null_observer'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Observer
|
6
|
+
class Logger < NullObserver
|
7
|
+
def initialize(logger = nil, options = {})
|
8
|
+
@logger_source = logger
|
9
|
+
super options
|
10
|
+
end
|
11
|
+
|
12
|
+
def processing_started(processor)
|
13
|
+
initialize_logger(processor)
|
14
|
+
logger.info "Processing of #{processor.name} started."
|
15
|
+
|
16
|
+
message = <<-MESSAGE.gsub(/^\s+/, '')
|
17
|
+
Proggress will be saved to the log file. Run
|
18
|
+
tail -f #{log_file_name}
|
19
|
+
to see log in realtime
|
20
|
+
MESSAGE
|
21
|
+
messenger.info message if use_log_file?
|
22
|
+
end
|
23
|
+
|
24
|
+
def before_record_processing(record)
|
25
|
+
logger.debug "Record #{id_for record} is going to be processed"
|
26
|
+
end
|
27
|
+
|
28
|
+
def after_record_processing(record, result)
|
29
|
+
logger.info "Successfully processed #{id_for record}: #{result}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def processing_finished(processor)
|
33
|
+
logger.info "Processing of #{processor.name} finished."
|
34
|
+
messenger.info "Log file saved to #{log_file_name}" if use_log_file?
|
35
|
+
end
|
36
|
+
|
37
|
+
def record_processing_error(record, exception)
|
38
|
+
logger.error "Error processing #{id_for record}: #{exception}"
|
39
|
+
end
|
40
|
+
|
41
|
+
def processing_error(processor, exception)
|
42
|
+
logger.fatal "Processing #{processor.name} failed: #{exception}"
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
attr_reader :logger, :log_file_name
|
47
|
+
|
48
|
+
def initialize_logger(processor)
|
49
|
+
@logger =
|
50
|
+
if @logger_source.is_a? Proc
|
51
|
+
@logger_source.call processor.name
|
52
|
+
else
|
53
|
+
@logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
|
54
|
+
logger.level = ::Logger::INFO
|
55
|
+
end
|
56
|
+
end
|
57
|
+
messenger.debug "Observer initialized with logger #{@logger}"
|
58
|
+
end
|
59
|
+
|
60
|
+
def create_log_filename(processor_name)
|
61
|
+
@log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
|
62
|
+
end
|
63
|
+
|
64
|
+
def use_log_file?
|
65
|
+
not log_file_name.nil?
|
66
|
+
end
|
67
|
+
|
68
|
+
def current_time_string
|
69
|
+
Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
|
70
|
+
end
|
71
|
+
|
72
|
+
def id_for record
|
73
|
+
[:uid, :id, :to_token, :token, :to_sym].each do |method|
|
74
|
+
return record.public_send method if record.respond_to? method
|
75
|
+
end
|
76
|
+
|
77
|
+
[:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
|
78
|
+
return record[method] if record.key? method
|
79
|
+
return record[method.to_s] if record.key? method.to_s
|
80
|
+
end if record.respond_to?(:key?) && record.respond_to?(:[])
|
81
|
+
|
82
|
+
record.to_s
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Observer
|
5
|
+
class NullObserver
|
6
|
+
def initialize(options = {})
|
7
|
+
@messenger = options.fetch :messenger do
|
8
|
+
::Logger.new(STDOUT).tap do |logger|
|
9
|
+
logger.formatter = -> _, _, _, msg do
|
10
|
+
"> #{msg}\n"
|
11
|
+
end
|
12
|
+
logger.level = ::Logger::INFO
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def method_missing(*); end
|
18
|
+
alias_method :update, :send
|
19
|
+
|
20
|
+
private
|
21
|
+
attr_reader :messenger
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Processor
|
2
|
+
module RecordsProcessor
|
3
|
+
class Successive
|
4
|
+
def call(records, processor, events, recursion_preventer)
|
5
|
+
records.each do |record|
|
6
|
+
recursion_preventer.call
|
7
|
+
begin
|
8
|
+
events.register :before_record_processing, record
|
9
|
+
|
10
|
+
result = processor.process(record)
|
11
|
+
|
12
|
+
events.register :after_record_processing, record, result
|
13
|
+
rescue RuntimeError => exception
|
14
|
+
events.register :record_processing_error, record, exception
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Processor
|
2
|
+
module RecordsProcessor
|
3
|
+
class Threads
|
4
|
+
def call(records, processor, events, recursion_preventer)
|
5
|
+
threads = []
|
6
|
+
begin
|
7
|
+
records.each do |record|
|
8
|
+
recursion_preventer.call
|
9
|
+
threads << ::Thread.new(processor, record) do |thread_data_processor, thread_record|
|
10
|
+
begin
|
11
|
+
events.register :before_record_processing, thread_record
|
12
|
+
|
13
|
+
result = thread_data_processor.process(thread_record)
|
14
|
+
|
15
|
+
events.register :after_record_processing, thread_record, result
|
16
|
+
rescue RuntimeError => exception
|
17
|
+
events.register :record_processing_error, thread_record, exception
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
ensure # join already created threads even if recursion was detected
|
22
|
+
threads.each(&:join)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require_relative "events_registrator"
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
class Runner
|
5
|
+
def initialize(processor, events_registrator)
|
6
|
+
@processor = processor
|
7
|
+
@events = events_registrator
|
8
|
+
end
|
9
|
+
|
10
|
+
def run(records_processor)
|
11
|
+
events.register :processing_started, processor
|
12
|
+
until processor.done?(records = processor.fetch_records)
|
13
|
+
records_processor.call records, processor, events, method(:recursion_preventer)
|
14
|
+
end
|
15
|
+
|
16
|
+
events.register :processing_finished, processor
|
17
|
+
rescue Exception => exception
|
18
|
+
events.register :processing_error, processor, exception
|
19
|
+
raise exception
|
20
|
+
end
|
21
|
+
|
22
|
+
protected
|
23
|
+
attr_writer :counter
|
24
|
+
def counter
|
25
|
+
@counter ||= 0
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
attr_reader :events, :processor
|
30
|
+
def recursion_preventer
|
31
|
+
self.counter += 1
|
32
|
+
raise Exception, "Processing fall into recursion. Check logs." if self.counter > max_records_to_process
|
33
|
+
end
|
34
|
+
|
35
|
+
def max_records_to_process
|
36
|
+
(processor.total_records * 1.1).round + 10
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'processor/runner'
|
2
|
+
require 'processor/records_processor/successive'
|
3
|
+
require 'processor/records_processor/threads'
|
4
|
+
|
5
|
+
module Processor
|
6
|
+
class Thread
|
7
|
+
def initialize(data_processor, *observers)
|
8
|
+
@runner = Runner.new data_processor, EventsRegistrator.new(observers)
|
9
|
+
end
|
10
|
+
|
11
|
+
def run_as(&records_processor)
|
12
|
+
runner.run records_processor
|
13
|
+
end
|
14
|
+
|
15
|
+
def run_successive
|
16
|
+
runner.run RecordsProcessor::Successive.new
|
17
|
+
end
|
18
|
+
|
19
|
+
def run_in_threads
|
20
|
+
runner.run RecordsProcessor::Threads.new
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
attr_reader :runner
|
25
|
+
end
|
26
|
+
end
|
data/lib/processor/version.rb
CHANGED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require_relative '../example/migrator'
|
3
|
+
require_relative '../example/migration'
|
4
|
+
|
5
|
+
describe "Example" do
|
6
|
+
before(:each) do
|
7
|
+
records = %w[item1 item2 item3 item4 item5]
|
8
|
+
records.each do |record|
|
9
|
+
record.should_receive(:do_something).once
|
10
|
+
end
|
11
|
+
@migration = Processor::Example::Migration.new records
|
12
|
+
end
|
13
|
+
|
14
|
+
it "migration should use 2 loggers and messenger" do
|
15
|
+
migrator = Processor::Example::Migrator.new @migration
|
16
|
+
migrator.migrate
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/data_processor'
|
3
|
+
|
4
|
+
describe Processor::DataProcessor do
|
5
|
+
it "should have a name equals to underscored class name" do
|
6
|
+
subject.name.should eq "processor_data_processor"
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should be done when there are 0 records to process" do
|
10
|
+
records = []
|
11
|
+
subject.done?(records).should be true
|
12
|
+
end
|
13
|
+
|
14
|
+
%w[done? fetch_records total_records process].each do |method_name|
|
15
|
+
it "should respond to #{method_name}" do
|
16
|
+
subject.should respond_to method_name
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/events_registrator'
|
3
|
+
|
4
|
+
describe Processor::EventsRegistrator do
|
5
|
+
subject { Processor::EventsRegistrator }
|
6
|
+
|
7
|
+
it "should broadcast events to all observers" do
|
8
|
+
observers = 3.times.map do
|
9
|
+
stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
|
10
|
+
end
|
11
|
+
|
12
|
+
events = subject.new(observers)
|
13
|
+
events.register :test_event
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
describe Processor::Observer::Logger do
|
5
|
+
let(:processor) { stub.as_null_object }
|
6
|
+
let(:no_messages) { ::Logger.new("/dev/null") }
|
7
|
+
|
8
|
+
subject { Processor::Observer::Logger }
|
9
|
+
it "accepts logger builder as parameter" do
|
10
|
+
external_logger = mock
|
11
|
+
logger_observer = subject.new -> name { external_logger }, messenger: no_messages
|
12
|
+
|
13
|
+
external_logger.should_receive(:info)
|
14
|
+
logger_observer.processing_started processor
|
15
|
+
end
|
16
|
+
|
17
|
+
it "accepts logger as parameter" do
|
18
|
+
external_logger = mock
|
19
|
+
logger_observer = subject.new external_logger, messenger: no_messages
|
20
|
+
|
21
|
+
external_logger.should_receive(:info)
|
22
|
+
logger_observer.processing_started processor
|
23
|
+
end
|
24
|
+
|
25
|
+
it "use ruby Logger if no external logger provided" do
|
26
|
+
logger_observer = subject.new nil, messenger: no_messages
|
27
|
+
|
28
|
+
Logger.should_receive(:new).and_return(stub.as_null_object)
|
29
|
+
logger_observer.processing_started processor
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
shared_examples_for "a records processor" do
|
2
|
+
let(:records_processor) { described_class.new }
|
3
|
+
let(:records) { 1..2 }
|
4
|
+
let(:processor) { stub }
|
5
|
+
|
6
|
+
let(:no_recursion_preventer) { Proc.new{} }
|
7
|
+
let(:no_events) { stub.as_null_object }
|
8
|
+
|
9
|
+
it "should send each found record to processor" do
|
10
|
+
records.each { |record| processor.should_receive(:process).with(record) }
|
11
|
+
records_processor.call records, processor, no_events, no_recursion_preventer
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "exception handling" do
|
15
|
+
describe "processing a record raised RuntimeError" do
|
16
|
+
it "should continue processing" do
|
17
|
+
processor.should_receive(:process).twice.and_raise(RuntimeError)
|
18
|
+
expect { records_processor.call records, processor, no_events, no_recursion_preventer }.to_not raise_error
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should register a record_processing_error event" do
|
22
|
+
event = mock.tap { |event| event.should_receive(:trigger).any_number_of_times }
|
23
|
+
|
24
|
+
events_registrator = stub
|
25
|
+
events_registrator.should_receive(:register) do |event_name, failed_record, exception|
|
26
|
+
next if event_name != :record_processing_error
|
27
|
+
event_name.should eq :record_processing_error
|
28
|
+
exception.should be_a RuntimeError
|
29
|
+
event.trigger
|
30
|
+
end.any_number_of_times
|
31
|
+
|
32
|
+
processor.stub(:process).and_raise(RuntimeError)
|
33
|
+
|
34
|
+
records_processor.call records, processor, events_registrator, no_recursion_preventer rescue nil
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/runner'
|
3
|
+
|
4
|
+
describe Processor::Runner do
|
5
|
+
let(:runner) { Processor::Runner.new(processor, events_registrator) }
|
6
|
+
let(:processor) { stub }
|
7
|
+
let(:events_registrator) { stub.as_null_object }
|
8
|
+
let(:no_records_processor) { Proc.new{} }
|
9
|
+
|
10
|
+
it "should fetch records from processor till it'll be done" do
|
11
|
+
processor.stub(:done?).and_return(false, false, true)
|
12
|
+
processor.should_receive(:fetch_records).exactly(3).times.and_return([])
|
13
|
+
runner.run no_records_processor
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "exception handling" do
|
17
|
+
let(:record) { stub }
|
18
|
+
before(:each) do
|
19
|
+
processor.stub(:fetch_records).and_return([record], [record], [])
|
20
|
+
end
|
21
|
+
|
22
|
+
describe "processing records raised" do
|
23
|
+
before(:each) do
|
24
|
+
processor.stub(done?: false)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should break processing and rerise" do
|
28
|
+
expect do
|
29
|
+
runner.run Proc.new { raise RuntimeError }
|
30
|
+
end.to raise_error(RuntimeError)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should register a processing_error event" do
|
34
|
+
register_processing_error_event mock.tap { |event| event.should_receive :trigger }
|
35
|
+
runner.run Proc.new { raise RuntimeError } rescue nil
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
describe "fetching records raised" do
|
40
|
+
it "should break processing and rerise Exception" do
|
41
|
+
processor.stub(:fetch_records).and_raise(RuntimeError)
|
42
|
+
processor.should_not_receive(:process)
|
43
|
+
expect { runner.run no_records_processor }.to raise_error(RuntimeError)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should register a processing_error" do
|
47
|
+
register_processing_error_event mock.tap { |event| event.should_receive :trigger }
|
48
|
+
|
49
|
+
processor.stub(:fetch_records).and_raise(RuntimeError)
|
50
|
+
runner.run no_records_processor rescue nil
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
def register_processing_error_event(event)
|
56
|
+
# Check that processing_error event was register
|
57
|
+
events_registrator.should_receive(:register) do |event_name, current_processor, exception|
|
58
|
+
next if event_name != :processing_error
|
59
|
+
event_name.should eq :processing_error
|
60
|
+
current_processor.should eq processor
|
61
|
+
exception.should be_a RuntimeError
|
62
|
+
event.trigger
|
63
|
+
end.any_number_of_times
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe "recursion" do
|
68
|
+
before(:each) do
|
69
|
+
processor.stub(done?: false)
|
70
|
+
processor.stub(total_records: 100)
|
71
|
+
processor.stub(fetch_records: 1..3)
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should not fall into recursion" do
|
75
|
+
processor.should_receive(:process).at_most(1000).times
|
76
|
+
|
77
|
+
expect do
|
78
|
+
records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
|
79
|
+
records.each do |record|
|
80
|
+
recursion_preventer.call
|
81
|
+
records_processor.process record
|
82
|
+
end
|
83
|
+
end
|
84
|
+
runner.run records_processor
|
85
|
+
end.to raise_error(Exception, /Processing fall into recursion/)
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should have 10% + 10 rerurns window" do
|
89
|
+
processor.should_receive(:process).exactly(120).times
|
90
|
+
|
91
|
+
expect do
|
92
|
+
records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
|
93
|
+
records.each do |record|
|
94
|
+
recursion_preventer.call
|
95
|
+
processor.process record
|
96
|
+
end
|
97
|
+
end
|
98
|
+
runner.run records_processor
|
99
|
+
end.to raise_error(Exception, /Processing fall into recursion/)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/thread'
|
3
|
+
require_relative '../../example/migration'
|
4
|
+
|
5
|
+
describe Processor::Thread do
|
6
|
+
before(:each) do
|
7
|
+
records = %w[item1 item2 item3 item4 item5]
|
8
|
+
records.each do |record|
|
9
|
+
record.should_receive(:do_something).once
|
10
|
+
end
|
11
|
+
@migration = Processor::Example::Migration.new records
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should run a migration using provided block" do
|
15
|
+
thread = Processor::Thread.new @migration
|
16
|
+
thread.run_as do |records, processor, *|
|
17
|
+
records.each do |record|
|
18
|
+
processor.process record
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should run a migration successive" do
|
24
|
+
thread = Processor::Thread.new @migration
|
25
|
+
thread.run_successive
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should run a migration in threads" do
|
29
|
+
thread = Processor::Thread.new @migration
|
30
|
+
thread.run_in_threads
|
31
|
+
end
|
32
|
+
|
33
|
+
pending "should run a migration in specifien number of threads" do
|
34
|
+
thread = Processor::Thread.new @migration
|
35
|
+
thread.run_in_threads 3
|
36
|
+
end
|
37
|
+
end
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: processor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Alexander Paramonov
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-05-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -67,15 +67,39 @@ extensions: []
|
|
67
67
|
extra_rdoc_files: []
|
68
68
|
files:
|
69
69
|
- .gitignore
|
70
|
+
- .rspec
|
70
71
|
- .rvmrc
|
71
72
|
- .travis.yml
|
72
73
|
- Gemfile
|
73
74
|
- LICENSE
|
74
75
|
- README.md
|
75
76
|
- Rakefile
|
77
|
+
- example/logging_thread.rb
|
78
|
+
- example/migration.rb
|
79
|
+
- example/migrator.rb
|
80
|
+
- example/observer/progress_bar.rb
|
81
|
+
- example/solr_migration.rb
|
82
|
+
- example/solr_pages_migration.rb
|
76
83
|
- lib/processor.rb
|
84
|
+
- lib/processor/data_processor.rb
|
85
|
+
- lib/processor/events_registrator.rb
|
86
|
+
- lib/processor/observer/logger.rb
|
87
|
+
- lib/processor/observer/null_observer.rb
|
88
|
+
- lib/processor/records_processor/successive.rb
|
89
|
+
- lib/processor/records_processor/threads.rb
|
90
|
+
- lib/processor/runner.rb
|
91
|
+
- lib/processor/thread.rb
|
77
92
|
- lib/processor/version.rb
|
78
93
|
- processor.gemspec
|
94
|
+
- spec/example_spec.rb
|
95
|
+
- spec/processor/data_processor_spec.rb
|
96
|
+
- spec/processor/events_registrator_spec.rb
|
97
|
+
- spec/processor/observer/logger_spec.rb
|
98
|
+
- spec/processor/records_processor/specs.rb
|
99
|
+
- spec/processor/records_processor/successive_spec.rb
|
100
|
+
- spec/processor/records_processor/threads_spec.rb
|
101
|
+
- spec/processor/runner_spec.rb
|
102
|
+
- spec/processor/thread_spec.rb
|
79
103
|
- spec/spec_helper_lite.rb
|
80
104
|
homepage: http://github.com/AlexParamonov/processor
|
81
105
|
licenses:
|
@@ -90,12 +114,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
90
114
|
- - ! '>='
|
91
115
|
- !ruby/object:Gem::Version
|
92
116
|
version: '0'
|
117
|
+
segments:
|
118
|
+
- 0
|
119
|
+
hash: 4470760754938067473
|
93
120
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
94
121
|
none: false
|
95
122
|
requirements:
|
96
|
-
- - ! '
|
123
|
+
- - ! '>='
|
97
124
|
- !ruby/object:Gem::Version
|
98
|
-
version:
|
125
|
+
version: '0'
|
126
|
+
segments:
|
127
|
+
- 0
|
128
|
+
hash: 4470760754938067473
|
99
129
|
requirements: []
|
100
130
|
rubyforge_project:
|
101
131
|
rubygems_version: 1.8.25
|
@@ -103,4 +133,13 @@ signing_key:
|
|
103
133
|
specification_version: 3
|
104
134
|
summary: Process records one by one
|
105
135
|
test_files:
|
136
|
+
- spec/example_spec.rb
|
137
|
+
- spec/processor/data_processor_spec.rb
|
138
|
+
- spec/processor/events_registrator_spec.rb
|
139
|
+
- spec/processor/observer/logger_spec.rb
|
140
|
+
- spec/processor/records_processor/specs.rb
|
141
|
+
- spec/processor/records_processor/successive_spec.rb
|
142
|
+
- spec/processor/records_processor/threads_spec.rb
|
143
|
+
- spec/processor/runner_spec.rb
|
144
|
+
- spec/processor/thread_spec.rb
|
106
145
|
- spec/spec_helper_lite.rb
|