processor 0.0.0.initial → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rspec +2 -0
- data/LICENSE +1 -1
- data/README.md +35 -3
- data/example/logging_thread.rb +15 -0
- data/example/migration.rb +29 -0
- data/example/migrator.rb +66 -0
- data/example/observer/progress_bar.rb +26 -0
- data/example/solr_migration.rb +31 -0
- data/example/solr_pages_migration.rb +48 -0
- data/lib/processor/data_processor.rb +28 -0
- data/lib/processor/events_registrator.rb +16 -0
- data/lib/processor/observer/logger.rb +86 -0
- data/lib/processor/observer/null_observer.rb +24 -0
- data/lib/processor/records_processor/successive.rb +20 -0
- data/lib/processor/records_processor/threads.rb +27 -0
- data/lib/processor/runner.rb +39 -0
- data/lib/processor/thread.rb +26 -0
- data/lib/processor/version.rb +1 -1
- data/spec/example_spec.rb +18 -0
- data/spec/processor/data_processor_spec.rb +20 -0
- data/spec/processor/events_registrator_spec.rb +15 -0
- data/spec/processor/observer/logger_spec.rb +32 -0
- data/spec/processor/records_processor/specs.rb +38 -0
- data/spec/processor/records_processor/successive_spec.rb +7 -0
- data/spec/processor/records_processor/threads_spec.rb +8 -0
- data/spec/processor/runner_spec.rb +103 -0
- data/spec/processor/thread_spec.rb +37 -0
- metadata +44 -5
data/.rspec
ADDED
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -8,9 +8,10 @@ Universal processor for data migration
|
|
8
8
|
Contents
|
9
9
|
---------
|
10
10
|
1. Installation
|
11
|
-
1. Contributing
|
12
11
|
1. Requirements
|
12
|
+
1. Usage
|
13
13
|
1. Compatibility
|
14
|
+
1. Contributing
|
14
15
|
1. Copyright
|
15
16
|
|
16
17
|
Installation
|
@@ -35,9 +36,40 @@ gem install processor
|
|
35
36
|
|
36
37
|
Requirements
|
37
38
|
------------
|
38
|
-
|
39
|
+
1. Ruby 1.9
|
40
|
+
1. Rspec2 for testing
|
41
|
+
|
42
|
+
Usage
|
43
|
+
------------
|
44
|
+
1. Implement a `DataProcessor`. See `Processor::Example::Migration`, `Processor::Example::SolrMigration`, `Processor::Example::SolrPagesMigration`
|
45
|
+
1. Run your `DataProcessor`:
|
46
|
+
|
47
|
+
``` ruby
|
48
|
+
thread = Processor::Thread.new data_processor
|
49
|
+
thread.run_successive
|
50
|
+
```
|
51
|
+
See `spec/processor/thread_spec.rb` and `spec/example_spec.rb` and `example` directory for other usage examples.
|
52
|
+
Below are specs for most valuable parts of the gem.
|
53
|
+
|
54
|
+
``` rspec
|
55
|
+
Processor::Thread
|
56
|
+
should run a migration using provided block
|
57
|
+
should run a migration successive
|
58
|
+
should run a migration in threads
|
39
59
|
|
40
|
-
|
60
|
+
Processor::DataProcessor
|
61
|
+
should have a name equals to underscored class name
|
62
|
+
should be done when there are 0 records to process
|
63
|
+
should respond to done?
|
64
|
+
should respond to fetch_records
|
65
|
+
should respond to total_records
|
66
|
+
should respond to process
|
67
|
+
|
68
|
+
Processor::Observer::Logger
|
69
|
+
accepts logger builder as parameter
|
70
|
+
accepts logger as parameter
|
71
|
+
use ruby Logger if no external logger provided
|
72
|
+
```
|
41
73
|
|
42
74
|
Compatibility
|
43
75
|
-------------
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'processor/thread'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
class LoggingThread < Processor::Thread
|
7
|
+
def initialize(processor)
|
8
|
+
super(
|
9
|
+
processor,
|
10
|
+
Processor::Observer::Logger.new
|
11
|
+
)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class Migration < DataProcessor
|
6
|
+
attr_reader :records
|
7
|
+
def initialize(records)
|
8
|
+
@records = records
|
9
|
+
end
|
10
|
+
|
11
|
+
def done?(records)
|
12
|
+
records.count < 1
|
13
|
+
end
|
14
|
+
|
15
|
+
def process(record)
|
16
|
+
record.do_something
|
17
|
+
"OK"
|
18
|
+
end
|
19
|
+
|
20
|
+
def fetch_records
|
21
|
+
records.shift(2)
|
22
|
+
end
|
23
|
+
|
24
|
+
def total_records
|
25
|
+
records.count
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/example/migrator.rb
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'processor/thread'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
class Migrator < Processor::Thread
|
7
|
+
def migrate
|
8
|
+
messenger.info "Migrator start migrating a records one by one at #{Time.now}"
|
9
|
+
run_successive
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(migration)
|
13
|
+
# logger could be an instance of Ruby Logger
|
14
|
+
logger1 = ::Logger.new(STDOUT).tap do |logger|
|
15
|
+
logger.formatter = -> _, _, _, msg do
|
16
|
+
"debug\t< #{msg}\n"
|
17
|
+
end
|
18
|
+
logger.level = ::Logger::DEBUG
|
19
|
+
end
|
20
|
+
|
21
|
+
# Logger could be a lambda
|
22
|
+
# Where "name" is a processor.name
|
23
|
+
logger2 = -> name do
|
24
|
+
::Logger.new(STDOUT).tap do |logger|
|
25
|
+
logger.formatter = -> _, _, _, msg do
|
26
|
+
"info\t< #{msg}\n"
|
27
|
+
end
|
28
|
+
logger.level = ::Logger::INFO
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# logger could be an instance of Rails Logger
|
33
|
+
# logger = Rails.logger
|
34
|
+
# Or be nil
|
35
|
+
# logger = nil
|
36
|
+
# in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
|
37
|
+
|
38
|
+
# You may customize a messenger:
|
39
|
+
messenger = ::Logger.new(STDOUT).tap do |logger|
|
40
|
+
logger.formatter = -> _, _, _, msg do
|
41
|
+
"message\t> #{msg}\n"
|
42
|
+
end
|
43
|
+
logger.level = ::Logger::DEBUG
|
44
|
+
end
|
45
|
+
|
46
|
+
stdout_logger_debug = Processor::Observer::Logger.new(logger1, messenger: messenger)
|
47
|
+
stdout_logger_info = Processor::Observer::Logger.new(logger2)
|
48
|
+
your_custom_observer1 = Observer::NullObserver.new
|
49
|
+
your_custom_observer2 = Observer::NullObserver.new
|
50
|
+
|
51
|
+
super(
|
52
|
+
migration,
|
53
|
+
stdout_logger_debug,
|
54
|
+
stdout_logger_info,
|
55
|
+
your_custom_observer1,
|
56
|
+
your_custom_observer2,
|
57
|
+
)
|
58
|
+
|
59
|
+
@messenger = messenger
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
attr_reader :messenger
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'processor/observer/null_observer'
|
2
|
+
require 'progressbar'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Example
|
6
|
+
module Observer
|
7
|
+
class ProgressBar < Processor::Observer::NullObserver
|
8
|
+
def processing_started
|
9
|
+
@progress_bar = ::ProgressBar.new("Records", processor.total_records)
|
10
|
+
messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
|
11
|
+
end
|
12
|
+
|
13
|
+
def before_record_processing(record)
|
14
|
+
progress_bar.inc
|
15
|
+
end
|
16
|
+
|
17
|
+
def processing_finished
|
18
|
+
progress_bar.finish
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
attr_reader :progress_bar
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class SolrMigration < DataProcessor
|
6
|
+
def process(user)
|
7
|
+
user.set_contact_method "Address book"
|
8
|
+
user.save!
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_records
|
12
|
+
query.results
|
13
|
+
end
|
14
|
+
|
15
|
+
def total_records
|
16
|
+
@total_records ||= query.total
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
# query will return 0 records when all users'll be processed successfully
|
21
|
+
def query
|
22
|
+
User.search {
|
23
|
+
fulltext "My company"
|
24
|
+
with :title, "Manager"
|
25
|
+
with :contact_method, "Direct contact"
|
26
|
+
paginate page: 1, per_page: 10
|
27
|
+
}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'processor/data_processor'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Example
|
5
|
+
class SolrPagesMigration < DataProcessor
|
6
|
+
def done?(records)
|
7
|
+
# Optional custom check for migration to be done.
|
8
|
+
|
9
|
+
# your custom check
|
10
|
+
|
11
|
+
# use a default check if you like
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def process(user)
|
16
|
+
user.set_contact_method "Address book"
|
17
|
+
user.save!
|
18
|
+
end
|
19
|
+
|
20
|
+
def fetch_records
|
21
|
+
query(next_page).results
|
22
|
+
end
|
23
|
+
|
24
|
+
def total_records
|
25
|
+
@total_records ||= query(1).total
|
26
|
+
end
|
27
|
+
|
28
|
+
# optional name to use in observers.
|
29
|
+
def name
|
30
|
+
"my_company_users_contact_method_migration"
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
def query(requested_page)
|
35
|
+
User.search {
|
36
|
+
fulltext "My company"
|
37
|
+
paginate page: requested_page, per_page: 10
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
def next_page
|
42
|
+
@page ||= 0
|
43
|
+
@page += 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Processor
|
2
|
+
class DataProcessor
|
3
|
+
def done?(records)
|
4
|
+
records.count < 1
|
5
|
+
end
|
6
|
+
|
7
|
+
def process(record)
|
8
|
+
raise NotImplementedError
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_records
|
12
|
+
raise NotImplementedError
|
13
|
+
end
|
14
|
+
|
15
|
+
def total_records
|
16
|
+
raise NotImplementedError
|
17
|
+
end
|
18
|
+
|
19
|
+
def name
|
20
|
+
# underscore a class name
|
21
|
+
self.class.name.to_s.
|
22
|
+
gsub(/::/, '_').
|
23
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
24
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
25
|
+
downcase
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Processor
|
2
|
+
class EventsRegistrator
|
3
|
+
def initialize(observers)
|
4
|
+
@observers = observers
|
5
|
+
end
|
6
|
+
|
7
|
+
def register(event, *data)
|
8
|
+
observers.each do |observer|
|
9
|
+
observer.update event, *data
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
attr_reader :observers
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require_relative 'null_observer'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Processor
|
5
|
+
module Observer
|
6
|
+
class Logger < NullObserver
|
7
|
+
def initialize(logger = nil, options = {})
|
8
|
+
@logger_source = logger
|
9
|
+
super options
|
10
|
+
end
|
11
|
+
|
12
|
+
def processing_started(processor)
|
13
|
+
initialize_logger(processor)
|
14
|
+
logger.info "Processing of #{processor.name} started."
|
15
|
+
|
16
|
+
message = <<-MESSAGE.gsub(/^\s+/, '')
|
17
|
+
Proggress will be saved to the log file. Run
|
18
|
+
tail -f #{log_file_name}
|
19
|
+
to see log in realtime
|
20
|
+
MESSAGE
|
21
|
+
messenger.info message if use_log_file?
|
22
|
+
end
|
23
|
+
|
24
|
+
def before_record_processing(record)
|
25
|
+
logger.debug "Record #{id_for record} is going to be processed"
|
26
|
+
end
|
27
|
+
|
28
|
+
def after_record_processing(record, result)
|
29
|
+
logger.info "Successfully processed #{id_for record}: #{result}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def processing_finished(processor)
|
33
|
+
logger.info "Processing of #{processor.name} finished."
|
34
|
+
messenger.info "Log file saved to #{log_file_name}" if use_log_file?
|
35
|
+
end
|
36
|
+
|
37
|
+
def record_processing_error(record, exception)
|
38
|
+
logger.error "Error processing #{id_for record}: #{exception}"
|
39
|
+
end
|
40
|
+
|
41
|
+
def processing_error(processor, exception)
|
42
|
+
logger.fatal "Processing #{processor.name} failed: #{exception}"
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
attr_reader :logger, :log_file_name
|
47
|
+
|
48
|
+
def initialize_logger(processor)
|
49
|
+
@logger =
|
50
|
+
if @logger_source.is_a? Proc
|
51
|
+
@logger_source.call processor.name
|
52
|
+
else
|
53
|
+
@logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
|
54
|
+
logger.level = ::Logger::INFO
|
55
|
+
end
|
56
|
+
end
|
57
|
+
messenger.debug "Observer initialized with logger #{@logger}"
|
58
|
+
end
|
59
|
+
|
60
|
+
def create_log_filename(processor_name)
|
61
|
+
@log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
|
62
|
+
end
|
63
|
+
|
64
|
+
def use_log_file?
|
65
|
+
not log_file_name.nil?
|
66
|
+
end
|
67
|
+
|
68
|
+
def current_time_string
|
69
|
+
Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
|
70
|
+
end
|
71
|
+
|
72
|
+
def id_for record
|
73
|
+
[:uid, :id, :to_token, :token, :to_sym].each do |method|
|
74
|
+
return record.public_send method if record.respond_to? method
|
75
|
+
end
|
76
|
+
|
77
|
+
[:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
|
78
|
+
return record[method] if record.key? method
|
79
|
+
return record[method.to_s] if record.key? method.to_s
|
80
|
+
end if record.respond_to?(:key?) && record.respond_to?(:[])
|
81
|
+
|
82
|
+
record.to_s
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
module Observer
|
5
|
+
class NullObserver
|
6
|
+
def initialize(options = {})
|
7
|
+
@messenger = options.fetch :messenger do
|
8
|
+
::Logger.new(STDOUT).tap do |logger|
|
9
|
+
logger.formatter = -> _, _, _, msg do
|
10
|
+
"> #{msg}\n"
|
11
|
+
end
|
12
|
+
logger.level = ::Logger::INFO
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def method_missing(*); end
|
18
|
+
alias_method :update, :send
|
19
|
+
|
20
|
+
private
|
21
|
+
attr_reader :messenger
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Processor
|
2
|
+
module RecordsProcessor
|
3
|
+
class Successive
|
4
|
+
def call(records, processor, events, recursion_preventer)
|
5
|
+
records.each do |record|
|
6
|
+
recursion_preventer.call
|
7
|
+
begin
|
8
|
+
events.register :before_record_processing, record
|
9
|
+
|
10
|
+
result = processor.process(record)
|
11
|
+
|
12
|
+
events.register :after_record_processing, record, result
|
13
|
+
rescue RuntimeError => exception
|
14
|
+
events.register :record_processing_error, record, exception
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Processor
|
2
|
+
module RecordsProcessor
|
3
|
+
class Threads
|
4
|
+
def call(records, processor, events, recursion_preventer)
|
5
|
+
threads = []
|
6
|
+
begin
|
7
|
+
records.each do |record|
|
8
|
+
recursion_preventer.call
|
9
|
+
threads << ::Thread.new(processor, record) do |thread_data_processor, thread_record|
|
10
|
+
begin
|
11
|
+
events.register :before_record_processing, thread_record
|
12
|
+
|
13
|
+
result = thread_data_processor.process(thread_record)
|
14
|
+
|
15
|
+
events.register :after_record_processing, thread_record, result
|
16
|
+
rescue RuntimeError => exception
|
17
|
+
events.register :record_processing_error, thread_record, exception
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
ensure # join already created threads even if recursion was detected
|
22
|
+
threads.each(&:join)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require_relative "events_registrator"
|
2
|
+
|
3
|
+
module Processor
|
4
|
+
class Runner
|
5
|
+
def initialize(processor, events_registrator)
|
6
|
+
@processor = processor
|
7
|
+
@events = events_registrator
|
8
|
+
end
|
9
|
+
|
10
|
+
def run(records_processor)
|
11
|
+
events.register :processing_started, processor
|
12
|
+
until processor.done?(records = processor.fetch_records)
|
13
|
+
records_processor.call records, processor, events, method(:recursion_preventer)
|
14
|
+
end
|
15
|
+
|
16
|
+
events.register :processing_finished, processor
|
17
|
+
rescue Exception => exception
|
18
|
+
events.register :processing_error, processor, exception
|
19
|
+
raise exception
|
20
|
+
end
|
21
|
+
|
22
|
+
protected
|
23
|
+
attr_writer :counter
|
24
|
+
def counter
|
25
|
+
@counter ||= 0
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
attr_reader :events, :processor
|
30
|
+
def recursion_preventer
|
31
|
+
self.counter += 1
|
32
|
+
raise Exception, "Processing fall into recursion. Check logs." if self.counter > max_records_to_process
|
33
|
+
end
|
34
|
+
|
35
|
+
def max_records_to_process
|
36
|
+
(processor.total_records * 1.1).round + 10
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'processor/runner'
|
2
|
+
require 'processor/records_processor/successive'
|
3
|
+
require 'processor/records_processor/threads'
|
4
|
+
|
5
|
+
module Processor
|
6
|
+
class Thread
|
7
|
+
def initialize(data_processor, *observers)
|
8
|
+
@runner = Runner.new data_processor, EventsRegistrator.new(observers)
|
9
|
+
end
|
10
|
+
|
11
|
+
def run_as(&records_processor)
|
12
|
+
runner.run records_processor
|
13
|
+
end
|
14
|
+
|
15
|
+
def run_successive
|
16
|
+
runner.run RecordsProcessor::Successive.new
|
17
|
+
end
|
18
|
+
|
19
|
+
def run_in_threads
|
20
|
+
runner.run RecordsProcessor::Threads.new
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
attr_reader :runner
|
25
|
+
end
|
26
|
+
end
|
data/lib/processor/version.rb
CHANGED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require_relative '../example/migrator'
|
3
|
+
require_relative '../example/migration'
|
4
|
+
|
5
|
+
describe "Example" do
|
6
|
+
before(:each) do
|
7
|
+
records = %w[item1 item2 item3 item4 item5]
|
8
|
+
records.each do |record|
|
9
|
+
record.should_receive(:do_something).once
|
10
|
+
end
|
11
|
+
@migration = Processor::Example::Migration.new records
|
12
|
+
end
|
13
|
+
|
14
|
+
it "migration should use 2 loggers and messenger" do
|
15
|
+
migrator = Processor::Example::Migrator.new @migration
|
16
|
+
migrator.migrate
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/data_processor'
|
3
|
+
|
4
|
+
describe Processor::DataProcessor do
|
5
|
+
it "should have a name equals to underscored class name" do
|
6
|
+
subject.name.should eq "processor_data_processor"
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should be done when there are 0 records to process" do
|
10
|
+
records = []
|
11
|
+
subject.done?(records).should be true
|
12
|
+
end
|
13
|
+
|
14
|
+
%w[done? fetch_records total_records process].each do |method_name|
|
15
|
+
it "should respond to #{method_name}" do
|
16
|
+
subject.should respond_to method_name
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/events_registrator'
|
3
|
+
|
4
|
+
describe Processor::EventsRegistrator do
|
5
|
+
subject { Processor::EventsRegistrator }
|
6
|
+
|
7
|
+
it "should broadcast events to all observers" do
|
8
|
+
observers = 3.times.map do
|
9
|
+
stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
|
10
|
+
end
|
11
|
+
|
12
|
+
events = subject.new(observers)
|
13
|
+
events.register :test_event
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/observer/logger'
|
3
|
+
|
4
|
+
describe Processor::Observer::Logger do
|
5
|
+
let(:processor) { stub.as_null_object }
|
6
|
+
let(:no_messages) { ::Logger.new("/dev/null") }
|
7
|
+
|
8
|
+
subject { Processor::Observer::Logger }
|
9
|
+
it "accepts logger builder as parameter" do
|
10
|
+
external_logger = mock
|
11
|
+
logger_observer = subject.new -> name { external_logger }, messenger: no_messages
|
12
|
+
|
13
|
+
external_logger.should_receive(:info)
|
14
|
+
logger_observer.processing_started processor
|
15
|
+
end
|
16
|
+
|
17
|
+
it "accepts logger as parameter" do
|
18
|
+
external_logger = mock
|
19
|
+
logger_observer = subject.new external_logger, messenger: no_messages
|
20
|
+
|
21
|
+
external_logger.should_receive(:info)
|
22
|
+
logger_observer.processing_started processor
|
23
|
+
end
|
24
|
+
|
25
|
+
it "use ruby Logger if no external logger provided" do
|
26
|
+
logger_observer = subject.new nil, messenger: no_messages
|
27
|
+
|
28
|
+
Logger.should_receive(:new).and_return(stub.as_null_object)
|
29
|
+
logger_observer.processing_started processor
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
shared_examples_for "a records processor" do
|
2
|
+
let(:records_processor) { described_class.new }
|
3
|
+
let(:records) { 1..2 }
|
4
|
+
let(:processor) { stub }
|
5
|
+
|
6
|
+
let(:no_recursion_preventer) { Proc.new{} }
|
7
|
+
let(:no_events) { stub.as_null_object }
|
8
|
+
|
9
|
+
it "should send each found record to processor" do
|
10
|
+
records.each { |record| processor.should_receive(:process).with(record) }
|
11
|
+
records_processor.call records, processor, no_events, no_recursion_preventer
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "exception handling" do
|
15
|
+
describe "processing a record raised RuntimeError" do
|
16
|
+
it "should continue processing" do
|
17
|
+
processor.should_receive(:process).twice.and_raise(RuntimeError)
|
18
|
+
expect { records_processor.call records, processor, no_events, no_recursion_preventer }.to_not raise_error
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should register a record_processing_error event" do
|
22
|
+
event = mock.tap { |event| event.should_receive(:trigger).any_number_of_times }
|
23
|
+
|
24
|
+
events_registrator = stub
|
25
|
+
events_registrator.should_receive(:register) do |event_name, failed_record, exception|
|
26
|
+
next if event_name != :record_processing_error
|
27
|
+
event_name.should eq :record_processing_error
|
28
|
+
exception.should be_a RuntimeError
|
29
|
+
event.trigger
|
30
|
+
end.any_number_of_times
|
31
|
+
|
32
|
+
processor.stub(:process).and_raise(RuntimeError)
|
33
|
+
|
34
|
+
records_processor.call records, processor, events_registrator, no_recursion_preventer rescue nil
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/runner'
|
3
|
+
|
4
|
+
describe Processor::Runner do
|
5
|
+
let(:runner) { Processor::Runner.new(processor, events_registrator) }
|
6
|
+
let(:processor) { stub }
|
7
|
+
let(:events_registrator) { stub.as_null_object }
|
8
|
+
let(:no_records_processor) { Proc.new{} }
|
9
|
+
|
10
|
+
it "should fetch records from processor till it'll be done" do
|
11
|
+
processor.stub(:done?).and_return(false, false, true)
|
12
|
+
processor.should_receive(:fetch_records).exactly(3).times.and_return([])
|
13
|
+
runner.run no_records_processor
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "exception handling" do
|
17
|
+
let(:record) { stub }
|
18
|
+
before(:each) do
|
19
|
+
processor.stub(:fetch_records).and_return([record], [record], [])
|
20
|
+
end
|
21
|
+
|
22
|
+
describe "processing records raised" do
|
23
|
+
before(:each) do
|
24
|
+
processor.stub(done?: false)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should break processing and rerise" do
|
28
|
+
expect do
|
29
|
+
runner.run Proc.new { raise RuntimeError }
|
30
|
+
end.to raise_error(RuntimeError)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should register a processing_error event" do
|
34
|
+
register_processing_error_event mock.tap { |event| event.should_receive :trigger }
|
35
|
+
runner.run Proc.new { raise RuntimeError } rescue nil
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
describe "fetching records raised" do
|
40
|
+
it "should break processing and rerise Exception" do
|
41
|
+
processor.stub(:fetch_records).and_raise(RuntimeError)
|
42
|
+
processor.should_not_receive(:process)
|
43
|
+
expect { runner.run no_records_processor }.to raise_error(RuntimeError)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should register a processing_error" do
|
47
|
+
register_processing_error_event mock.tap { |event| event.should_receive :trigger }
|
48
|
+
|
49
|
+
processor.stub(:fetch_records).and_raise(RuntimeError)
|
50
|
+
runner.run no_records_processor rescue nil
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
def register_processing_error_event(event)
|
56
|
+
# Check that processing_error event was register
|
57
|
+
events_registrator.should_receive(:register) do |event_name, current_processor, exception|
|
58
|
+
next if event_name != :processing_error
|
59
|
+
event_name.should eq :processing_error
|
60
|
+
current_processor.should eq processor
|
61
|
+
exception.should be_a RuntimeError
|
62
|
+
event.trigger
|
63
|
+
end.any_number_of_times
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe "recursion" do
|
68
|
+
before(:each) do
|
69
|
+
processor.stub(done?: false)
|
70
|
+
processor.stub(total_records: 100)
|
71
|
+
processor.stub(fetch_records: 1..3)
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should not fall into recursion" do
|
75
|
+
processor.should_receive(:process).at_most(1000).times
|
76
|
+
|
77
|
+
expect do
|
78
|
+
records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
|
79
|
+
records.each do |record|
|
80
|
+
recursion_preventer.call
|
81
|
+
records_processor.process record
|
82
|
+
end
|
83
|
+
end
|
84
|
+
runner.run records_processor
|
85
|
+
end.to raise_error(Exception, /Processing fall into recursion/)
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should have 10% + 10 rerurns window" do
|
89
|
+
processor.should_receive(:process).exactly(120).times
|
90
|
+
|
91
|
+
expect do
|
92
|
+
records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
|
93
|
+
records.each do |record|
|
94
|
+
recursion_preventer.call
|
95
|
+
processor.process record
|
96
|
+
end
|
97
|
+
end
|
98
|
+
runner.run records_processor
|
99
|
+
end.to raise_error(Exception, /Processing fall into recursion/)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/thread'
|
3
|
+
require_relative '../../example/migration'
|
4
|
+
|
5
|
+
describe Processor::Thread do
|
6
|
+
before(:each) do
|
7
|
+
records = %w[item1 item2 item3 item4 item5]
|
8
|
+
records.each do |record|
|
9
|
+
record.should_receive(:do_something).once
|
10
|
+
end
|
11
|
+
@migration = Processor::Example::Migration.new records
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should run a migration using provided block" do
|
15
|
+
thread = Processor::Thread.new @migration
|
16
|
+
thread.run_as do |records, processor, *|
|
17
|
+
records.each do |record|
|
18
|
+
processor.process record
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should run a migration successive" do
|
24
|
+
thread = Processor::Thread.new @migration
|
25
|
+
thread.run_successive
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should run a migration in threads" do
|
29
|
+
thread = Processor::Thread.new @migration
|
30
|
+
thread.run_in_threads
|
31
|
+
end
|
32
|
+
|
33
|
+
pending "should run a migration in specifien number of threads" do
|
34
|
+
thread = Processor::Thread.new @migration
|
35
|
+
thread.run_in_threads 3
|
36
|
+
end
|
37
|
+
end
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: processor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Alexander Paramonov
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-05-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -67,15 +67,39 @@ extensions: []
|
|
67
67
|
extra_rdoc_files: []
|
68
68
|
files:
|
69
69
|
- .gitignore
|
70
|
+
- .rspec
|
70
71
|
- .rvmrc
|
71
72
|
- .travis.yml
|
72
73
|
- Gemfile
|
73
74
|
- LICENSE
|
74
75
|
- README.md
|
75
76
|
- Rakefile
|
77
|
+
- example/logging_thread.rb
|
78
|
+
- example/migration.rb
|
79
|
+
- example/migrator.rb
|
80
|
+
- example/observer/progress_bar.rb
|
81
|
+
- example/solr_migration.rb
|
82
|
+
- example/solr_pages_migration.rb
|
76
83
|
- lib/processor.rb
|
84
|
+
- lib/processor/data_processor.rb
|
85
|
+
- lib/processor/events_registrator.rb
|
86
|
+
- lib/processor/observer/logger.rb
|
87
|
+
- lib/processor/observer/null_observer.rb
|
88
|
+
- lib/processor/records_processor/successive.rb
|
89
|
+
- lib/processor/records_processor/threads.rb
|
90
|
+
- lib/processor/runner.rb
|
91
|
+
- lib/processor/thread.rb
|
77
92
|
- lib/processor/version.rb
|
78
93
|
- processor.gemspec
|
94
|
+
- spec/example_spec.rb
|
95
|
+
- spec/processor/data_processor_spec.rb
|
96
|
+
- spec/processor/events_registrator_spec.rb
|
97
|
+
- spec/processor/observer/logger_spec.rb
|
98
|
+
- spec/processor/records_processor/specs.rb
|
99
|
+
- spec/processor/records_processor/successive_spec.rb
|
100
|
+
- spec/processor/records_processor/threads_spec.rb
|
101
|
+
- spec/processor/runner_spec.rb
|
102
|
+
- spec/processor/thread_spec.rb
|
79
103
|
- spec/spec_helper_lite.rb
|
80
104
|
homepage: http://github.com/AlexParamonov/processor
|
81
105
|
licenses:
|
@@ -90,12 +114,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
90
114
|
- - ! '>='
|
91
115
|
- !ruby/object:Gem::Version
|
92
116
|
version: '0'
|
117
|
+
segments:
|
118
|
+
- 0
|
119
|
+
hash: 4470760754938067473
|
93
120
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
94
121
|
none: false
|
95
122
|
requirements:
|
96
|
-
- - ! '
|
123
|
+
- - ! '>='
|
97
124
|
- !ruby/object:Gem::Version
|
98
|
-
version:
|
125
|
+
version: '0'
|
126
|
+
segments:
|
127
|
+
- 0
|
128
|
+
hash: 4470760754938067473
|
99
129
|
requirements: []
|
100
130
|
rubyforge_project:
|
101
131
|
rubygems_version: 1.8.25
|
@@ -103,4 +133,13 @@ signing_key:
|
|
103
133
|
specification_version: 3
|
104
134
|
summary: Process records one by one
|
105
135
|
test_files:
|
136
|
+
- spec/example_spec.rb
|
137
|
+
- spec/processor/data_processor_spec.rb
|
138
|
+
- spec/processor/events_registrator_spec.rb
|
139
|
+
- spec/processor/observer/logger_spec.rb
|
140
|
+
- spec/processor/records_processor/specs.rb
|
141
|
+
- spec/processor/records_processor/successive_spec.rb
|
142
|
+
- spec/processor/records_processor/threads_spec.rb
|
143
|
+
- spec/processor/runner_spec.rb
|
144
|
+
- spec/processor/thread_spec.rb
|
106
145
|
- spec/spec_helper_lite.rb
|