processor 0.0.0.initial → 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --format documentation
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2012 Alexander Paramonov
1
+ Copyright (c) 2013 Alexander Paramonov
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -8,9 +8,10 @@ Universal processor for data migration
8
8
  Contents
9
9
  ---------
10
10
  1. Installation
11
- 1. Contributing
12
11
  1. Requirements
12
+ 1. Usage
13
13
  1. Compatibility
14
+ 1. Contributing
14
15
  1. Copyright
15
16
 
16
17
  Installation
@@ -35,9 +36,40 @@ gem install processor
35
36
 
36
37
  Requirements
37
38
  ------------
38
- none
39
+ 1. Ruby 1.9
40
+ 1. Rspec2 for testing
41
+
42
+ Usage
43
+ ------------
44
+ 1. Implement a `DataProcessor`. See `Processor::Example::Migration`, `Processor::Example::SolrMigration`, `Processor::Example::SolrPagesMigration`
45
+ 1. Run your `DataProcessor`:
46
+
47
+ ``` ruby
48
+ thread = Processor::Thread.new data_processor
49
+ thread.run_successive
50
+ ```
51
+ See `spec/processor/thread_spec.rb` and `spec/example_spec.rb` and `example` directory for other usage examples.
52
+ Below are specs for most valuable parts of the gem.
53
+
54
+ ``` rspec
55
+ Processor::Thread
56
+ should run a migration using provided block
57
+ should run a migration successive
58
+ should run a migration in threads
39
59
 
40
- rspec2 for testing
60
+ Processor::DataProcessor
61
+ should have a name equals to underscored class name
62
+ should be done when there are 0 records to process
63
+ should respond to done?
64
+ should respond to fetch_records
65
+ should respond to total_records
66
+ should respond to process
67
+
68
+ Processor::Observer::Logger
69
+ accepts logger builder as parameter
70
+ accepts logger as parameter
71
+ use ruby Logger if no external logger provided
72
+ ```
41
73
 
42
74
  Compatibility
43
75
  -------------
@@ -0,0 +1,15 @@
1
+ require 'processor/thread'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class LoggingThread < Processor::Thread
7
+ def initialize(processor)
8
+ super(
9
+ processor,
10
+ Processor::Observer::Logger.new
11
+ )
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,29 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class Migration < DataProcessor
6
+ attr_reader :records
7
+ def initialize(records)
8
+ @records = records
9
+ end
10
+
11
+ def done?(records)
12
+ records.count < 1
13
+ end
14
+
15
+ def process(record)
16
+ record.do_something
17
+ "OK"
18
+ end
19
+
20
+ def fetch_records
21
+ records.shift(2)
22
+ end
23
+
24
+ def total_records
25
+ records.count
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,66 @@
1
+ require 'processor/thread'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class Migrator < Processor::Thread
7
+ def migrate
8
+ messenger.info "Migrator start migrating a records one by one at #{Time.now}"
9
+ run_successive
10
+ end
11
+
12
+ def initialize(migration)
13
+ # logger could be an instance of Ruby Logger
14
+ logger1 = ::Logger.new(STDOUT).tap do |logger|
15
+ logger.formatter = -> _, _, _, msg do
16
+ "debug\t< #{msg}\n"
17
+ end
18
+ logger.level = ::Logger::DEBUG
19
+ end
20
+
21
+ # Logger could be a lambda
22
+ # Where "name" is a processor.name
23
+ logger2 = -> name do
24
+ ::Logger.new(STDOUT).tap do |logger|
25
+ logger.formatter = -> _, _, _, msg do
26
+ "info\t< #{msg}\n"
27
+ end
28
+ logger.level = ::Logger::INFO
29
+ end
30
+ end
31
+
32
+ # logger could be an instance of Rails Logger
33
+ # logger = Rails.logger
34
+ # Or be nil
35
+ # logger = nil
36
+ # in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
37
+
38
+ # You may customize a messenger:
39
+ messenger = ::Logger.new(STDOUT).tap do |logger|
40
+ logger.formatter = -> _, _, _, msg do
41
+ "message\t> #{msg}\n"
42
+ end
43
+ logger.level = ::Logger::DEBUG
44
+ end
45
+
46
+ stdout_logger_debug = Processor::Observer::Logger.new(logger1, messenger: messenger)
47
+ stdout_logger_info = Processor::Observer::Logger.new(logger2)
48
+ your_custom_observer1 = Observer::NullObserver.new
49
+ your_custom_observer2 = Observer::NullObserver.new
50
+
51
+ super(
52
+ migration,
53
+ stdout_logger_debug,
54
+ stdout_logger_info,
55
+ your_custom_observer1,
56
+ your_custom_observer2,
57
+ )
58
+
59
+ @messenger = messenger
60
+ end
61
+
62
+ private
63
+ attr_reader :messenger
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,26 @@
1
+ require 'processor/observer/null_observer'
2
+ require 'progressbar'
3
+
4
+ module Processor
5
+ module Example
6
+ module Observer
7
+ class ProgressBar < Processor::Observer::NullObserver
8
+ def processing_started
9
+ @progress_bar = ::ProgressBar.new("Records", processor.total_records)
10
+ messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
11
+ end
12
+
13
+ def before_record_processing(record)
14
+ progress_bar.inc
15
+ end
16
+
17
+ def processing_finished
18
+ progress_bar.finish
19
+ end
20
+
21
+ private
22
+ attr_reader :progress_bar
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,31 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class SolrMigration < DataProcessor
6
+ def process(user)
7
+ user.set_contact_method "Address book"
8
+ user.save!
9
+ end
10
+
11
+ def fetch_records
12
+ query.results
13
+ end
14
+
15
+ def total_records
16
+ @total_records ||= query.total
17
+ end
18
+
19
+ private
20
+ # query will return 0 records when all users'll be processed successfully
21
+ def query
22
+ User.search {
23
+ fulltext "My company"
24
+ with :title, "Manager"
25
+ with :contact_method, "Direct contact"
26
+ paginate page: 1, per_page: 10
27
+ }
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,48 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class SolrPagesMigration < DataProcessor
6
+ def done?(records)
7
+ # Optional custom check for migration to be done.
8
+
9
+ # your custom check
10
+
11
+ # use a default check if you like
12
+ super
13
+ end
14
+
15
+ def process(user)
16
+ user.set_contact_method "Address book"
17
+ user.save!
18
+ end
19
+
20
+ def fetch_records
21
+ query(next_page).results
22
+ end
23
+
24
+ def total_records
25
+ @total_records ||= query(1).total
26
+ end
27
+
28
+ # optional name to use in observers.
29
+ def name
30
+ "my_company_users_contact_method_migration"
31
+ end
32
+
33
+ private
34
+ def query(requested_page)
35
+ User.search {
36
+ fulltext "My company"
37
+ paginate page: requested_page, per_page: 10
38
+ }
39
+ end
40
+
41
+ def next_page
42
+ @page ||= 0
43
+ @page += 1
44
+ end
45
+ end
46
+ end
47
+ end
48
+
@@ -0,0 +1,28 @@
1
+ module Processor
2
+ class DataProcessor
3
+ def done?(records)
4
+ records.count < 1
5
+ end
6
+
7
+ def process(record)
8
+ raise NotImplementedError
9
+ end
10
+
11
+ def fetch_records
12
+ raise NotImplementedError
13
+ end
14
+
15
+ def total_records
16
+ raise NotImplementedError
17
+ end
18
+
19
+ def name
20
+ # underscore a class name
21
+ self.class.name.to_s.
22
+ gsub(/::/, '_').
23
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
24
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
25
+ downcase
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,16 @@
1
+ module Processor
2
+ class EventsRegistrator
3
+ def initialize(observers)
4
+ @observers = observers
5
+ end
6
+
7
+ def register(event, *data)
8
+ observers.each do |observer|
9
+ observer.update event, *data
10
+ end
11
+ end
12
+
13
+ private
14
+ attr_reader :observers
15
+ end
16
+ end
@@ -0,0 +1,86 @@
1
+ require_relative 'null_observer'
2
+ require 'logger'
3
+
4
+ module Processor
5
+ module Observer
6
+ class Logger < NullObserver
7
+ def initialize(logger = nil, options = {})
8
+ @logger_source = logger
9
+ super options
10
+ end
11
+
12
+ def processing_started(processor)
13
+ initialize_logger(processor)
14
+ logger.info "Processing of #{processor.name} started."
15
+
16
+ message = <<-MESSAGE.gsub(/^\s+/, '')
17
+ Proggress will be saved to the log file. Run
18
+ tail -f #{log_file_name}
19
+ to see log in realtime
20
+ MESSAGE
21
+ messenger.info message if use_log_file?
22
+ end
23
+
24
+ def before_record_processing(record)
25
+ logger.debug "Record #{id_for record} is going to be processed"
26
+ end
27
+
28
+ def after_record_processing(record, result)
29
+ logger.info "Successfully processed #{id_for record}: #{result}"
30
+ end
31
+
32
+ def processing_finished(processor)
33
+ logger.info "Processing of #{processor.name} finished."
34
+ messenger.info "Log file saved to #{log_file_name}" if use_log_file?
35
+ end
36
+
37
+ def record_processing_error(record, exception)
38
+ logger.error "Error processing #{id_for record}: #{exception}"
39
+ end
40
+
41
+ def processing_error(processor, exception)
42
+ logger.fatal "Processing #{processor.name} failed: #{exception}"
43
+ end
44
+
45
+ private
46
+ attr_reader :logger, :log_file_name
47
+
48
+ def initialize_logger(processor)
49
+ @logger =
50
+ if @logger_source.is_a? Proc
51
+ @logger_source.call processor.name
52
+ else
53
+ @logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
54
+ logger.level = ::Logger::INFO
55
+ end
56
+ end
57
+ messenger.debug "Observer initialized with logger #{@logger}"
58
+ end
59
+
60
+ def create_log_filename(processor_name)
61
+ @log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
62
+ end
63
+
64
+ def use_log_file?
65
+ not log_file_name.nil?
66
+ end
67
+
68
+ def current_time_string
69
+ Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
70
+ end
71
+
72
+ def id_for record
73
+ [:uid, :id, :to_token, :token, :to_sym].each do |method|
74
+ return record.public_send method if record.respond_to? method
75
+ end
76
+
77
+ [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
78
+ return record[method] if record.key? method
79
+ return record[method.to_s] if record.key? method.to_s
80
+ end if record.respond_to?(:key?) && record.respond_to?(:[])
81
+
82
+ record.to_s
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,24 @@
1
+ require 'logger'
2
+
3
+ module Processor
4
+ module Observer
5
+ class NullObserver
6
+ def initialize(options = {})
7
+ @messenger = options.fetch :messenger do
8
+ ::Logger.new(STDOUT).tap do |logger|
9
+ logger.formatter = -> _, _, _, msg do
10
+ "> #{msg}\n"
11
+ end
12
+ logger.level = ::Logger::INFO
13
+ end
14
+ end
15
+ end
16
+
17
+ def method_missing(*); end
18
+ alias_method :update, :send
19
+
20
+ private
21
+ attr_reader :messenger
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,20 @@
1
+ module Processor
2
+ module RecordsProcessor
3
+ class Successive
4
+ def call(records, processor, events, recursion_preventer)
5
+ records.each do |record|
6
+ recursion_preventer.call
7
+ begin
8
+ events.register :before_record_processing, record
9
+
10
+ result = processor.process(record)
11
+
12
+ events.register :after_record_processing, record, result
13
+ rescue RuntimeError => exception
14
+ events.register :record_processing_error, record, exception
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,27 @@
1
+ module Processor
2
+ module RecordsProcessor
3
+ class Threads
4
+ def call(records, processor, events, recursion_preventer)
5
+ threads = []
6
+ begin
7
+ records.each do |record|
8
+ recursion_preventer.call
9
+ threads << ::Thread.new(processor, record) do |thread_data_processor, thread_record|
10
+ begin
11
+ events.register :before_record_processing, thread_record
12
+
13
+ result = thread_data_processor.process(thread_record)
14
+
15
+ events.register :after_record_processing, thread_record, result
16
+ rescue RuntimeError => exception
17
+ events.register :record_processing_error, thread_record, exception
18
+ end
19
+ end
20
+ end
21
+ ensure # join already created threads even if recursion was detected
22
+ threads.each(&:join)
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,39 @@
1
+ require_relative "events_registrator"
2
+
3
+ module Processor
4
+ class Runner
5
+ def initialize(processor, events_registrator)
6
+ @processor = processor
7
+ @events = events_registrator
8
+ end
9
+
10
+ def run(records_processor)
11
+ events.register :processing_started, processor
12
+ until processor.done?(records = processor.fetch_records)
13
+ records_processor.call records, processor, events, method(:recursion_preventer)
14
+ end
15
+
16
+ events.register :processing_finished, processor
17
+ rescue Exception => exception
18
+ events.register :processing_error, processor, exception
19
+ raise exception
20
+ end
21
+
22
+ protected
23
+ attr_writer :counter
24
+ def counter
25
+ @counter ||= 0
26
+ end
27
+
28
+ private
29
+ attr_reader :events, :processor
30
+ def recursion_preventer
31
+ self.counter += 1
32
+ raise Exception, "Processing fall into recursion. Check logs." if self.counter > max_records_to_process
33
+ end
34
+
35
+ def max_records_to_process
36
+ (processor.total_records * 1.1).round + 10
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,26 @@
1
+ require 'processor/runner'
2
+ require 'processor/records_processor/successive'
3
+ require 'processor/records_processor/threads'
4
+
5
+ module Processor
6
+ class Thread
7
+ def initialize(data_processor, *observers)
8
+ @runner = Runner.new data_processor, EventsRegistrator.new(observers)
9
+ end
10
+
11
+ def run_as(&records_processor)
12
+ runner.run records_processor
13
+ end
14
+
15
+ def run_successive
16
+ runner.run RecordsProcessor::Successive.new
17
+ end
18
+
19
+ def run_in_threads
20
+ runner.run RecordsProcessor::Threads.new
21
+ end
22
+
23
+ private
24
+ attr_reader :runner
25
+ end
26
+ end
@@ -1,3 +1,3 @@
1
1
  module Processor
2
- VERSION = "0.0.0.initial"
2
+ VERSION = "0.0.1"
3
3
  end
@@ -0,0 +1,18 @@
1
+ require 'spec_helper_lite'
2
+ require_relative '../example/migrator'
3
+ require_relative '../example/migration'
4
+
5
+ describe "Example" do
6
+ before(:each) do
7
+ records = %w[item1 item2 item3 item4 item5]
8
+ records.each do |record|
9
+ record.should_receive(:do_something).once
10
+ end
11
+ @migration = Processor::Example::Migration.new records
12
+ end
13
+
14
+ it "migration should use 2 loggers and messenger" do
15
+ migrator = Processor::Example::Migrator.new @migration
16
+ migrator.migrate
17
+ end
18
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/data_processor'
3
+
4
+ describe Processor::DataProcessor do
5
+ it "should have a name equals to underscored class name" do
6
+ subject.name.should eq "processor_data_processor"
7
+ end
8
+
9
+ it "should be done when there are 0 records to process" do
10
+ records = []
11
+ subject.done?(records).should be true
12
+ end
13
+
14
+ %w[done? fetch_records total_records process].each do |method_name|
15
+ it "should respond to #{method_name}" do
16
+ subject.should respond_to method_name
17
+ end
18
+ end
19
+ end
20
+
@@ -0,0 +1,15 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/events_registrator'
3
+
4
+ describe Processor::EventsRegistrator do
5
+ subject { Processor::EventsRegistrator }
6
+
7
+ it "should broadcast events to all observers" do
8
+ observers = 3.times.map do
9
+ stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
10
+ end
11
+
12
+ events = subject.new(observers)
13
+ events.register :test_event
14
+ end
15
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/observer/logger'
3
+
4
+ describe Processor::Observer::Logger do
5
+ let(:processor) { stub.as_null_object }
6
+ let(:no_messages) { ::Logger.new("/dev/null") }
7
+
8
+ subject { Processor::Observer::Logger }
9
+ it "accepts logger builder as parameter" do
10
+ external_logger = mock
11
+ logger_observer = subject.new -> name { external_logger }, messenger: no_messages
12
+
13
+ external_logger.should_receive(:info)
14
+ logger_observer.processing_started processor
15
+ end
16
+
17
+ it "accepts logger as parameter" do
18
+ external_logger = mock
19
+ logger_observer = subject.new external_logger, messenger: no_messages
20
+
21
+ external_logger.should_receive(:info)
22
+ logger_observer.processing_started processor
23
+ end
24
+
25
+ it "use ruby Logger if no external logger provided" do
26
+ logger_observer = subject.new nil, messenger: no_messages
27
+
28
+ Logger.should_receive(:new).and_return(stub.as_null_object)
29
+ logger_observer.processing_started processor
30
+ end
31
+ end
32
+
@@ -0,0 +1,38 @@
1
+ shared_examples_for "a records processor" do
2
+ let(:records_processor) { described_class.new }
3
+ let(:records) { 1..2 }
4
+ let(:processor) { stub }
5
+
6
+ let(:no_recursion_preventer) { Proc.new{} }
7
+ let(:no_events) { stub.as_null_object }
8
+
9
+ it "should send each found record to processor" do
10
+ records.each { |record| processor.should_receive(:process).with(record) }
11
+ records_processor.call records, processor, no_events, no_recursion_preventer
12
+ end
13
+
14
+ describe "exception handling" do
15
+ describe "processing a record raised RuntimeError" do
16
+ it "should continue processing" do
17
+ processor.should_receive(:process).twice.and_raise(RuntimeError)
18
+ expect { records_processor.call records, processor, no_events, no_recursion_preventer }.to_not raise_error
19
+ end
20
+
21
+ it "should register a record_processing_error event" do
22
+ event = mock.tap { |event| event.should_receive(:trigger).any_number_of_times }
23
+
24
+ events_registrator = stub
25
+ events_registrator.should_receive(:register) do |event_name, failed_record, exception|
26
+ next if event_name != :record_processing_error
27
+ event_name.should eq :record_processing_error
28
+ exception.should be_a RuntimeError
29
+ event.trigger
30
+ end.any_number_of_times
31
+
32
+ processor.stub(:process).and_raise(RuntimeError)
33
+
34
+ records_processor.call records, processor, events_registrator, no_recursion_preventer rescue nil
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,7 @@
1
+ require 'spec_helper_lite'
2
+ require_relative 'specs'
3
+ require 'processor/records_processor/successive'
4
+
5
+ describe Processor::RecordsProcessor::Successive do
6
+ it_behaves_like "a records processor"
7
+ end
@@ -0,0 +1,8 @@
1
+ require 'spec_helper_lite'
2
+ require_relative 'specs'
3
+ require 'processor/records_processor/threads'
4
+
5
+ describe Processor::RecordsProcessor::Threads do
6
+ it_behaves_like "a records processor"
7
+ it "should run in defined number of threads"
8
+ end
@@ -0,0 +1,103 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/runner'
3
+
4
+ describe Processor::Runner do
5
+ let(:runner) { Processor::Runner.new(processor, events_registrator) }
6
+ let(:processor) { stub }
7
+ let(:events_registrator) { stub.as_null_object }
8
+ let(:no_records_processor) { Proc.new{} }
9
+
10
+ it "should fetch records from processor till it'll be done" do
11
+ processor.stub(:done?).and_return(false, false, true)
12
+ processor.should_receive(:fetch_records).exactly(3).times.and_return([])
13
+ runner.run no_records_processor
14
+ end
15
+
16
+ describe "exception handling" do
17
+ let(:record) { stub }
18
+ before(:each) do
19
+ processor.stub(:fetch_records).and_return([record], [record], [])
20
+ end
21
+
22
+ describe "processing records raised" do
23
+ before(:each) do
24
+ processor.stub(done?: false)
25
+ end
26
+
27
+ it "should break processing and rerise" do
28
+ expect do
29
+ runner.run Proc.new { raise RuntimeError }
30
+ end.to raise_error(RuntimeError)
31
+ end
32
+
33
+ it "should register a processing_error event" do
34
+ register_processing_error_event mock.tap { |event| event.should_receive :trigger }
35
+ runner.run Proc.new { raise RuntimeError } rescue nil
36
+ end
37
+ end
38
+
39
+ describe "fetching records raised" do
40
+ it "should break processing and rerise Exception" do
41
+ processor.stub(:fetch_records).and_raise(RuntimeError)
42
+ processor.should_not_receive(:process)
43
+ expect { runner.run no_records_processor }.to raise_error(RuntimeError)
44
+ end
45
+
46
+ it "should register a processing_error" do
47
+ register_processing_error_event mock.tap { |event| event.should_receive :trigger }
48
+
49
+ processor.stub(:fetch_records).and_raise(RuntimeError)
50
+ runner.run no_records_processor rescue nil
51
+ end
52
+ end
53
+
54
+ private
55
+ def register_processing_error_event(event)
56
+ # Check that processing_error event was register
57
+ events_registrator.should_receive(:register) do |event_name, current_processor, exception|
58
+ next if event_name != :processing_error
59
+ event_name.should eq :processing_error
60
+ current_processor.should eq processor
61
+ exception.should be_a RuntimeError
62
+ event.trigger
63
+ end.any_number_of_times
64
+ end
65
+ end
66
+
67
+ describe "recursion" do
68
+ before(:each) do
69
+ processor.stub(done?: false)
70
+ processor.stub(total_records: 100)
71
+ processor.stub(fetch_records: 1..3)
72
+ end
73
+
74
+ it "should not fall into recursion" do
75
+ processor.should_receive(:process).at_most(1000).times
76
+
77
+ expect do
78
+ records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
79
+ records.each do |record|
80
+ recursion_preventer.call
81
+ records_processor.process record
82
+ end
83
+ end
84
+ runner.run records_processor
85
+ end.to raise_error(Exception, /Processing fall into recursion/)
86
+ end
87
+
88
+ it "should have 10% + 10 rerurns window" do
89
+ processor.should_receive(:process).exactly(120).times
90
+
91
+ expect do
92
+ records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
93
+ records.each do |record|
94
+ recursion_preventer.call
95
+ processor.process record
96
+ end
97
+ end
98
+ runner.run records_processor
99
+ end.to raise_error(Exception, /Processing fall into recursion/)
100
+ end
101
+ end
102
+ end
103
+
@@ -0,0 +1,37 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/thread'
3
+ require_relative '../../example/migration'
4
+
5
+ describe Processor::Thread do
6
+ before(:each) do
7
+ records = %w[item1 item2 item3 item4 item5]
8
+ records.each do |record|
9
+ record.should_receive(:do_something).once
10
+ end
11
+ @migration = Processor::Example::Migration.new records
12
+ end
13
+
14
+ it "should run a migration using provided block" do
15
+ thread = Processor::Thread.new @migration
16
+ thread.run_as do |records, processor, *|
17
+ records.each do |record|
18
+ processor.process record
19
+ end
20
+ end
21
+ end
22
+
23
+ it "should run a migration successive" do
24
+ thread = Processor::Thread.new @migration
25
+ thread.run_successive
26
+ end
27
+
28
+ it "should run a migration in threads" do
29
+ thread = Processor::Thread.new @migration
30
+ thread.run_in_threads
31
+ end
32
+
33
+ pending "should run a migration in specifien number of threads" do
34
+ thread = Processor::Thread.new @migration
35
+ thread.run_in_threads 3
36
+ end
37
+ end
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: processor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0.initial
5
- prerelease: 6
4
+ version: 0.0.1
5
+ prerelease:
6
6
  platform: ruby
7
7
  authors:
8
8
  - Alexander Paramonov
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-04-30 00:00:00.000000000 Z
12
+ date: 2013-05-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
@@ -67,15 +67,39 @@ extensions: []
67
67
  extra_rdoc_files: []
68
68
  files:
69
69
  - .gitignore
70
+ - .rspec
70
71
  - .rvmrc
71
72
  - .travis.yml
72
73
  - Gemfile
73
74
  - LICENSE
74
75
  - README.md
75
76
  - Rakefile
77
+ - example/logging_thread.rb
78
+ - example/migration.rb
79
+ - example/migrator.rb
80
+ - example/observer/progress_bar.rb
81
+ - example/solr_migration.rb
82
+ - example/solr_pages_migration.rb
76
83
  - lib/processor.rb
84
+ - lib/processor/data_processor.rb
85
+ - lib/processor/events_registrator.rb
86
+ - lib/processor/observer/logger.rb
87
+ - lib/processor/observer/null_observer.rb
88
+ - lib/processor/records_processor/successive.rb
89
+ - lib/processor/records_processor/threads.rb
90
+ - lib/processor/runner.rb
91
+ - lib/processor/thread.rb
77
92
  - lib/processor/version.rb
78
93
  - processor.gemspec
94
+ - spec/example_spec.rb
95
+ - spec/processor/data_processor_spec.rb
96
+ - spec/processor/events_registrator_spec.rb
97
+ - spec/processor/observer/logger_spec.rb
98
+ - spec/processor/records_processor/specs.rb
99
+ - spec/processor/records_processor/successive_spec.rb
100
+ - spec/processor/records_processor/threads_spec.rb
101
+ - spec/processor/runner_spec.rb
102
+ - spec/processor/thread_spec.rb
79
103
  - spec/spec_helper_lite.rb
80
104
  homepage: http://github.com/AlexParamonov/processor
81
105
  licenses:
@@ -90,12 +114,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
90
114
  - - ! '>='
91
115
  - !ruby/object:Gem::Version
92
116
  version: '0'
117
+ segments:
118
+ - 0
119
+ hash: 4470760754938067473
93
120
  required_rubygems_version: !ruby/object:Gem::Requirement
94
121
  none: false
95
122
  requirements:
96
- - - ! '>'
123
+ - - ! '>='
97
124
  - !ruby/object:Gem::Version
98
- version: 1.3.1
125
+ version: '0'
126
+ segments:
127
+ - 0
128
+ hash: 4470760754938067473
99
129
  requirements: []
100
130
  rubyforge_project:
101
131
  rubygems_version: 1.8.25
@@ -103,4 +133,13 @@ signing_key:
103
133
  specification_version: 3
104
134
  summary: Process records one by one
105
135
  test_files:
136
+ - spec/example_spec.rb
137
+ - spec/processor/data_processor_spec.rb
138
+ - spec/processor/events_registrator_spec.rb
139
+ - spec/processor/observer/logger_spec.rb
140
+ - spec/processor/records_processor/specs.rb
141
+ - spec/processor/records_processor/successive_spec.rb
142
+ - spec/processor/records_processor/threads_spec.rb
143
+ - spec/processor/runner_spec.rb
144
+ - spec/processor/thread_spec.rb
106
145
  - spec/spec_helper_lite.rb