processor 0.0.0.initial → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --format documentation
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2012 Alexander Paramonov
1
+ Copyright (c) 2013 Alexander Paramonov
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -8,9 +8,10 @@ Universal processor for data migration
8
8
  Contents
9
9
  ---------
10
10
  1. Installation
11
- 1. Contributing
12
11
  1. Requirements
12
+ 1. Usage
13
13
  1. Compatibility
14
+ 1. Contributing
14
15
  1. Copyright
15
16
 
16
17
  Installation
@@ -35,9 +36,40 @@ gem install processor
35
36
 
36
37
  Requirements
37
38
  ------------
38
- none
39
+ 1. Ruby 1.9
40
+ 1. Rspec2 for testing
41
+
42
+ Usage
43
+ ------------
44
+ 1. Implement a `DataProcessor`. See `Processor::Example::Migration`, `Processor::Example::SolrMigration`, `Processor::Example::SolrPagesMigration`
45
+ 1. Run your `DataProcessor`:
46
+
47
+ ``` ruby
48
+ thread = Processor::Thread.new data_processor
49
+ thread.run_successive
50
+ ```
51
+ See `spec/processor/thread_spec.rb` and `spec/example_spec.rb` and `example` directory for other usage examples.
52
+ Below are specs for most valuable parts of the gem.
53
+
54
+ ``` rspec
55
+ Processor::Thread
56
+ should run a migration using provided block
57
+ should run a migration successive
58
+ should run a migration in threads
39
59
 
40
- rspec2 for testing
60
+ Processor::DataProcessor
61
+ should have a name equals to underscored class name
62
+ should be done when there are 0 records to process
63
+ should respond to done?
64
+ should respond to fetch_records
65
+ should respond to total_records
66
+ should respond to process
67
+
68
+ Processor::Observer::Logger
69
+ accepts logger builder as parameter
70
+ accepts logger as parameter
71
+ use ruby Logger if no external logger provided
72
+ ```
41
73
 
42
74
  Compatibility
43
75
  -------------
@@ -0,0 +1,15 @@
1
+ require 'processor/thread'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class LoggingThread < Processor::Thread
7
+ def initialize(processor)
8
+ super(
9
+ processor,
10
+ Processor::Observer::Logger.new
11
+ )
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,29 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class Migration < DataProcessor
6
+ attr_reader :records
7
+ def initialize(records)
8
+ @records = records
9
+ end
10
+
11
+ def done?(records)
12
+ records.count < 1
13
+ end
14
+
15
+ def process(record)
16
+ record.do_something
17
+ "OK"
18
+ end
19
+
20
+ def fetch_records
21
+ records.shift(2)
22
+ end
23
+
24
+ def total_records
25
+ records.count
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,66 @@
1
+ require 'processor/thread'
2
+ require 'processor/observer/logger'
3
+
4
+ module Processor
5
+ module Example
6
+ class Migrator < Processor::Thread
7
+ def migrate
8
+ messenger.info "Migrator start migrating a records one by one at #{Time.now}"
9
+ run_successive
10
+ end
11
+
12
+ def initialize(migration)
13
+ # logger could be an instance of Ruby Logger
14
+ logger1 = ::Logger.new(STDOUT).tap do |logger|
15
+ logger.formatter = -> _, _, _, msg do
16
+ "debug\t< #{msg}\n"
17
+ end
18
+ logger.level = ::Logger::DEBUG
19
+ end
20
+
21
+ # Logger could be a lambda
22
+ # Where "name" is a processor.name
23
+ logger2 = -> name do
24
+ ::Logger.new(STDOUT).tap do |logger|
25
+ logger.formatter = -> _, _, _, msg do
26
+ "info\t< #{msg}\n"
27
+ end
28
+ logger.level = ::Logger::INFO
29
+ end
30
+ end
31
+
32
+ # logger could be an instance of Rails Logger
33
+ # logger = Rails.logger
34
+ # Or be nil
35
+ # logger = nil
36
+ # in this case logger will be initialized as Ruby Logger and write to log/name_of_processor_time_stamp.log
37
+
38
+ # You may customize a messenger:
39
+ messenger = ::Logger.new(STDOUT).tap do |logger|
40
+ logger.formatter = -> _, _, _, msg do
41
+ "message\t> #{msg}\n"
42
+ end
43
+ logger.level = ::Logger::DEBUG
44
+ end
45
+
46
+ stdout_logger_debug = Processor::Observer::Logger.new(logger1, messenger: messenger)
47
+ stdout_logger_info = Processor::Observer::Logger.new(logger2)
48
+ your_custom_observer1 = Observer::NullObserver.new
49
+ your_custom_observer2 = Observer::NullObserver.new
50
+
51
+ super(
52
+ migration,
53
+ stdout_logger_debug,
54
+ stdout_logger_info,
55
+ your_custom_observer1,
56
+ your_custom_observer2,
57
+ )
58
+
59
+ @messenger = messenger
60
+ end
61
+
62
+ private
63
+ attr_reader :messenger
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,26 @@
1
+ require 'processor/observer/null_observer'
2
+ require 'progressbar'
3
+
4
+ module Processor
5
+ module Example
6
+ module Observer
7
+ class ProgressBar < Processor::Observer::NullObserver
8
+ def processing_started
9
+ @progress_bar = ::ProgressBar.new("Records", processor.total_records)
10
+ messenger.debug "Initialized ProgressBar with #{processor.total_records} records"
11
+ end
12
+
13
+ def before_record_processing(record)
14
+ progress_bar.inc
15
+ end
16
+
17
+ def processing_finished
18
+ progress_bar.finish
19
+ end
20
+
21
+ private
22
+ attr_reader :progress_bar
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,31 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class SolrMigration < DataProcessor
6
+ def process(user)
7
+ user.set_contact_method "Address book"
8
+ user.save!
9
+ end
10
+
11
+ def fetch_records
12
+ query.results
13
+ end
14
+
15
+ def total_records
16
+ @total_records ||= query.total
17
+ end
18
+
19
+ private
20
+ # query will return 0 records when all users'll be processed successfully
21
+ def query
22
+ User.search {
23
+ fulltext "My company"
24
+ with :title, "Manager"
25
+ with :contact_method, "Direct contact"
26
+ paginate page: 1, per_page: 10
27
+ }
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,48 @@
1
+ require 'processor/data_processor'
2
+
3
+ module Processor
4
+ module Example
5
+ class SolrPagesMigration < DataProcessor
6
+ def done?(records)
7
+ # Optional custom check for migration to be done.
8
+
9
+ # your custom check
10
+
11
+ # use a default check if you like
12
+ super
13
+ end
14
+
15
+ def process(user)
16
+ user.set_contact_method "Address book"
17
+ user.save!
18
+ end
19
+
20
+ def fetch_records
21
+ query(next_page).results
22
+ end
23
+
24
+ def total_records
25
+ @total_records ||= query(1).total
26
+ end
27
+
28
+ # optional name to use in observers.
29
+ def name
30
+ "my_company_users_contact_method_migration"
31
+ end
32
+
33
+ private
34
+ def query(requested_page)
35
+ User.search {
36
+ fulltext "My company"
37
+ paginate page: requested_page, per_page: 10
38
+ }
39
+ end
40
+
41
+ def next_page
42
+ @page ||= 0
43
+ @page += 1
44
+ end
45
+ end
46
+ end
47
+ end
48
+
@@ -0,0 +1,28 @@
1
+ module Processor
2
+ class DataProcessor
3
+ def done?(records)
4
+ records.count < 1
5
+ end
6
+
7
+ def process(record)
8
+ raise NotImplementedError
9
+ end
10
+
11
+ def fetch_records
12
+ raise NotImplementedError
13
+ end
14
+
15
+ def total_records
16
+ raise NotImplementedError
17
+ end
18
+
19
+ def name
20
+ # underscore a class name
21
+ self.class.name.to_s.
22
+ gsub(/::/, '_').
23
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
24
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
25
+ downcase
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,16 @@
1
+ module Processor
2
+ class EventsRegistrator
3
+ def initialize(observers)
4
+ @observers = observers
5
+ end
6
+
7
+ def register(event, *data)
8
+ observers.each do |observer|
9
+ observer.update event, *data
10
+ end
11
+ end
12
+
13
+ private
14
+ attr_reader :observers
15
+ end
16
+ end
@@ -0,0 +1,86 @@
1
+ require_relative 'null_observer'
2
+ require 'logger'
3
+
4
+ module Processor
5
+ module Observer
6
+ class Logger < NullObserver
7
+ def initialize(logger = nil, options = {})
8
+ @logger_source = logger
9
+ super options
10
+ end
11
+
12
+ def processing_started(processor)
13
+ initialize_logger(processor)
14
+ logger.info "Processing of #{processor.name} started."
15
+
16
+ message = <<-MESSAGE.gsub(/^\s+/, '')
17
+ Proggress will be saved to the log file. Run
18
+ tail -f #{log_file_name}
19
+ to see log in realtime
20
+ MESSAGE
21
+ messenger.info message if use_log_file?
22
+ end
23
+
24
+ def before_record_processing(record)
25
+ logger.debug "Record #{id_for record} is going to be processed"
26
+ end
27
+
28
+ def after_record_processing(record, result)
29
+ logger.info "Successfully processed #{id_for record}: #{result}"
30
+ end
31
+
32
+ def processing_finished(processor)
33
+ logger.info "Processing of #{processor.name} finished."
34
+ messenger.info "Log file saved to #{log_file_name}" if use_log_file?
35
+ end
36
+
37
+ def record_processing_error(record, exception)
38
+ logger.error "Error processing #{id_for record}: #{exception}"
39
+ end
40
+
41
+ def processing_error(processor, exception)
42
+ logger.fatal "Processing #{processor.name} failed: #{exception}"
43
+ end
44
+
45
+ private
46
+ attr_reader :logger, :log_file_name
47
+
48
+ def initialize_logger(processor)
49
+ @logger =
50
+ if @logger_source.is_a? Proc
51
+ @logger_source.call processor.name
52
+ else
53
+ @logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
54
+ logger.level = ::Logger::INFO
55
+ end
56
+ end
57
+ messenger.debug "Observer initialized with logger #{@logger}"
58
+ end
59
+
60
+ def create_log_filename(processor_name)
61
+ @log_file_name = "log/#{processor_name}_on_#{current_time_string}.log"
62
+ end
63
+
64
+ def use_log_file?
65
+ not log_file_name.nil?
66
+ end
67
+
68
+ def current_time_string
69
+ Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
70
+ end
71
+
72
+ def id_for record
73
+ [:uid, :id, :to_token, :token, :to_sym].each do |method|
74
+ return record.public_send method if record.respond_to? method
75
+ end
76
+
77
+ [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
78
+ return record[method] if record.key? method
79
+ return record[method.to_s] if record.key? method.to_s
80
+ end if record.respond_to?(:key?) && record.respond_to?(:[])
81
+
82
+ record.to_s
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,24 @@
1
+ require 'logger'
2
+
3
+ module Processor
4
+ module Observer
5
+ class NullObserver
6
+ def initialize(options = {})
7
+ @messenger = options.fetch :messenger do
8
+ ::Logger.new(STDOUT).tap do |logger|
9
+ logger.formatter = -> _, _, _, msg do
10
+ "> #{msg}\n"
11
+ end
12
+ logger.level = ::Logger::INFO
13
+ end
14
+ end
15
+ end
16
+
17
+ def method_missing(*); end
18
+ alias_method :update, :send
19
+
20
+ private
21
+ attr_reader :messenger
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,20 @@
1
+ module Processor
2
+ module RecordsProcessor
3
+ class Successive
4
+ def call(records, processor, events, recursion_preventer)
5
+ records.each do |record|
6
+ recursion_preventer.call
7
+ begin
8
+ events.register :before_record_processing, record
9
+
10
+ result = processor.process(record)
11
+
12
+ events.register :after_record_processing, record, result
13
+ rescue RuntimeError => exception
14
+ events.register :record_processing_error, record, exception
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,27 @@
1
+ module Processor
2
+ module RecordsProcessor
3
+ class Threads
4
+ def call(records, processor, events, recursion_preventer)
5
+ threads = []
6
+ begin
7
+ records.each do |record|
8
+ recursion_preventer.call
9
+ threads << ::Thread.new(processor, record) do |thread_data_processor, thread_record|
10
+ begin
11
+ events.register :before_record_processing, thread_record
12
+
13
+ result = thread_data_processor.process(thread_record)
14
+
15
+ events.register :after_record_processing, thread_record, result
16
+ rescue RuntimeError => exception
17
+ events.register :record_processing_error, thread_record, exception
18
+ end
19
+ end
20
+ end
21
+ ensure # join already created threads even if recursion was detected
22
+ threads.each(&:join)
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,39 @@
1
+ require_relative "events_registrator"
2
+
3
+ module Processor
4
+ class Runner
5
+ def initialize(processor, events_registrator)
6
+ @processor = processor
7
+ @events = events_registrator
8
+ end
9
+
10
+ def run(records_processor)
11
+ events.register :processing_started, processor
12
+ until processor.done?(records = processor.fetch_records)
13
+ records_processor.call records, processor, events, method(:recursion_preventer)
14
+ end
15
+
16
+ events.register :processing_finished, processor
17
+ rescue Exception => exception
18
+ events.register :processing_error, processor, exception
19
+ raise exception
20
+ end
21
+
22
+ protected
23
+ attr_writer :counter
24
+ def counter
25
+ @counter ||= 0
26
+ end
27
+
28
+ private
29
+ attr_reader :events, :processor
30
+ def recursion_preventer
31
+ self.counter += 1
32
+ raise Exception, "Processing fall into recursion. Check logs." if self.counter > max_records_to_process
33
+ end
34
+
35
+ def max_records_to_process
36
+ (processor.total_records * 1.1).round + 10
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,26 @@
1
+ require 'processor/runner'
2
+ require 'processor/records_processor/successive'
3
+ require 'processor/records_processor/threads'
4
+
5
+ module Processor
6
+ class Thread
7
+ def initialize(data_processor, *observers)
8
+ @runner = Runner.new data_processor, EventsRegistrator.new(observers)
9
+ end
10
+
11
+ def run_as(&records_processor)
12
+ runner.run records_processor
13
+ end
14
+
15
+ def run_successive
16
+ runner.run RecordsProcessor::Successive.new
17
+ end
18
+
19
+ def run_in_threads
20
+ runner.run RecordsProcessor::Threads.new
21
+ end
22
+
23
+ private
24
+ attr_reader :runner
25
+ end
26
+ end
@@ -1,3 +1,3 @@
1
1
  module Processor
2
- VERSION = "0.0.0.initial"
2
+ VERSION = "0.0.1"
3
3
  end
@@ -0,0 +1,18 @@
1
+ require 'spec_helper_lite'
2
+ require_relative '../example/migrator'
3
+ require_relative '../example/migration'
4
+
5
+ describe "Example" do
6
+ before(:each) do
7
+ records = %w[item1 item2 item3 item4 item5]
8
+ records.each do |record|
9
+ record.should_receive(:do_something).once
10
+ end
11
+ @migration = Processor::Example::Migration.new records
12
+ end
13
+
14
+ it "migration should use 2 loggers and messenger" do
15
+ migrator = Processor::Example::Migrator.new @migration
16
+ migrator.migrate
17
+ end
18
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/data_processor'
3
+
4
+ describe Processor::DataProcessor do
5
+ it "should have a name equals to underscored class name" do
6
+ subject.name.should eq "processor_data_processor"
7
+ end
8
+
9
+ it "should be done when there are 0 records to process" do
10
+ records = []
11
+ subject.done?(records).should be true
12
+ end
13
+
14
+ %w[done? fetch_records total_records process].each do |method_name|
15
+ it "should respond to #{method_name}" do
16
+ subject.should respond_to method_name
17
+ end
18
+ end
19
+ end
20
+
@@ -0,0 +1,15 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/events_registrator'
3
+
4
+ describe Processor::EventsRegistrator do
5
+ subject { Processor::EventsRegistrator }
6
+
7
+ it "should broadcast events to all observers" do
8
+ observers = 3.times.map do
9
+ stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
10
+ end
11
+
12
+ events = subject.new(observers)
13
+ events.register :test_event
14
+ end
15
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/observer/logger'
3
+
4
+ describe Processor::Observer::Logger do
5
+ let(:processor) { stub.as_null_object }
6
+ let(:no_messages) { ::Logger.new("/dev/null") }
7
+
8
+ subject { Processor::Observer::Logger }
9
+ it "accepts logger builder as parameter" do
10
+ external_logger = mock
11
+ logger_observer = subject.new -> name { external_logger }, messenger: no_messages
12
+
13
+ external_logger.should_receive(:info)
14
+ logger_observer.processing_started processor
15
+ end
16
+
17
+ it "accepts logger as parameter" do
18
+ external_logger = mock
19
+ logger_observer = subject.new external_logger, messenger: no_messages
20
+
21
+ external_logger.should_receive(:info)
22
+ logger_observer.processing_started processor
23
+ end
24
+
25
+ it "use ruby Logger if no external logger provided" do
26
+ logger_observer = subject.new nil, messenger: no_messages
27
+
28
+ Logger.should_receive(:new).and_return(stub.as_null_object)
29
+ logger_observer.processing_started processor
30
+ end
31
+ end
32
+
@@ -0,0 +1,38 @@
1
+ shared_examples_for "a records processor" do
2
+ let(:records_processor) { described_class.new }
3
+ let(:records) { 1..2 }
4
+ let(:processor) { stub }
5
+
6
+ let(:no_recursion_preventer) { Proc.new{} }
7
+ let(:no_events) { stub.as_null_object }
8
+
9
+ it "should send each found record to processor" do
10
+ records.each { |record| processor.should_receive(:process).with(record) }
11
+ records_processor.call records, processor, no_events, no_recursion_preventer
12
+ end
13
+
14
+ describe "exception handling" do
15
+ describe "processing a record raised RuntimeError" do
16
+ it "should continue processing" do
17
+ processor.should_receive(:process).twice.and_raise(RuntimeError)
18
+ expect { records_processor.call records, processor, no_events, no_recursion_preventer }.to_not raise_error
19
+ end
20
+
21
+ it "should register a record_processing_error event" do
22
+ event = mock.tap { |event| event.should_receive(:trigger).any_number_of_times }
23
+
24
+ events_registrator = stub
25
+ events_registrator.should_receive(:register) do |event_name, failed_record, exception|
26
+ next if event_name != :record_processing_error
27
+ event_name.should eq :record_processing_error
28
+ exception.should be_a RuntimeError
29
+ event.trigger
30
+ end.any_number_of_times
31
+
32
+ processor.stub(:process).and_raise(RuntimeError)
33
+
34
+ records_processor.call records, processor, events_registrator, no_recursion_preventer rescue nil
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,7 @@
1
+ require 'spec_helper_lite'
2
+ require_relative 'specs'
3
+ require 'processor/records_processor/successive'
4
+
5
+ describe Processor::RecordsProcessor::Successive do
6
+ it_behaves_like "a records processor"
7
+ end
@@ -0,0 +1,8 @@
1
+ require 'spec_helper_lite'
2
+ require_relative 'specs'
3
+ require 'processor/records_processor/threads'
4
+
5
+ describe Processor::RecordsProcessor::Threads do
6
+ it_behaves_like "a records processor"
7
+ it "should run in defined number of threads"
8
+ end
@@ -0,0 +1,103 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/runner'
3
+
4
+ describe Processor::Runner do
5
+ let(:runner) { Processor::Runner.new(processor, events_registrator) }
6
+ let(:processor) { stub }
7
+ let(:events_registrator) { stub.as_null_object }
8
+ let(:no_records_processor) { Proc.new{} }
9
+
10
+ it "should fetch records from processor till it'll be done" do
11
+ processor.stub(:done?).and_return(false, false, true)
12
+ processor.should_receive(:fetch_records).exactly(3).times.and_return([])
13
+ runner.run no_records_processor
14
+ end
15
+
16
+ describe "exception handling" do
17
+ let(:record) { stub }
18
+ before(:each) do
19
+ processor.stub(:fetch_records).and_return([record], [record], [])
20
+ end
21
+
22
+ describe "processing records raised" do
23
+ before(:each) do
24
+ processor.stub(done?: false)
25
+ end
26
+
27
+ it "should break processing and rerise" do
28
+ expect do
29
+ runner.run Proc.new { raise RuntimeError }
30
+ end.to raise_error(RuntimeError)
31
+ end
32
+
33
+ it "should register a processing_error event" do
34
+ register_processing_error_event mock.tap { |event| event.should_receive :trigger }
35
+ runner.run Proc.new { raise RuntimeError } rescue nil
36
+ end
37
+ end
38
+
39
+ describe "fetching records raised" do
40
+ it "should break processing and rerise Exception" do
41
+ processor.stub(:fetch_records).and_raise(RuntimeError)
42
+ processor.should_not_receive(:process)
43
+ expect { runner.run no_records_processor }.to raise_error(RuntimeError)
44
+ end
45
+
46
+ it "should register a processing_error" do
47
+ register_processing_error_event mock.tap { |event| event.should_receive :trigger }
48
+
49
+ processor.stub(:fetch_records).and_raise(RuntimeError)
50
+ runner.run no_records_processor rescue nil
51
+ end
52
+ end
53
+
54
+ private
55
+ def register_processing_error_event(event)
56
+ # Check that processing_error event was register
57
+ events_registrator.should_receive(:register) do |event_name, current_processor, exception|
58
+ next if event_name != :processing_error
59
+ event_name.should eq :processing_error
60
+ current_processor.should eq processor
61
+ exception.should be_a RuntimeError
62
+ event.trigger
63
+ end.any_number_of_times
64
+ end
65
+ end
66
+
67
+ describe "recursion" do
68
+ before(:each) do
69
+ processor.stub(done?: false)
70
+ processor.stub(total_records: 100)
71
+ processor.stub(fetch_records: 1..3)
72
+ end
73
+
74
+ it "should not fall into recursion" do
75
+ processor.should_receive(:process).at_most(1000).times
76
+
77
+ expect do
78
+ records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
79
+ records.each do |record|
80
+ recursion_preventer.call
81
+ records_processor.process record
82
+ end
83
+ end
84
+ runner.run records_processor
85
+ end.to raise_error(Exception, /Processing fall into recursion/)
86
+ end
87
+
88
+ it "should have 10% + 10 rerurns window" do
89
+ processor.should_receive(:process).exactly(120).times
90
+
91
+ expect do
92
+ records_processor = Proc.new do |records, records_processor, events, recursion_preventer|
93
+ records.each do |record|
94
+ recursion_preventer.call
95
+ processor.process record
96
+ end
97
+ end
98
+ runner.run records_processor
99
+ end.to raise_error(Exception, /Processing fall into recursion/)
100
+ end
101
+ end
102
+ end
103
+
@@ -0,0 +1,37 @@
1
+ require 'spec_helper_lite'
2
+ require 'processor/thread'
3
+ require_relative '../../example/migration'
4
+
5
+ describe Processor::Thread do
6
+ before(:each) do
7
+ records = %w[item1 item2 item3 item4 item5]
8
+ records.each do |record|
9
+ record.should_receive(:do_something).once
10
+ end
11
+ @migration = Processor::Example::Migration.new records
12
+ end
13
+
14
+ it "should run a migration using provided block" do
15
+ thread = Processor::Thread.new @migration
16
+ thread.run_as do |records, processor, *|
17
+ records.each do |record|
18
+ processor.process record
19
+ end
20
+ end
21
+ end
22
+
23
+ it "should run a migration successive" do
24
+ thread = Processor::Thread.new @migration
25
+ thread.run_successive
26
+ end
27
+
28
+ it "should run a migration in threads" do
29
+ thread = Processor::Thread.new @migration
30
+ thread.run_in_threads
31
+ end
32
+
33
+ pending "should run a migration in specifien number of threads" do
34
+ thread = Processor::Thread.new @migration
35
+ thread.run_in_threads 3
36
+ end
37
+ end
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: processor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0.initial
5
- prerelease: 6
4
+ version: 0.0.1
5
+ prerelease:
6
6
  platform: ruby
7
7
  authors:
8
8
  - Alexander Paramonov
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-04-30 00:00:00.000000000 Z
12
+ date: 2013-05-11 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
@@ -67,15 +67,39 @@ extensions: []
67
67
  extra_rdoc_files: []
68
68
  files:
69
69
  - .gitignore
70
+ - .rspec
70
71
  - .rvmrc
71
72
  - .travis.yml
72
73
  - Gemfile
73
74
  - LICENSE
74
75
  - README.md
75
76
  - Rakefile
77
+ - example/logging_thread.rb
78
+ - example/migration.rb
79
+ - example/migrator.rb
80
+ - example/observer/progress_bar.rb
81
+ - example/solr_migration.rb
82
+ - example/solr_pages_migration.rb
76
83
  - lib/processor.rb
84
+ - lib/processor/data_processor.rb
85
+ - lib/processor/events_registrator.rb
86
+ - lib/processor/observer/logger.rb
87
+ - lib/processor/observer/null_observer.rb
88
+ - lib/processor/records_processor/successive.rb
89
+ - lib/processor/records_processor/threads.rb
90
+ - lib/processor/runner.rb
91
+ - lib/processor/thread.rb
77
92
  - lib/processor/version.rb
78
93
  - processor.gemspec
94
+ - spec/example_spec.rb
95
+ - spec/processor/data_processor_spec.rb
96
+ - spec/processor/events_registrator_spec.rb
97
+ - spec/processor/observer/logger_spec.rb
98
+ - spec/processor/records_processor/specs.rb
99
+ - spec/processor/records_processor/successive_spec.rb
100
+ - spec/processor/records_processor/threads_spec.rb
101
+ - spec/processor/runner_spec.rb
102
+ - spec/processor/thread_spec.rb
79
103
  - spec/spec_helper_lite.rb
80
104
  homepage: http://github.com/AlexParamonov/processor
81
105
  licenses:
@@ -90,12 +114,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
90
114
  - - ! '>='
91
115
  - !ruby/object:Gem::Version
92
116
  version: '0'
117
+ segments:
118
+ - 0
119
+ hash: 4470760754938067473
93
120
  required_rubygems_version: !ruby/object:Gem::Requirement
94
121
  none: false
95
122
  requirements:
96
- - - ! '>'
123
+ - - ! '>='
97
124
  - !ruby/object:Gem::Version
98
- version: 1.3.1
125
+ version: '0'
126
+ segments:
127
+ - 0
128
+ hash: 4470760754938067473
99
129
  requirements: []
100
130
  rubyforge_project:
101
131
  rubygems_version: 1.8.25
@@ -103,4 +133,13 @@ signing_key:
103
133
  specification_version: 3
104
134
  summary: Process records one by one
105
135
  test_files:
136
+ - spec/example_spec.rb
137
+ - spec/processor/data_processor_spec.rb
138
+ - spec/processor/events_registrator_spec.rb
139
+ - spec/processor/observer/logger_spec.rb
140
+ - spec/processor/records_processor/specs.rb
141
+ - spec/processor/records_processor/successive_spec.rb
142
+ - spec/processor/records_processor/threads_spec.rb
143
+ - spec/processor/runner_spec.rb
144
+ - spec/processor/thread_spec.rb
106
145
  - spec/spec_helper_lite.rb