processor 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.coveralls.yml +1 -0
- data/.gitignore +1 -0
- data/README.md +68 -41
- data/example/migrator.rb +2 -0
- data/lib/processor/data/array_processor.rb +4 -0
- data/lib/processor/data/null_processor.rb +7 -10
- data/lib/processor/environment.rb +3 -0
- data/lib/processor/event_processor.rb +25 -0
- data/lib/processor/logger_messages.rb +56 -0
- data/lib/processor/messenger.rb +16 -3
- data/lib/processor/observer/logger.rb +43 -34
- data/lib/processor/observer/null_observer.rb +8 -3
- data/lib/processor/process_runner/threads.rb +3 -8
- data/lib/processor/runner.rb +8 -23
- data/lib/processor/subroutine/counter.rb +25 -0
- data/lib/processor/subroutine/name.rb +28 -0
- data/lib/processor/subroutine/recursion.rb +29 -0
- data/lib/processor/thread.rb +5 -1
- data/lib/processor/version.rb +1 -1
- data/lib/processor.rb +2 -1
- data/processor.gemspec +6 -4
- data/spec/processor/data/array_processor_spec.rb +1 -2
- data/spec/processor/data/null_processor_spec.rb +27 -2
- data/spec/processor/event_processor_spec.rb +58 -0
- data/spec/processor/logger_messages_spec.rb +69 -0
- data/spec/processor/messenger_spec.rb +15 -0
- data/spec/processor/observer/logger_spec.rb +47 -16
- data/spec/processor/observer/null_observer_spec.rb +33 -0
- data/spec/processor/process_runner/specs.rb +12 -23
- data/spec/processor/process_runner/threads_spec.rb +1 -3
- data/spec/processor/runner_spec.rb +28 -54
- data/spec/processor/subroutine/counter_spec.rb +44 -0
- data/spec/processor/subroutine/name_spec.rb +23 -0
- data/spec/processor/subroutine/recursion_spec.rb +23 -0
- data/spec/processor/thread_spec.rb +1 -1
- data/spec/spec_helper_lite.rb +7 -2
- data/spec/support/dummy_file +0 -0
- metadata +47 -12
- data/lib/processor/data/solr_processor.rb +0 -23
- data/lib/processor/events_registrator.rb +0 -16
- data/spec/processor/events_registrator_spec.rb +0 -15
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'delegate'
|
3
|
+
require 'processor/subroutine/name'
|
4
|
+
|
5
|
+
describe Processor::Subroutine::Name do
|
6
|
+
let(:processor) { stub.tap { |p| p.stub_chain(:class, :name).and_return("NameSpace::DataProcessor") } }
|
7
|
+
let(:subroutine) { Processor::Subroutine::Name.new processor }
|
8
|
+
|
9
|
+
it "should have a name equals to underscored processor class name" do
|
10
|
+
subroutine.name.should eq "name_space_data_processor"
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should use processor's name method if defined" do
|
14
|
+
processor.stub(name: "Processor's name")
|
15
|
+
subroutine.name.should eq "Processor's name"
|
16
|
+
end
|
17
|
+
|
18
|
+
it "should use real object's name, not delegator" do
|
19
|
+
double_wrapped_processor = Processor::Subroutine::Name.new(SimpleDelegator.new processor)
|
20
|
+
double_wrapped_processor.name.should eq "name_space_data_processor"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper_lite'
|
2
|
+
require 'processor/subroutine/recursion'
|
3
|
+
|
4
|
+
describe Processor::Subroutine::Recursion do
|
5
|
+
let(:processor) { stub(total_records: 100) }
|
6
|
+
let(:subroutine) { Processor::Subroutine::Recursion.new processor }
|
7
|
+
|
8
|
+
it "should not fall into recursion" do
|
9
|
+
processor.should_receive(:process).at_most(1000).times
|
10
|
+
|
11
|
+
expect do
|
12
|
+
1001.times { subroutine.process }
|
13
|
+
end.to raise_error(Exception, /Processing fall into recursion/)
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should have 10% + 10 rerurns window" do
|
17
|
+
processor.should_receive(:process).exactly(120).times
|
18
|
+
|
19
|
+
expect do
|
20
|
+
121.times { subroutine.process }
|
21
|
+
end.to raise_error(Exception, /Processing fall into recursion/)
|
22
|
+
end
|
23
|
+
end
|
@@ -13,7 +13,7 @@ describe Processor::Thread do
|
|
13
13
|
|
14
14
|
it "should run a migration using provided block" do
|
15
15
|
thread = Processor::Thread.new @migration
|
16
|
-
thread.run_as do |processor
|
16
|
+
thread.run_as do |processor|
|
17
17
|
processor.records.each do |record|
|
18
18
|
processor.process record
|
19
19
|
end
|
data/spec/spec_helper_lite.rb
CHANGED
File without changes
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: processor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-06-08 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -59,15 +59,32 @@ dependencies:
|
|
59
59
|
- - ! '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
|
-
|
63
|
-
|
64
|
-
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: pry-plus
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
description: ! 'Processor is a tool that helps to iterate over collection and perform
|
79
|
+
complex actions on a result. It is extremely useful in data migrations, report generation,
|
80
|
+
etc. '
|
65
81
|
email:
|
66
82
|
- alexander.n.paramonov@gmail.com
|
67
83
|
executables: []
|
68
84
|
extensions: []
|
69
85
|
extra_rdoc_files: []
|
70
86
|
files:
|
87
|
+
- .coveralls.yml
|
71
88
|
- .gitignore
|
72
89
|
- .rspec
|
73
90
|
- .rvmrc
|
@@ -86,14 +103,18 @@ files:
|
|
86
103
|
- lib/processor/data/csv_processor.rb
|
87
104
|
- lib/processor/data/null_processor.rb
|
88
105
|
- lib/processor/data/solr_pages_processor.rb
|
89
|
-
- lib/processor/
|
90
|
-
- lib/processor/
|
106
|
+
- lib/processor/environment.rb
|
107
|
+
- lib/processor/event_processor.rb
|
108
|
+
- lib/processor/logger_messages.rb
|
91
109
|
- lib/processor/messenger.rb
|
92
110
|
- lib/processor/observer/logger.rb
|
93
111
|
- lib/processor/observer/null_observer.rb
|
94
112
|
- lib/processor/process_runner/successive.rb
|
95
113
|
- lib/processor/process_runner/threads.rb
|
96
114
|
- lib/processor/runner.rb
|
115
|
+
- lib/processor/subroutine/counter.rb
|
116
|
+
- lib/processor/subroutine/name.rb
|
117
|
+
- lib/processor/subroutine/recursion.rb
|
97
118
|
- lib/processor/thread.rb
|
98
119
|
- lib/processor/version.rb
|
99
120
|
- processor.gemspec
|
@@ -101,15 +122,21 @@ files:
|
|
101
122
|
- spec/processor/data/array_processor_spec.rb
|
102
123
|
- spec/processor/data/batch_processor_spec.rb
|
103
124
|
- spec/processor/data/null_processor_spec.rb
|
104
|
-
- spec/processor/
|
125
|
+
- spec/processor/event_processor_spec.rb
|
126
|
+
- spec/processor/logger_messages_spec.rb
|
105
127
|
- spec/processor/messenger_spec.rb
|
106
128
|
- spec/processor/observer/logger_spec.rb
|
129
|
+
- spec/processor/observer/null_observer_spec.rb
|
107
130
|
- spec/processor/process_runner/specs.rb
|
108
131
|
- spec/processor/process_runner/successive_spec.rb
|
109
132
|
- spec/processor/process_runner/threads_spec.rb
|
110
133
|
- spec/processor/runner_spec.rb
|
134
|
+
- spec/processor/subroutine/counter_spec.rb
|
135
|
+
- spec/processor/subroutine/name_spec.rb
|
136
|
+
- spec/processor/subroutine/recursion_spec.rb
|
111
137
|
- spec/processor/thread_spec.rb
|
112
138
|
- spec/spec_helper_lite.rb
|
139
|
+
- spec/support/dummy_file
|
113
140
|
homepage: http://github.com/AlexParamonov/processor
|
114
141
|
licenses:
|
115
142
|
- MIT
|
@@ -125,7 +152,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
125
152
|
version: '0'
|
126
153
|
segments:
|
127
154
|
- 0
|
128
|
-
hash:
|
155
|
+
hash: 81022750956299977
|
129
156
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
130
157
|
none: false
|
131
158
|
requirements:
|
@@ -134,24 +161,32 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
134
161
|
version: '0'
|
135
162
|
segments:
|
136
163
|
- 0
|
137
|
-
hash:
|
164
|
+
hash: 81022750956299977
|
138
165
|
requirements: []
|
139
166
|
rubyforge_project:
|
140
167
|
rubygems_version: 1.8.25
|
141
168
|
signing_key:
|
142
169
|
specification_version: 3
|
143
|
-
summary: Universal processor for
|
170
|
+
summary: Universal processor for iteration over a collection with threads, logging
|
171
|
+
and post processing
|
144
172
|
test_files:
|
145
173
|
- spec/example_spec.rb
|
146
174
|
- spec/processor/data/array_processor_spec.rb
|
147
175
|
- spec/processor/data/batch_processor_spec.rb
|
148
176
|
- spec/processor/data/null_processor_spec.rb
|
149
|
-
- spec/processor/
|
177
|
+
- spec/processor/event_processor_spec.rb
|
178
|
+
- spec/processor/logger_messages_spec.rb
|
150
179
|
- spec/processor/messenger_spec.rb
|
151
180
|
- spec/processor/observer/logger_spec.rb
|
181
|
+
- spec/processor/observer/null_observer_spec.rb
|
152
182
|
- spec/processor/process_runner/specs.rb
|
153
183
|
- spec/processor/process_runner/successive_spec.rb
|
154
184
|
- spec/processor/process_runner/threads_spec.rb
|
155
185
|
- spec/processor/runner_spec.rb
|
186
|
+
- spec/processor/subroutine/counter_spec.rb
|
187
|
+
- spec/processor/subroutine/name_spec.rb
|
188
|
+
- spec/processor/subroutine/recursion_spec.rb
|
156
189
|
- spec/processor/thread_spec.rb
|
157
190
|
- spec/spec_helper_lite.rb
|
191
|
+
- spec/support/dummy_file
|
192
|
+
has_rdoc:
|
@@ -1,23 +0,0 @@
|
|
1
|
-
require_relative "batch_processor"
|
2
|
-
|
3
|
-
module Processor
|
4
|
-
module Data
|
5
|
-
class SolrProcessor < BatchProcessor
|
6
|
-
def process(record)
|
7
|
-
raise NotImplementedError
|
8
|
-
end
|
9
|
-
|
10
|
-
def query
|
11
|
-
raise NotImplementedError
|
12
|
-
end
|
13
|
-
|
14
|
-
def fetch_batch
|
15
|
-
query.results
|
16
|
-
end
|
17
|
-
|
18
|
-
def total_records
|
19
|
-
@total_records ||= query.total
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
@@ -1,16 +0,0 @@
|
|
1
|
-
module Processor
|
2
|
-
class EventsRegistrator
|
3
|
-
def initialize(observers)
|
4
|
-
@observers = observers
|
5
|
-
end
|
6
|
-
|
7
|
-
def register(event, *data)
|
8
|
-
observers.each do |observer|
|
9
|
-
observer.update event, *data
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
private
|
14
|
-
attr_reader :observers
|
15
|
-
end
|
16
|
-
end
|
@@ -1,15 +0,0 @@
|
|
1
|
-
require 'spec_helper_lite'
|
2
|
-
require 'processor/events_registrator'
|
3
|
-
|
4
|
-
describe Processor::EventsRegistrator do
|
5
|
-
subject { Processor::EventsRegistrator }
|
6
|
-
|
7
|
-
it "should broadcast events to all observers" do
|
8
|
-
observers = 3.times.map do
|
9
|
-
stub(:observer).tap { |observer| observer.should_receive(:update).with(:test_event).once }
|
10
|
-
end
|
11
|
-
|
12
|
-
events = subject.new(observers)
|
13
|
-
events.register :test_event
|
14
|
-
end
|
15
|
-
end
|