turbot-runner-morph 0.0.13 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +13 -5
- data/.gitignore +8 -5
- data/.gitmodules +1 -1
- data/.rspec +3 -0
- data/.travis.yml +11 -0
- data/README.md +12 -1
- data/lib/turbot_runner/prerun.rb +2 -2
- data/lib/turbot_runner/processor.rb +3 -0
- data/lib/turbot_runner/runner.rb +13 -7
- data/lib/turbot_runner/validator.rb +10 -12
- data/lib/turbot_runner/version.rb +1 -1
- data/schema/schemas/gazette-notice-schema.json +6 -6
- data/spec/lib/processor_spec.rb +1 -1
- data/spec/lib/runner_spec.rb +30 -51
- data/spec/lib/validator_spec.rb +41 -3
- data/spec/spec_helper.rb +13 -16
- data/spec/support/custom_matchers.rb +58 -0
- data/spec/support/helpers.rb +6 -0
- data/turbot-runner-morph.gemspec +4 -4
- metadata +25 -36
- data/bin/rspec +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
OTE4ZDRmNDVhZjA4Mzg0MzdlNDJiYmU3ZTY5OTEzNDMzZjg3ZTUzYQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
YmM3MjhkN2RmMDQ0NzU1OGM3NzQ3MGI3MTM1YWQxZDkyOTc4NzcwMg==
|
5
7
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MmEyZDIwNGU4ODg4NDlhOGZkZWMzYmY0M2UzYWY5NTFkNjQzYmNkNDQ1YjEx
|
10
|
+
NDA0NDFiYzBiZmQwN2M2ODg0YzMxYjE3OGExOWE0MmViYWE2MzBlNjk5ZDJi
|
11
|
+
OWE0YWQxYjllZWVjMWEyYzVmYWM4MTEwNThlZGIyMzc1YmQ3MTA=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ZWE3Mzc5Y2E5ZTBiY2RiYmEwZjIyNzI5YjNiYTg3YmJkOWI0YjljZGRjOTQy
|
14
|
+
Y2VjM2Q2NGMyNzI1NGNiZDAxNWYwZjU4NWIwNmVhNDNmMmY0NGNlZjgxYjlk
|
15
|
+
MDdjZWI5MDE5NDE5YWZmNWEyODViNzIyMjk0NzQyNTM1NDI5MTk=
|
data/.gitignore
CHANGED
data/.gitmodules
CHANGED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,5 +1,11 @@
|
|
1
1
|
# turbot-runner-morph
|
2
2
|
|
3
|
+
[](https://badge.fury.io/rb/turbot-runner-morph)
|
4
|
+
[](https://travis-ci.org/openc/turbot-runner-morph)
|
5
|
+
[](https://gemnasium.com/openc/turbot-runner-morph)
|
6
|
+
[](https://coveralls.io/r/openc/turbot-runner-morph)
|
7
|
+
[](https://codeclimate.com/github/openc/turbot-runner-morph)
|
8
|
+
|
3
9
|
## Getting started
|
4
10
|
|
5
11
|
git submodule update --init
|
@@ -17,4 +23,9 @@ Bump the version in `lib/turbot_runner/version.rb` according to the [Semantic Ve
|
|
17
23
|
git commit lib/turbot_runner/version.rb -m 'Release new version'
|
18
24
|
rake release # requires Rubygems credentials
|
19
25
|
|
20
|
-
|
26
|
+
In [morph](https://github.com/sebbacon/morph), run:
|
27
|
+
|
28
|
+
bundle update turbot-runner-morph
|
29
|
+
git commit Gemfile.lock -m 'Bump turbot-runner-morph' && git push
|
30
|
+
|
31
|
+
Finally, deploy [morph](https://github.com/sebbacon/morph).
|
data/lib/turbot_runner/prerun.rb
CHANGED
@@ -1,3 +1,3 @@
|
|
1
1
|
# Disable output buffering
|
2
|
-
|
3
|
-
|
2
|
+
$stdout.sync = true
|
3
|
+
$stderr.sync = true
|
@@ -7,6 +7,7 @@ module TurbotRunner
|
|
7
7
|
@data_type = script_config[:data_type]
|
8
8
|
@identifying_fields = script_config[:identifying_fields]
|
9
9
|
@record_handler = record_handler
|
10
|
+
# @seen_uids = script_config[:duplicates_allowed] ? nil : Set.new
|
10
11
|
end
|
11
12
|
|
12
13
|
def process(line)
|
@@ -17,12 +18,14 @@ module TurbotRunner
|
|
17
18
|
else
|
18
19
|
record = Openc::JsonSchema.convert_dates(schema_path, JSON.parse(line))
|
19
20
|
|
21
|
+
# TODO Document why we aren't passing retrieved_at to the validator.
|
20
22
|
record_to_validate = record.select {|k, v| k != 'retrieved_at'}
|
21
23
|
|
22
24
|
error_message = Validator.validate(
|
23
25
|
@data_type,
|
24
26
|
record_to_validate,
|
25
27
|
@identifying_fields
|
28
|
+
# @seen_uids
|
26
29
|
)
|
27
30
|
|
28
31
|
if error_message.nil?
|
data/lib/turbot_runner/runner.rb
CHANGED
@@ -4,9 +4,9 @@ require 'pathname'
|
|
4
4
|
|
5
5
|
module TurbotRunner
|
6
6
|
class Runner
|
7
|
-
|
8
|
-
|
9
|
-
|
7
|
+
RC_OK = 0
|
8
|
+
RC_SCRAPER_FAILED = 1
|
9
|
+
RC_TRANSFORMER_FAILED = 2
|
10
10
|
|
11
11
|
attr_reader :base_directory
|
12
12
|
|
@@ -17,13 +17,13 @@ module TurbotRunner
|
|
17
17
|
@record_handler = options[:record_handler]
|
18
18
|
@log_to_file = options[:log_to_file]
|
19
19
|
@timeout = options[:timeout]
|
20
|
+
@scraper_provided = options[:scraper_provided]
|
20
21
|
if options[:output_directory]
|
21
22
|
assert_absolute_path(options[:output_directory])
|
22
23
|
@output_directory = options[:output_directory]
|
23
24
|
else
|
24
25
|
@output_directory = File.join(@base_directory, 'output')
|
25
26
|
end
|
26
|
-
@scraper_provided = options[:scraper_provided]
|
27
27
|
end
|
28
28
|
|
29
29
|
def run
|
@@ -38,9 +38,10 @@ module TurbotRunner
|
|
38
38
|
# Run the transformers even if the scraper fails
|
39
39
|
transformers_succeeded = true
|
40
40
|
transformers.each do |transformer_config|
|
41
|
-
config = transformer_config.merge(
|
41
|
+
config = transformer_config.merge({
|
42
42
|
:base_directory => @base_directory,
|
43
|
-
|
43
|
+
# :duplicates_allowed => duplicates_allowed,
|
44
|
+
})
|
44
45
|
transformers_succeeded = run_script(config, input_file=scraper_output_file) && transformers_succeeded
|
45
46
|
end
|
46
47
|
|
@@ -164,7 +165,8 @@ module TurbotRunner
|
|
164
165
|
:base_directory => @base_directory,
|
165
166
|
:file => scraper_script,
|
166
167
|
:data_type => scraper_data_type,
|
167
|
-
:identifying_fields => scraper_identifying_fields
|
168
|
+
:identifying_fields => scraper_identifying_fields,
|
169
|
+
# :duplicates_allowed => duplicates_allowed,
|
168
170
|
}
|
169
171
|
end
|
170
172
|
|
@@ -192,6 +194,10 @@ module TurbotRunner
|
|
192
194
|
@config[:identifying_fields]
|
193
195
|
end
|
194
196
|
|
197
|
+
# def duplicates_allowed
|
198
|
+
# @config[:duplicates_allowed]
|
199
|
+
# end
|
200
|
+
|
195
201
|
def assert_absolute_path(path)
|
196
202
|
unless Pathname.new(path).absolute?
|
197
203
|
raise "#{path} must be an absolute path"
|
@@ -6,23 +6,21 @@ module TurbotRunner
|
|
6
6
|
schema_path = TurbotRunner.schema_path(data_type)
|
7
7
|
error = Openc::JsonSchema.validate(schema_path, record)
|
8
8
|
|
9
|
-
|
9
|
+
if error
|
10
|
+
return error[:message]
|
11
|
+
end
|
10
12
|
|
11
|
-
|
12
|
-
flattened_record = TurbotRunner::Utils.flatten(record)
|
13
|
+
flattened_record = TurbotRunner::Utils.flatten(record)
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
15
|
+
identifying_attributes = flattened_record.reject do |k, v|
|
16
|
+
!identifying_fields.include?(k) || v.nil? || v == ''
|
17
|
+
end
|
17
18
|
|
18
|
-
|
19
|
-
|
20
|
-
end
|
21
|
-
else
|
22
|
-
message = error[:message]
|
19
|
+
if identifying_attributes.empty?
|
20
|
+
return "There were no values provided for any of the identifying fields: #{identifying_fields.join(', ')}"
|
23
21
|
end
|
24
22
|
|
25
|
-
|
23
|
+
nil
|
26
24
|
end
|
27
25
|
end
|
28
26
|
end
|
@@ -45,7 +45,7 @@
|
|
45
45
|
"description": "The type of creation",
|
46
46
|
"type": "array",
|
47
47
|
"items": {
|
48
|
-
"
|
48
|
+
"type": "string"
|
49
49
|
}
|
50
50
|
},
|
51
51
|
"body": {
|
@@ -99,7 +99,7 @@
|
|
99
99
|
"description": "The type of registration",
|
100
100
|
"type": "array",
|
101
101
|
"items": {
|
102
|
-
"
|
102
|
+
"type": "string"
|
103
103
|
}
|
104
104
|
},
|
105
105
|
"body": {
|
@@ -158,7 +158,7 @@
|
|
158
158
|
"description": "The type of sale",
|
159
159
|
"type": "array",
|
160
160
|
"items": {
|
161
|
-
"
|
161
|
+
"type": "string"
|
162
162
|
}
|
163
163
|
},
|
164
164
|
"body": {
|
@@ -243,7 +243,7 @@
|
|
243
243
|
"description": "The type of judgment",
|
244
244
|
"type": "array",
|
245
245
|
"items": {
|
246
|
-
"
|
246
|
+
"type": "string"
|
247
247
|
}
|
248
248
|
},
|
249
249
|
"body": {
|
@@ -271,7 +271,7 @@
|
|
271
271
|
"description": "The type of judgment",
|
272
272
|
"type": "array",
|
273
273
|
"items": {
|
274
|
-
"
|
274
|
+
"type": "string"
|
275
275
|
}
|
276
276
|
},
|
277
277
|
"body": {
|
@@ -353,7 +353,7 @@
|
|
353
353
|
"description": "The type of filing",
|
354
354
|
"type": "array",
|
355
355
|
"items": {
|
356
|
-
"
|
356
|
+
"type": "string"
|
357
357
|
}
|
358
358
|
},
|
359
359
|
"body": {
|
data/spec/lib/processor_spec.rb
CHANGED
@@ -142,7 +142,7 @@ describe TurbotRunner::Processor do
|
|
142
142
|
}
|
143
143
|
|
144
144
|
expect(TurbotRunner::Validator).to receive(:validate).
|
145
|
-
with('primary data', expected_record_to_validate, ['number'])
|
145
|
+
with('primary data', expected_record_to_validate, ['number']) #, Set.new)
|
146
146
|
@processor.process(record.to_json)
|
147
147
|
end
|
148
148
|
end
|
data/spec/lib/runner_spec.rb
CHANGED
@@ -173,7 +173,7 @@ describe TurbotRunner::Runner do
|
|
173
173
|
context 'with a handler that interrupts the runner' do
|
174
174
|
before do
|
175
175
|
class Handler < TurbotRunner::BaseHandler
|
176
|
-
def initialize
|
176
|
+
def initialize
|
177
177
|
@count = 0
|
178
178
|
super
|
179
179
|
end
|
@@ -249,13 +249,34 @@ describe TurbotRunner::Runner do
|
|
249
249
|
end
|
250
250
|
end
|
251
251
|
|
252
|
+
# context 'with a bot that produces duplicate data' do
|
253
|
+
# before do
|
254
|
+
# @runner = test_runner('bot-that-produces-duplicates')
|
255
|
+
# end
|
256
|
+
|
257
|
+
# it 'raises returns false' do
|
258
|
+
# expect(@runner).to fail_in_scraper
|
259
|
+
# end
|
260
|
+
# end
|
261
|
+
|
262
|
+
# context 'with a bot that is expected to produce duplicate data' do
|
263
|
+
# before do
|
264
|
+
# @runner = test_runner('bot-that-is-allowed-to-produce-duplicates')
|
265
|
+
# end
|
266
|
+
|
267
|
+
# it 'raises returns false' do
|
268
|
+
# expect(@runner).to succeed
|
269
|
+
# end
|
270
|
+
# end
|
271
|
+
|
252
272
|
context 'when the scraped data is provided' do
|
253
273
|
before do
|
274
|
+
@runner = test_runner('bot-with-transformer', :scraper_provided => true)
|
275
|
+
@runner.set_up_output_directory
|
254
276
|
FileUtils.cp(
|
255
277
|
File.join('spec', 'outputs', 'full-scraper.out'),
|
256
278
|
File.join(File.dirname(__FILE__), '../bots', 'bot-with-transformer', 'output', 'scraper.out')
|
257
279
|
)
|
258
|
-
@runner = test_runner('bot-with-transformer', :scraper_provided => true)
|
259
280
|
end
|
260
281
|
|
261
282
|
it 'does not run scraper' do
|
@@ -281,8 +302,8 @@ describe TurbotRunner::Runner do
|
|
281
302
|
class Handler < TurbotRunner::BaseHandler
|
282
303
|
attr_reader :records_seen
|
283
304
|
|
284
|
-
def initialize
|
285
|
-
@records_seen = Hash.new
|
305
|
+
def initialize
|
306
|
+
@records_seen = Hash.new(0)
|
286
307
|
super
|
287
308
|
end
|
288
309
|
|
@@ -309,7 +330,7 @@ describe TurbotRunner::Runner do
|
|
309
330
|
it 'can cope when scraper has failed immediately' do
|
310
331
|
test_runner('bot-that-crashes-immediately').run
|
311
332
|
|
312
|
-
runner = test_runner('bot-
|
333
|
+
runner = test_runner('bot-that-crashes-immediately',
|
313
334
|
:record_handler => @handler
|
314
335
|
)
|
315
336
|
|
@@ -323,63 +344,21 @@ describe TurbotRunner::Runner do
|
|
323
344
|
end
|
324
345
|
|
325
346
|
it 'clears existing output' do
|
347
|
+
@runner.set_up_output_directory
|
326
348
|
path = File.join(@runner.base_directory, 'output', 'scraper.out')
|
327
349
|
FileUtils.touch(path)
|
350
|
+
|
328
351
|
@runner.set_up_output_directory
|
329
352
|
expect(File.exist?(path)).to be(false)
|
330
353
|
end
|
331
354
|
|
332
355
|
it 'does not clear existing files that are not output files' do
|
356
|
+
@runner.set_up_output_directory
|
333
357
|
path = File.join(@runner.base_directory, 'output', 'stdout')
|
334
358
|
FileUtils.touch(path)
|
359
|
+
|
335
360
|
@runner.set_up_output_directory
|
336
361
|
expect(File.exist?(path)).to be(true)
|
337
362
|
end
|
338
363
|
end
|
339
364
|
end
|
340
|
-
|
341
|
-
|
342
|
-
RSpec::Matchers.define(:have_output) do |script, expected|
|
343
|
-
match do |runner|
|
344
|
-
expected_path = File.join('spec', 'outputs', expected)
|
345
|
-
expected_output = File.readlines(expected_path).map {|line| JSON.parse(line)}
|
346
|
-
actual_path = File.join(runner.base_directory, 'output', "#{script}.out")
|
347
|
-
actual_output = File.readlines(actual_path).map {|line| JSON.parse(line)}
|
348
|
-
expect(expected_output).to eq(actual_output)
|
349
|
-
end
|
350
|
-
end
|
351
|
-
|
352
|
-
|
353
|
-
RSpec::Matchers.define(:have_error_output_matching) do |script, expected|
|
354
|
-
match do |runner|
|
355
|
-
actual_path = File.join(runner.base_directory, 'output', "#{script}.err")
|
356
|
-
actual_output = File.read(actual_path)
|
357
|
-
expect(actual_output).to match(expected)
|
358
|
-
end
|
359
|
-
end
|
360
|
-
|
361
|
-
|
362
|
-
RSpec::Matchers.define(:succeed) do
|
363
|
-
match do |runner|
|
364
|
-
expect(runner.run).to eq(TurbotRunner::Runner::RC_OK)
|
365
|
-
end
|
366
|
-
end
|
367
|
-
|
368
|
-
|
369
|
-
RSpec::Matchers.define(:fail_in_scraper) do
|
370
|
-
match do |runner|
|
371
|
-
expect(runner.run).to eq(TurbotRunner::Runner::RC_SCRAPER_FAILED)
|
372
|
-
end
|
373
|
-
end
|
374
|
-
|
375
|
-
|
376
|
-
RSpec::Matchers.define(:fail_in_transformer) do
|
377
|
-
match do |runner|
|
378
|
-
expect(runner.run).to eq(TurbotRunner::Runner::RC_TRANSFORMER_FAILED)
|
379
|
-
end
|
380
|
-
end
|
381
|
-
|
382
|
-
def test_runner(name, opts={})
|
383
|
-
test_bot_location = File.join(File.dirname(__FILE__), '../bots', name)
|
384
|
-
TurbotRunner::Runner.new(test_bot_location, opts)
|
385
|
-
end
|
data/spec/lib/validator_spec.rb
CHANGED
@@ -58,7 +58,7 @@ describe TurbotRunner::Validator do
|
|
58
58
|
'four' => {}
|
59
59
|
}
|
60
60
|
identifying_fields = ['one.two.three', 'four.five.six']
|
61
|
-
error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
|
61
|
+
error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields) #, Set.new)
|
62
62
|
expect(error).to eq('There were no values provided for any of the identifying fields: one.two.three, four.five.six')
|
63
63
|
end
|
64
64
|
|
@@ -69,7 +69,7 @@ describe TurbotRunner::Validator do
|
|
69
69
|
'one' => {'two' => {'three' => 123}}
|
70
70
|
}
|
71
71
|
identifying_fields = ['one.two.three', 'four.five.six']
|
72
|
-
error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
|
72
|
+
error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields) #, Set.new)
|
73
73
|
expect(error).to eq(nil)
|
74
74
|
end
|
75
75
|
|
@@ -81,9 +81,47 @@ describe TurbotRunner::Validator do
|
|
81
81
|
'four' => {'five' => {'six' => 456}}
|
82
82
|
}
|
83
83
|
identifying_fields = ['one.two.three', 'four.five.six']
|
84
|
-
error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields)
|
84
|
+
error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields) #, Set.new)
|
85
85
|
expect(error).to eq(nil)
|
86
86
|
end
|
87
87
|
end
|
88
|
+
|
89
|
+
# specify 'with duplicate record' do
|
90
|
+
# record = {
|
91
|
+
# 'sample_date' => '2014-06-01',
|
92
|
+
# 'source_url' => 'http://example.com/123',
|
93
|
+
# 'number' => 123
|
94
|
+
# }
|
95
|
+
|
96
|
+
# seen_uids = Set.new
|
97
|
+
# error = TurbotRunner::Validator.validate('primary-data', record, ['number'], seen_uids)
|
98
|
+
# expect(error).to eq(nil)
|
99
|
+
|
100
|
+
# error = TurbotRunner::Validator.validate('primary-data', record, ['number'], seen_uids)
|
101
|
+
# expect(error).to eq('Already seen record with these identifying fields: {"number"=>123}')
|
102
|
+
# end
|
103
|
+
# end
|
104
|
+
|
105
|
+
# describe '.identifying_hash' do
|
106
|
+
# specify 'returns expected hash' do
|
107
|
+
# record = {'aaa' => 'bbb', 'yyy' => 'zzz'}
|
108
|
+
# expect(TurbotRunner::Validator.identifying_hash(record, ['aaa'])).to eq({'aaa' => 'bbb'})
|
109
|
+
|
110
|
+
# record = {'aaa' => {'bbb' => 'ccc'}, 'yyy' => 'zzz'}
|
111
|
+
# expect(TurbotRunner::Validator.identifying_hash(record, ['aaa.bbb'])).to eq({'aaa.bbb' => 'ccc'})
|
112
|
+
# end
|
113
|
+
|
114
|
+
# specify 'returns empty hash for records with no values for identifying fields' do
|
115
|
+
# record = {'yyy' => 'zzz'}
|
116
|
+
# expect(TurbotRunner::Validator.identifying_hash(record, ['aaa'])).to eq({})
|
117
|
+
# end
|
118
|
+
|
119
|
+
# specify 'returns nil for records where value of identifying field is a hash' do
|
120
|
+
# record = {'aaa' => {'bbb' => 'ccc'}, 'yyy' => 'zzz'}
|
121
|
+
# expect(TurbotRunner::Validator.identifying_hash(record, ['aaa'])).to be(nil)
|
122
|
+
|
123
|
+
# record = {'aaa' => {'bbb' => {'ccc' => 'ddd'}}, 'yyy' => 'zzz'}
|
124
|
+
# expect(TurbotRunner::Validator.identifying_hash(record, ['aaa.bbb'])).to be(nil)
|
125
|
+
# end
|
88
126
|
end
|
89
127
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,20 +1,17 @@
|
|
1
|
-
require '
|
1
|
+
require 'rubygems'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
end
|
9
|
-
|
10
|
-
failure_message do |actual|
|
11
|
-
"Expected error to be #{expected_error}, but was #{@error}"
|
12
|
-
end
|
3
|
+
require 'simplecov'
|
4
|
+
require 'coveralls'
|
5
|
+
SimpleCov.formatter = Coveralls::SimpleCov::Formatter
|
6
|
+
SimpleCov.start do
|
7
|
+
add_filter 'spec'
|
13
8
|
end
|
14
9
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
10
|
+
require 'rspec'
|
11
|
+
|
12
|
+
Dir['./spec/support/**/*.rb'].sort.each { |f| require f}
|
13
|
+
require File.dirname(__FILE__) + '/../lib/turbot_runner'
|
14
|
+
|
15
|
+
RSpec.configure do |c|
|
16
|
+
c.include(Helpers)
|
20
17
|
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
RSpec::Matchers.define(:fail_validation_with) do |expected_error|
|
2
|
+
match do |record|
|
3
|
+
identifying_fields = ['number']
|
4
|
+
@error = TurbotRunner::Validator.validate('primary-data', record, identifying_fields) #, Set.new)
|
5
|
+
expect(@error).to eq(expected_error)
|
6
|
+
end
|
7
|
+
|
8
|
+
failure_message do |actual|
|
9
|
+
"Expected error to be #{expected_error}, but was #{@error}"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
RSpec::Matchers.define(:be_valid) do
|
14
|
+
match do |record|
|
15
|
+
identifying_fields = ['number']
|
16
|
+
expect(TurbotRunner::Validator.validate('primary-data', record, identifying_fields)).to eq(nil) #, Set.new)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
RSpec::Matchers.define(:have_output) do |script, expected|
|
21
|
+
match do |runner|
|
22
|
+
expected_path = File.join('spec', 'outputs', expected)
|
23
|
+
@expected_output = File.readlines(expected_path).map {|line| JSON.parse(line)}
|
24
|
+
actual_path = File.join(runner.base_directory, 'output', "#{script}.out")
|
25
|
+
@actual_output = File.readlines(actual_path).map {|line| JSON.parse(line)}
|
26
|
+
expect(@expected_output).to eq(@actual_output)
|
27
|
+
end
|
28
|
+
|
29
|
+
failure_message do |actual|
|
30
|
+
"Expected output to be #{@expected_output}, but was #{@actual_output}"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
RSpec::Matchers.define(:have_error_output_matching) do |script, expected|
|
35
|
+
match do |runner|
|
36
|
+
actual_path = File.join(runner.base_directory, 'output', "#{script}.err")
|
37
|
+
actual_output = File.read(actual_path)
|
38
|
+
expect(actual_output).to match(expected)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
RSpec::Matchers.define(:succeed) do
|
43
|
+
match do |runner|
|
44
|
+
expect(runner.run).to eq(TurbotRunner::Runner::RC_OK)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
RSpec::Matchers.define(:fail_in_scraper) do
|
49
|
+
match do |runner|
|
50
|
+
expect(runner.run).to eq(TurbotRunner::Runner::RC_SCRAPER_FAILED)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
RSpec::Matchers.define(:fail_in_transformer) do
|
55
|
+
match do |runner|
|
56
|
+
expect(runner.run).to eq(TurbotRunner::Runner::RC_TRANSFORMER_FAILED)
|
57
|
+
end
|
58
|
+
end
|
data/turbot-runner-morph.gemspec
CHANGED
@@ -17,10 +17,10 @@ Gem::Specification.new do |gem|
|
|
17
17
|
|
18
18
|
gem.required_ruby_version = '>=1.9.2'
|
19
19
|
|
20
|
-
#
|
20
|
+
# gem.add_dependency "activesupport", '~> 4.1.4'
|
21
21
|
gem.add_dependency "openc-json_schema"
|
22
|
-
|
22
|
+
|
23
|
+
gem.add_development_dependency "coveralls"
|
23
24
|
gem.add_development_dependency "rake"
|
24
|
-
gem.add_development_dependency "rspec", "
|
25
|
-
gem.add_development_dependency "rspec-mocks", ">= 3.0"
|
25
|
+
gem.add_development_dependency "rspec", "~> 3.4.0"
|
26
26
|
end
|
metadata
CHANGED
@@ -1,98 +1,84 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: turbot-runner-morph
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- OpenCorporates
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: openc-json_schema
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ! '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ! '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: coveralls
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ! '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ! '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ! '>='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ! '>='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rspec
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ~>
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 3.4.0
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ~>
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: rspec-mocks
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '3.0'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '3.0'
|
68
|
+
version: 3.4.0
|
83
69
|
description:
|
84
70
|
email: bots@opencorporates.com
|
85
|
-
executables:
|
86
|
-
- rspec
|
71
|
+
executables: []
|
87
72
|
extensions: []
|
88
73
|
extra_rdoc_files: []
|
89
74
|
files:
|
90
|
-
-
|
91
|
-
-
|
75
|
+
- .gitignore
|
76
|
+
- .gitmodules
|
77
|
+
- .rspec
|
78
|
+
- .travis.yml
|
92
79
|
- Gemfile
|
93
80
|
- README.md
|
94
81
|
- Rakefile
|
95
|
-
- bin/rspec
|
96
82
|
- lib/turbot_runner.rb
|
97
83
|
- lib/turbot_runner/base_handler.rb
|
98
84
|
- lib/turbot_runner/exceptions.rb
|
@@ -191,6 +177,8 @@ files:
|
|
191
177
|
- spec/outputs/full-transformer.out
|
192
178
|
- spec/outputs/truncated-scraper.out
|
193
179
|
- spec/spec_helper.rb
|
180
|
+
- spec/support/custom_matchers.rb
|
181
|
+
- spec/support/helpers.rb
|
194
182
|
- turbot-runner-morph.gemspec
|
195
183
|
homepage: http://turbot.opencorporates.com/
|
196
184
|
licenses:
|
@@ -202,17 +190,17 @@ require_paths:
|
|
202
190
|
- lib
|
203
191
|
required_ruby_version: !ruby/object:Gem::Requirement
|
204
192
|
requirements:
|
205
|
-
- -
|
193
|
+
- - ! '>='
|
206
194
|
- !ruby/object:Gem::Version
|
207
195
|
version: 1.9.2
|
208
196
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
209
197
|
requirements:
|
210
|
-
- -
|
198
|
+
- - ! '>='
|
211
199
|
- !ruby/object:Gem::Version
|
212
200
|
version: '0'
|
213
201
|
requirements: []
|
214
202
|
rubyforge_project:
|
215
|
-
rubygems_version: 2.
|
203
|
+
rubygems_version: 2.5.2
|
216
204
|
signing_key:
|
217
205
|
specification_version: 4
|
218
206
|
summary: Utilities for running bots with Turbot
|
@@ -268,4 +256,5 @@ test_files:
|
|
268
256
|
- spec/outputs/full-transformer.out
|
269
257
|
- spec/outputs/truncated-scraper.out
|
270
258
|
- spec/spec_helper.rb
|
271
|
-
|
259
|
+
- spec/support/custom_matchers.rb
|
260
|
+
- spec/support/helpers.rb
|
data/bin/rspec
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
#
|
3
|
-
# This file was generated by Bundler.
|
4
|
-
#
|
5
|
-
# The application 'rspec' is installed as part of a gem, and
|
6
|
-
# this file is here to facilitate running it.
|
7
|
-
#
|
8
|
-
|
9
|
-
require 'pathname'
|
10
|
-
ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
|
11
|
-
Pathname.new(__FILE__).realpath)
|
12
|
-
|
13
|
-
require 'rubygems'
|
14
|
-
require 'bundler/setup'
|
15
|
-
|
16
|
-
load Gem.bin_path('rspec-core', 'rspec')
|