turbot-runner 0.1.18 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/turbot_runner/runner.rb +25 -12
- data/lib/turbot_runner/script_runner.rb +46 -45
- data/lib/turbot_runner/validators.rb +12 -0
- data/lib/turbot_runner/version.rb +1 -1
- data/lib/turbot_runner.rb +1 -0
- data/spec/bots/bot-that-expects-file/manifest.json +8 -0
- data/spec/bots/bot-that-expects-file/scraper.rb +11 -0
- data/spec/bots/bot-that-expects-file/something.txt +1 -0
- data/spec/bots/bot-with-invalid-sample-date/manifest.json +8 -0
- data/spec/bots/bot-with-invalid-sample-date/scraper.rb +10 -0
- data/spec/lib/runner_spec.rb +51 -37
- data/spec/manual_spec.rb +4 -2
- metadata +91 -63
- checksums.yaml +0 -15
data/lib/turbot_runner/runner.rb
CHANGED
@@ -1,17 +1,24 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'fileutils'
|
3
|
+
require 'pathname'
|
3
4
|
|
4
5
|
module TurbotRunner
|
5
6
|
class Runner
|
6
|
-
attr_reader :
|
7
|
+
attr_reader :base_directory
|
7
8
|
|
8
9
|
def initialize(directory, options={})
|
9
|
-
|
10
|
+
assert_absolute_path(directory)
|
11
|
+
@base_directory = directory
|
10
12
|
@config = load_config(directory)
|
11
13
|
@record_handler = options[:record_handler]
|
12
14
|
@log_to_file = options[:log_to_file]
|
13
15
|
@timeout = options[:timeout]
|
14
|
-
|
16
|
+
if options[:output_directory]
|
17
|
+
assert_absolute_path(options[:output_directory])
|
18
|
+
@output_directory = options[:output_directory]
|
19
|
+
else
|
20
|
+
@output_directory = File.join(@base_directory, 'output')
|
21
|
+
end
|
15
22
|
end
|
16
23
|
|
17
24
|
def run
|
@@ -20,14 +27,15 @@ module TurbotRunner
|
|
20
27
|
succeeded = run_script(scraper_config)
|
21
28
|
# Run the transformers even if the scraper fails
|
22
29
|
transformers.each do |transformer_config|
|
23
|
-
succeeded = run_script(
|
30
|
+
succeeded = run_script(
|
31
|
+
transformer_config.merge(:base_directory => @base_directory),
|
32
|
+
input_file=scraper_output_file) && succeeded
|
24
33
|
end
|
25
34
|
succeeded
|
26
35
|
end
|
27
36
|
|
28
37
|
def set_up_output_directory
|
29
38
|
FileUtils.mkdir_p(@output_directory)
|
30
|
-
|
31
39
|
FileUtils.rm_f(File.join(@output_directory, 'scraper.out'))
|
32
40
|
FileUtils.rm_f(File.join(@output_directory, 'scraper.err'))
|
33
41
|
|
@@ -41,7 +49,7 @@ module TurbotRunner
|
|
41
49
|
process_script_output(scraper_config)
|
42
50
|
|
43
51
|
transformers.each do |transformer_config|
|
44
|
-
process_script_output(transformer_config)
|
52
|
+
process_script_output(transformer_config.merge(:base_directory => @base_directory))
|
45
53
|
end
|
46
54
|
end
|
47
55
|
|
@@ -70,9 +78,9 @@ module TurbotRunner
|
|
70
78
|
end
|
71
79
|
end
|
72
80
|
|
81
|
+
|
73
82
|
def run_script(script_config, input_file=nil)
|
74
83
|
command = build_command(script_config[:file], input_file)
|
75
|
-
|
76
84
|
script_runner = ScriptRunner.new(
|
77
85
|
command,
|
78
86
|
output_file(script_config[:file]),
|
@@ -90,8 +98,8 @@ module TurbotRunner
|
|
90
98
|
# (e.g. interruptions etc) is required; we just want to do
|
91
99
|
# record handling.
|
92
100
|
processor = Processor.new(nil, script_config, @record_handler)
|
93
|
-
|
94
|
-
File.open(
|
101
|
+
file = output_file(script_config[:file])
|
102
|
+
File.open(file) do |f|
|
95
103
|
f.each_line do |line|
|
96
104
|
processor.process(line)
|
97
105
|
end
|
@@ -104,11 +112,9 @@ module TurbotRunner
|
|
104
112
|
|
105
113
|
def build_command(script, input_file=nil)
|
106
114
|
raise "Could not run #{script} with #{language}" unless script_extension == File.extname(script)
|
107
|
-
|
108
|
-
command = "#{full_interpreter_path} #{additional_args} #{path_to_script} >#{output_file(script)}"
|
115
|
+
command = "#{full_interpreter_path} #{additional_args} #{script} >#{output_file(script)}"
|
109
116
|
command << " 2>#{output_file(script, '.err')}" if @log_to_file
|
110
117
|
command << " <#{input_file}" unless input_file.nil?
|
111
|
-
|
112
118
|
command
|
113
119
|
end
|
114
120
|
|
@@ -133,6 +139,7 @@ module TurbotRunner
|
|
133
139
|
|
134
140
|
def scraper_config
|
135
141
|
{
|
142
|
+
:base_directory => @base_directory,
|
136
143
|
:file => scraper_script,
|
137
144
|
:data_type => scraper_data_type,
|
138
145
|
:identifying_fields => scraper_identifying_fields
|
@@ -162,5 +169,11 @@ module TurbotRunner
|
|
162
169
|
def scraper_identifying_fields
|
163
170
|
@config[:identifying_fields]
|
164
171
|
end
|
172
|
+
|
173
|
+
def assert_absolute_path(path)
|
174
|
+
unless Pathname.new(path).absolute?
|
175
|
+
raise "#{path} must be an absolute path"
|
176
|
+
end
|
177
|
+
end
|
165
178
|
end
|
166
179
|
end
|
@@ -9,65 +9,67 @@ module TurbotRunner
|
|
9
9
|
def initialize(command, output_file, script_config, options={})
|
10
10
|
@command = command
|
11
11
|
@output_file = output_file
|
12
|
-
|
12
|
+
@script_config = script_config
|
13
13
|
record_handler = options[:record_handler] || BaseHandler.new # A BaseHandler does nothing
|
14
14
|
@processor = Processor.new(self, script_config, record_handler)
|
15
|
-
|
16
15
|
@timeout = options[:timeout] || 3600
|
17
16
|
end
|
18
17
|
|
19
18
|
def run
|
20
|
-
@
|
21
|
-
@failed = false
|
22
|
-
|
23
|
-
# Start a thread that spawns a subprocess that runs the script and
|
24
|
-
# redirects the script's output to a file at a known location.
|
25
|
-
script_thread = Thread.new { run_command(@command) }
|
19
|
+
Dir.chdir(@script_config[:base_directory]) do
|
26
20
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
f = File.open(@output_file)
|
31
|
-
rescue Errno::ENOENT
|
32
|
-
sleep 0.1
|
33
|
-
retry
|
34
|
-
end
|
35
|
-
|
36
|
-
# Read from output file buildling up lines byte by byte byte by byte
|
37
|
-
# until either we reach the end of the file and the script has exited, or
|
38
|
-
# @interrupted becomes true. We cannot use IO#readline here because if
|
39
|
-
# only half a line has been synced to the file by the time we read it,
|
40
|
-
# then the incomplete line will be read, causing chaos down the line.
|
41
|
-
line = ''
|
21
|
+
begin
|
22
|
+
@interrupted = false
|
23
|
+
@failed = false
|
42
24
|
|
43
|
-
|
25
|
+
# Start a thread that spawns a subprocess that runs the script and
|
26
|
+
# redirects the script's output to a file at a known location.
|
27
|
+
script_thread = Thread.new { run_command(@command) }
|
44
28
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
29
|
+
# Wait for the output file to be created, so that we can start to read
|
30
|
+
# from it.
|
31
|
+
begin
|
32
|
+
f = File.open(@output_file, "r")
|
33
|
+
rescue Errno::ENOENT
|
49
34
|
sleep 0.1
|
50
|
-
|
51
|
-
else
|
52
|
-
break
|
35
|
+
retry
|
53
36
|
end
|
54
|
-
|
55
|
-
|
56
|
-
|
37
|
+
# Read from output file buildling up lines byte by byte byte by byte
|
38
|
+
# until either we reach the end of the file and the script has exited, or
|
39
|
+
# @interrupted becomes true. We cannot use IO#readline here because if
|
40
|
+
# only half a line has been synced to the file by the time we read it,
|
41
|
+
# then the incomplete line will be read, causing chaos down the line.
|
57
42
|
line = ''
|
58
|
-
|
43
|
+
|
59
44
|
time_of_last_read = Time.now
|
60
|
-
|
61
|
-
|
62
|
-
|
45
|
+
until @interrupted do
|
46
|
+
byte = f.read(1)
|
47
|
+
if byte.nil?
|
48
|
+
if script_thread.alive?
|
49
|
+
sleep 0.1
|
50
|
+
interrupt_and_mark_as_failed if (Time.now - time_of_last_read) > @timeout
|
51
|
+
else
|
52
|
+
break
|
53
|
+
end
|
54
|
+
elsif byte == "\n"
|
55
|
+
@processor.process(line)
|
56
|
+
time_of_last_read = Time.now
|
57
|
+
line = ''
|
58
|
+
else
|
59
|
+
time_of_last_read = Time.now
|
60
|
+
line << byte
|
61
|
+
end
|
62
|
+
end
|
63
63
|
|
64
|
-
|
65
|
-
|
66
|
-
|
64
|
+
# script_thread may still be alive if we exited the loop above becuase
|
65
|
+
# @interrupted became true, and so we must kill it.
|
66
|
+
kill_running_processes if script_thread.alive?
|
67
67
|
|
68
|
-
|
69
|
-
|
70
|
-
|
68
|
+
@failed ? false : script_thread.join.value
|
69
|
+
ensure
|
70
|
+
f.close if f
|
71
|
+
end
|
72
|
+
end
|
71
73
|
end
|
72
74
|
|
73
75
|
def interrupt
|
@@ -82,7 +84,6 @@ module TurbotRunner
|
|
82
84
|
private
|
83
85
|
def run_command(command)
|
84
86
|
system(command)
|
85
|
-
|
86
87
|
# A nil exitstatus indicates that the script was interrupted. A
|
87
88
|
# termsig of 2 indicates that the script was interrupted by a SIGINT.
|
88
89
|
$?.exitstatus == 0 || ($?.exitstatus.nil? && $?.termsig == 2)
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'json-schema'
|
3
|
+
|
4
|
+
date_format_validator = -> value {
|
5
|
+
begin
|
6
|
+
Date.strptime(value, '%Y-%m-%d')
|
7
|
+
rescue ArgumentError
|
8
|
+
raise JSON::Schema::CustomFormatError.new('must be of format yyyy-mm-dd')
|
9
|
+
end
|
10
|
+
}
|
11
|
+
|
12
|
+
JSON::Validator.register_format_validator('date', date_format_validator)
|
data/lib/turbot_runner.rb
CHANGED
@@ -0,0 +1,8 @@
|
|
1
|
+
{
|
2
|
+
"bot_id": "bot-that-expects-file",
|
3
|
+
"description": "This bot depends on being able to open a file in the current directory",
|
4
|
+
"language": "ruby",
|
5
|
+
"data_type": "primary data",
|
6
|
+
"identifying_fields": ["licence_number"],
|
7
|
+
"files": ["scraper.rb", "something.txt"]
|
8
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
ehlo
|
data/spec/lib/runner_spec.rb
CHANGED
@@ -14,7 +14,7 @@ describe TurbotRunner::Runner do
|
|
14
14
|
describe '#run' do
|
15
15
|
context 'with a bot written in ruby' do
|
16
16
|
before do
|
17
|
-
@runner =
|
17
|
+
@runner = test_runner('ruby-bot')
|
18
18
|
end
|
19
19
|
|
20
20
|
it 'produces expected output' do
|
@@ -29,7 +29,7 @@ describe TurbotRunner::Runner do
|
|
29
29
|
|
30
30
|
context 'with a bot written in python' do
|
31
31
|
before do
|
32
|
-
@runner =
|
32
|
+
@runner = test_runner('python-bot')
|
33
33
|
end
|
34
34
|
|
35
35
|
it 'produces expected output' do
|
@@ -40,7 +40,7 @@ describe TurbotRunner::Runner do
|
|
40
40
|
|
41
41
|
context 'with a bot with a transformer' do
|
42
42
|
before do
|
43
|
-
@runner =
|
43
|
+
@runner = test_runner('bot-with-transformer')
|
44
44
|
end
|
45
45
|
|
46
46
|
it 'produces expected outputs' do
|
@@ -56,7 +56,7 @@ describe TurbotRunner::Runner do
|
|
56
56
|
|
57
57
|
context 'with a bot with multiple transformers' do
|
58
58
|
before do
|
59
|
-
@runner =
|
59
|
+
@runner = test_runner('bot-with-transformers')
|
60
60
|
end
|
61
61
|
|
62
62
|
it 'produces expected outputs' do
|
@@ -75,10 +75,7 @@ describe TurbotRunner::Runner do
|
|
75
75
|
context 'when logging to file enabled' do
|
76
76
|
it 'logs to file' do
|
77
77
|
expected_log = "doing...\ndone\n"
|
78
|
-
runner =
|
79
|
-
'spec/bots/logging-bot',
|
80
|
-
:log_to_file => true
|
81
|
-
)
|
78
|
+
runner = test_runner('logging-bot', :log_to_file => true)
|
82
79
|
runner.run
|
83
80
|
expect([runner, 'scraper']).to have_error_output_matching(expected_log)
|
84
81
|
end
|
@@ -93,10 +90,7 @@ describe TurbotRunner::Runner do
|
|
93
90
|
|
94
91
|
context 'with a bot that outputs RUN ENDED' do
|
95
92
|
before do
|
96
|
-
@runner =
|
97
|
-
'spec/bots/bot-that-emits-run-ended',
|
98
|
-
:log_to_file => true
|
99
|
-
)
|
93
|
+
@runner = test_runner('bot-that-emits-run-ended', :log_to_file => true)
|
100
94
|
end
|
101
95
|
it 'calls handle_run_ended on the handler' do
|
102
96
|
expect_any_instance_of(TurbotRunner::BaseHandler).to receive(:handle_run_ended)
|
@@ -112,10 +106,7 @@ describe TurbotRunner::Runner do
|
|
112
106
|
|
113
107
|
context 'with a bot that crashes in scraper' do
|
114
108
|
before do
|
115
|
-
@runner =
|
116
|
-
'spec/bots/bot-that-crashes-in-scraper',
|
117
|
-
:log_to_file => true
|
118
|
-
)
|
109
|
+
@runner = test_runner('bot-that-crashes-in-scraper', :log_to_file => true)
|
119
110
|
end
|
120
111
|
|
121
112
|
it 'returns false' do
|
@@ -136,12 +127,19 @@ describe TurbotRunner::Runner do
|
|
136
127
|
end
|
137
128
|
end
|
138
129
|
|
130
|
+
context 'with a bot that expects a file to be present in the working directory' do
|
131
|
+
before do
|
132
|
+
@runner = test_runner('bot-that-expects-file')
|
133
|
+
end
|
134
|
+
|
135
|
+
it 'returns true' do
|
136
|
+
expect(@runner.run).to be(true)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
139
140
|
context 'with a bot that crashes in transformer' do
|
140
141
|
before do
|
141
|
-
@runner =
|
142
|
-
'spec/bots/bot-that-crashes-in-transformer',
|
143
|
-
:log_to_file => true
|
144
|
-
)
|
142
|
+
@runner = test_runner('bot-that-crashes-in-transformer', :log_to_file => true)
|
145
143
|
end
|
146
144
|
|
147
145
|
it 'returns false' do
|
@@ -174,8 +172,7 @@ describe TurbotRunner::Runner do
|
|
174
172
|
end
|
175
173
|
end
|
176
174
|
|
177
|
-
@runner =
|
178
|
-
'spec/bots/slow-bot',
|
175
|
+
@runner = test_runner('slow-bot',
|
179
176
|
:record_handler => Handler.new,
|
180
177
|
:log_to_file => true
|
181
178
|
)
|
@@ -193,28 +190,42 @@ describe TurbotRunner::Runner do
|
|
193
190
|
|
194
191
|
context 'with a scraper that produces an invalid record' do
|
195
192
|
it 'returns false' do
|
196
|
-
@runner =
|
193
|
+
@runner = test_runner('invalid-record-bot')
|
197
194
|
expect(@runner.run).to be(false)
|
198
195
|
end
|
199
196
|
end
|
200
197
|
|
201
198
|
context 'with a scraper that produces invalid JSON' do
|
202
199
|
it 'returns false' do
|
203
|
-
@runner =
|
200
|
+
@runner = test_runner('invalid-json-bot')
|
204
201
|
expect(@runner.run).to be(false)
|
205
202
|
end
|
206
203
|
end
|
207
204
|
|
208
205
|
context 'with a scraper that hangs' do
|
206
|
+
# XXX This spec fails because the loop in ScriptRunner#run that
|
207
|
+
# reads lines from the output file doesn't start until the
|
208
|
+
# output file is created; however, the way we're redirecting
|
209
|
+
# stdout using the shell means the file doesn't get created
|
210
|
+
# until
|
209
211
|
it 'returns false' do
|
210
|
-
@runner =
|
211
|
-
'spec/bots/bot-with-pause',
|
212
|
+
@runner = test_runner('bot-with-pause',
|
212
213
|
:timeout => 1,
|
213
214
|
:log_to_file => true
|
214
215
|
)
|
215
216
|
expect(@runner.run).to be(false)
|
216
217
|
end
|
217
218
|
end
|
219
|
+
|
220
|
+
context 'with a bot that emits an invalid sample date' do
|
221
|
+
before do
|
222
|
+
@runner = TurbotRunner::Runner.new('spec/bots/bot-with-invalid-sample-date')
|
223
|
+
end
|
224
|
+
|
225
|
+
it 'returns false' do
|
226
|
+
expect(@runner.run).to be(true)
|
227
|
+
end
|
228
|
+
end
|
218
229
|
end
|
219
230
|
|
220
231
|
describe '#process_output' do
|
@@ -236,10 +247,9 @@ describe TurbotRunner::Runner do
|
|
236
247
|
end
|
237
248
|
|
238
249
|
it 'calls handler once for each line of output' do
|
239
|
-
|
250
|
+
test_runner('bot-with-transformer').run
|
240
251
|
|
241
|
-
runner =
|
242
|
-
'spec/bots/bot-with-transformer',
|
252
|
+
runner = test_runner('bot-with-transformer',
|
243
253
|
:record_handler => @handler
|
244
254
|
)
|
245
255
|
|
@@ -249,10 +259,9 @@ describe TurbotRunner::Runner do
|
|
249
259
|
end
|
250
260
|
|
251
261
|
it 'can cope when scraper has failed immediately' do
|
252
|
-
|
262
|
+
test_runner('bot-that-crashes-immediately').run
|
253
263
|
|
254
|
-
runner =
|
255
|
-
'spec/bots/bot-with-transformer',
|
264
|
+
runner = test_runner('bot-with-transformer',
|
256
265
|
:record_handler => @handler
|
257
266
|
)
|
258
267
|
|
@@ -262,18 +271,18 @@ describe TurbotRunner::Runner do
|
|
262
271
|
|
263
272
|
describe '#set_up_output_directory' do
|
264
273
|
before do
|
265
|
-
@runner =
|
274
|
+
@runner = test_runner('bot-with-transformer')
|
266
275
|
end
|
267
276
|
|
268
277
|
it 'clears existing output' do
|
269
|
-
path = File.join(@runner.
|
278
|
+
path = File.join(@runner.base_directory, 'output', 'scraper.out')
|
270
279
|
FileUtils.touch(path)
|
271
280
|
@runner.set_up_output_directory
|
272
281
|
expect(File.exist?(path)).to be(false)
|
273
282
|
end
|
274
283
|
|
275
284
|
it 'does not clear existing files that are not output files' do
|
276
|
-
path = File.join(@runner.
|
285
|
+
path = File.join(@runner.base_directory, 'output', 'stdout')
|
277
286
|
FileUtils.touch(path)
|
278
287
|
@runner.set_up_output_directory
|
279
288
|
expect(File.exist?(path)).to be(true)
|
@@ -288,7 +297,7 @@ RSpec::Matchers.define :have_output do |expected|
|
|
288
297
|
|
289
298
|
expected_path = File.join('spec', 'outputs', expected)
|
290
299
|
expected_output = File.readlines(expected_path).map {|line| JSON.parse(line)}
|
291
|
-
actual_path = File.join(runner.
|
300
|
+
actual_path = File.join(runner.base_directory, 'output', "#{script}.out")
|
292
301
|
actual_output = File.readlines(actual_path).map {|line| JSON.parse(line)}
|
293
302
|
expect(expected_output).to eq(actual_output)
|
294
303
|
end
|
@@ -299,8 +308,13 @@ RSpec::Matchers.define :have_error_output_matching do |expected|
|
|
299
308
|
match do |actual|
|
300
309
|
runner, script = actual
|
301
310
|
|
302
|
-
actual_path = File.join(runner.
|
311
|
+
actual_path = File.join(runner.base_directory, 'output', "#{script}.err")
|
303
312
|
actual_output = File.read(actual_path)
|
304
313
|
expect(actual_output).to match(expected)
|
305
314
|
end
|
306
315
|
end
|
316
|
+
|
317
|
+
def test_runner(name, opts={})
|
318
|
+
test_bot_location = File.join(File.dirname(__FILE__), '../bots', name)
|
319
|
+
TurbotRunner::Runner.new(test_bot_location, opts)
|
320
|
+
end
|
data/spec/manual_spec.rb
CHANGED
@@ -18,7 +18,8 @@ eos
|
|
18
18
|
|
19
19
|
gets
|
20
20
|
|
21
|
-
|
21
|
+
bot_location = File.join(File.dirname(__FILE__), 'bots/logging-bot')
|
22
|
+
runner = TurbotRunner::Runner.new(bot_location).run
|
22
23
|
|
23
24
|
puts
|
24
25
|
puts 'Did you see the expected lines? [y]/n'
|
@@ -38,7 +39,8 @@ eos
|
|
38
39
|
|
39
40
|
gets
|
40
41
|
|
41
|
-
|
42
|
+
bot_location = File.join(File.dirname(__FILE__), 'bots/bot-with-pause')
|
43
|
+
runner = TurbotRunner::Runner.new(bot_location).run
|
42
44
|
|
43
45
|
expected_output = File.readlines('spec/outputs/truncated-scraper.out').map {|line| JSON.parse(line)}
|
44
46
|
actual_output = File.readlines('spec/bots/bot-with-pause/output/scraper.out').map {|line| JSON.parse(line)}
|
metadata
CHANGED
@@ -1,35 +1,46 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: turbot-runner
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 61
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 19
|
10
|
+
version: 0.1.19
|
5
11
|
platform: ruby
|
6
|
-
authors:
|
12
|
+
authors:
|
7
13
|
- OpenCorporates
|
8
14
|
autorequire:
|
9
15
|
bindir: bin
|
10
16
|
cert_chain: []
|
11
|
-
|
12
|
-
|
13
|
-
|
17
|
+
|
18
|
+
date: 2015-01-07 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
14
21
|
name: json-schema
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - '='
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 2.2.2
|
20
|
-
type: :runtime
|
21
22
|
prerelease: false
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ~>
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 11
|
29
|
+
segments:
|
30
|
+
- 2
|
31
|
+
- 4
|
32
|
+
version: "2.4"
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
27
35
|
description:
|
28
36
|
email: bots@opencorporates.com
|
29
37
|
executables: []
|
38
|
+
|
30
39
|
extensions: []
|
40
|
+
|
31
41
|
extra_rdoc_files: []
|
32
|
-
|
42
|
+
|
43
|
+
files:
|
33
44
|
- bin/rspec
|
34
45
|
- lib/turbot_runner.rb
|
35
46
|
- lib/turbot_runner/base_handler.rb
|
@@ -38,36 +49,8 @@ files:
|
|
38
49
|
- lib/turbot_runner/processor.rb
|
39
50
|
- lib/turbot_runner/runner.rb
|
40
51
|
- lib/turbot_runner/script_runner.rb
|
52
|
+
- lib/turbot_runner/validators.rb
|
41
53
|
- lib/turbot_runner/version.rb
|
42
|
-
- schema/schemas/company-schema.json
|
43
|
-
- schema/schemas/financial-payment-schema.json
|
44
|
-
- schema/schemas/includes/address.json
|
45
|
-
- schema/schemas/includes/alternative_name.json
|
46
|
-
- schema/schemas/includes/company.json
|
47
|
-
- schema/schemas/includes/filing.json
|
48
|
-
- schema/schemas/includes/financial-payment-data-object.json
|
49
|
-
- schema/schemas/includes/identifier.json
|
50
|
-
- schema/schemas/includes/industry_code.json
|
51
|
-
- schema/schemas/includes/licence-data-object.json
|
52
|
-
- schema/schemas/includes/officer.json
|
53
|
-
- schema/schemas/includes/organisation.json
|
54
|
-
- schema/schemas/includes/permission.json
|
55
|
-
- schema/schemas/includes/person.json
|
56
|
-
- schema/schemas/includes/person_name.json
|
57
|
-
- schema/schemas/includes/previous_name.json
|
58
|
-
- schema/schemas/includes/share-parcel-data.json
|
59
|
-
- schema/schemas/includes/share-parcel.json
|
60
|
-
- schema/schemas/includes/subsidiary-relationship-data.json
|
61
|
-
- schema/schemas/includes/total-shares.json
|
62
|
-
- schema/schemas/includes/unknown_entity_type.json
|
63
|
-
- schema/schemas/licence-schema.json
|
64
|
-
- schema/schemas/primary-data-schema.json
|
65
|
-
- schema/schemas/rich-licence-schema.json
|
66
|
-
- schema/schemas/share-parcel-schema.json
|
67
|
-
- schema/schemas/simple-financial-payment-schema.json
|
68
|
-
- schema/schemas/simple-licence-schema.json
|
69
|
-
- schema/schemas/simple-subsidiary-schema.json
|
70
|
-
- schema/schemas/subsidiary-relationship-schema.json
|
71
54
|
- spec/bots/bot-that-crashes-immediately/manifest.json
|
72
55
|
- spec/bots/bot-that-crashes-immediately/scraper.rb
|
73
56
|
- spec/bots/bot-that-crashes-immediately/transformer1.rb
|
@@ -80,6 +63,11 @@ files:
|
|
80
63
|
- spec/bots/bot-that-crashes-in-transformer/transformer2.rb
|
81
64
|
- spec/bots/bot-that-emits-run-ended/manifest.json
|
82
65
|
- spec/bots/bot-that-emits-run-ended/scraper.rb
|
66
|
+
- spec/bots/bot-that-expects-file/manifest.json
|
67
|
+
- spec/bots/bot-that-expects-file/scraper.rb
|
68
|
+
- spec/bots/bot-that-expects-file/something.txt
|
69
|
+
- spec/bots/bot-with-invalid-sample-date/manifest.json
|
70
|
+
- spec/bots/bot-with-invalid-sample-date/scraper.rb
|
83
71
|
- spec/bots/bot-with-pause/manifest.json
|
84
72
|
- spec/bots/bot-with-pause/scraper.rb
|
85
73
|
- spec/bots/bot-with-transformer/manifest.json
|
@@ -107,29 +95,69 @@ files:
|
|
107
95
|
- spec/outputs/full-scraper.out
|
108
96
|
- spec/outputs/full-transformer.out
|
109
97
|
- spec/outputs/truncated-scraper.out
|
98
|
+
- schema/schemas/company-schema.json
|
99
|
+
- schema/schemas/financial-payment-schema.json
|
100
|
+
- schema/schemas/includes/address.json
|
101
|
+
- schema/schemas/includes/alternative_name.json
|
102
|
+
- schema/schemas/includes/company.json
|
103
|
+
- schema/schemas/includes/filing.json
|
104
|
+
- schema/schemas/includes/financial-payment-data-object.json
|
105
|
+
- schema/schemas/includes/identifier.json
|
106
|
+
- schema/schemas/includes/industry_code.json
|
107
|
+
- schema/schemas/includes/licence-data-object.json
|
108
|
+
- schema/schemas/includes/officer.json
|
109
|
+
- schema/schemas/includes/organisation.json
|
110
|
+
- schema/schemas/includes/permission.json
|
111
|
+
- schema/schemas/includes/person.json
|
112
|
+
- schema/schemas/includes/person_name.json
|
113
|
+
- schema/schemas/includes/previous_name.json
|
114
|
+
- schema/schemas/includes/share-parcel-data.json
|
115
|
+
- schema/schemas/includes/share-parcel.json
|
116
|
+
- schema/schemas/includes/subsidiary-relationship-data.json
|
117
|
+
- schema/schemas/includes/total-shares.json
|
118
|
+
- schema/schemas/includes/unknown_entity_type.json
|
119
|
+
- schema/schemas/licence-schema.json
|
120
|
+
- schema/schemas/primary-data-schema.json
|
121
|
+
- schema/schemas/rich-licence-schema.json
|
122
|
+
- schema/schemas/share-parcel-schema.json
|
123
|
+
- schema/schemas/simple-financial-payment-schema.json
|
124
|
+
- schema/schemas/simple-licence-schema.json
|
125
|
+
- schema/schemas/simple-subsidiary-schema.json
|
126
|
+
- schema/schemas/subsidiary-relationship-schema.json
|
110
127
|
homepage: http://turbot.opencorporates.com/
|
111
|
-
licenses:
|
128
|
+
licenses:
|
112
129
|
- MIT
|
113
|
-
metadata: {}
|
114
130
|
post_install_message:
|
115
131
|
rdoc_options: []
|
116
|
-
|
132
|
+
|
133
|
+
require_paths:
|
117
134
|
- lib
|
118
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
-
|
120
|
-
|
121
|
-
|
135
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
136
|
+
none: false
|
137
|
+
requirements:
|
138
|
+
- - ">="
|
139
|
+
- !ruby/object:Gem::Version
|
140
|
+
hash: 55
|
141
|
+
segments:
|
142
|
+
- 1
|
143
|
+
- 9
|
144
|
+
- 2
|
122
145
|
version: 1.9.2
|
123
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
146
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
147
|
+
none: false
|
148
|
+
requirements:
|
149
|
+
- - ">="
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
hash: 3
|
152
|
+
segments:
|
153
|
+
- 0
|
154
|
+
version: "0"
|
128
155
|
requirements: []
|
156
|
+
|
129
157
|
rubyforge_project:
|
130
|
-
rubygems_version:
|
158
|
+
rubygems_version: 1.8.15
|
131
159
|
signing_key:
|
132
|
-
specification_version:
|
160
|
+
specification_version: 3
|
133
161
|
summary: Utilities for running bots with Turbot
|
134
162
|
test_files: []
|
135
|
-
|
163
|
+
|
checksums.yaml
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
---
|
2
|
-
!binary "U0hBMQ==":
|
3
|
-
metadata.gz: !binary |-
|
4
|
-
NDlhMjNhODQ5ZDI1NmVlMWUxMTJkNDMwOTQ4MDY0MzljYjZhZTRkMg==
|
5
|
-
data.tar.gz: !binary |-
|
6
|
-
M2FjMGQ0ODAxYTUwZjc5MDBmY2ViN2RlMDlmZmY0MzZlODA2MTU2MQ==
|
7
|
-
SHA512:
|
8
|
-
metadata.gz: !binary |-
|
9
|
-
YmJkMjNjOThlNjJmNDM0MmRjYjIxNDQ1YjI5MDMxNGQxNDhmOTRiOGNiMTU0
|
10
|
-
MGVlZTRhOGU4OTE2YjVjN2I3ZTU4ODhhZDdhZWJhMDg5NGMxOTcyOGQ4ZmFk
|
11
|
-
MzA2YmViNmM1YmQwOTgwODBlYWM0OGEzNDc3YzYxYjM0YjIzOTQ=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
MjZkNTQ4MGY4OGNlYTAwOWMzMTZlNDM2MmJkZDFhZTgxZDExOWVhN2QxOWEy
|
14
|
-
MjQzMGJiOWY2MTJiNzAxZWQ3ZGRkODA1ZjdmNzk0YjY2OTc3MDFlNzk3ZDIx
|
15
|
-
YTExNDViYTU5MjZiNzY3OThkMDBkNzg3MmRmNWY5NDAwMzM3MWQ=
|