turbot-runner 0.1.18 → 0.1.19

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,17 +1,24 @@
1
1
  require 'json'
2
2
  require 'fileutils'
3
+ require 'pathname'
3
4
 
4
5
  module TurbotRunner
5
6
  class Runner
6
- attr_reader :directory
7
+ attr_reader :base_directory
7
8
 
8
9
  def initialize(directory, options={})
9
- @directory = directory
10
+ assert_absolute_path(directory)
11
+ @base_directory = directory
10
12
  @config = load_config(directory)
11
13
  @record_handler = options[:record_handler]
12
14
  @log_to_file = options[:log_to_file]
13
15
  @timeout = options[:timeout]
14
- @output_directory = options[:output_directory] || File.join(@directory, 'output')
16
+ if options[:output_directory]
17
+ assert_absolute_path(options[:output_directory])
18
+ @output_directory = options[:output_directory]
19
+ else
20
+ @output_directory = File.join(@base_directory, 'output')
21
+ end
15
22
  end
16
23
 
17
24
  def run
@@ -20,14 +27,15 @@ module TurbotRunner
20
27
  succeeded = run_script(scraper_config)
21
28
  # Run the transformers even if the scraper fails
22
29
  transformers.each do |transformer_config|
23
- succeeded = run_script(transformer_config, input_file=scraper_output_file) && succeeded
30
+ succeeded = run_script(
31
+ transformer_config.merge(:base_directory => @base_directory),
32
+ input_file=scraper_output_file) && succeeded
24
33
  end
25
34
  succeeded
26
35
  end
27
36
 
28
37
  def set_up_output_directory
29
38
  FileUtils.mkdir_p(@output_directory)
30
-
31
39
  FileUtils.rm_f(File.join(@output_directory, 'scraper.out'))
32
40
  FileUtils.rm_f(File.join(@output_directory, 'scraper.err'))
33
41
 
@@ -41,7 +49,7 @@ module TurbotRunner
41
49
  process_script_output(scraper_config)
42
50
 
43
51
  transformers.each do |transformer_config|
44
- process_script_output(transformer_config)
52
+ process_script_output(transformer_config.merge(:base_directory => @base_directory))
45
53
  end
46
54
  end
47
55
 
@@ -70,9 +78,9 @@ module TurbotRunner
70
78
  end
71
79
  end
72
80
 
81
+
73
82
  def run_script(script_config, input_file=nil)
74
83
  command = build_command(script_config[:file], input_file)
75
-
76
84
  script_runner = ScriptRunner.new(
77
85
  command,
78
86
  output_file(script_config[:file]),
@@ -90,8 +98,8 @@ module TurbotRunner
90
98
  # (e.g. interruptions etc) is required; we just want to do
91
99
  # record handling.
92
100
  processor = Processor.new(nil, script_config, @record_handler)
93
-
94
- File.open(output_file(script_config[:file])) do |f|
101
+ file = output_file(script_config[:file])
102
+ File.open(file) do |f|
95
103
  f.each_line do |line|
96
104
  processor.process(line)
97
105
  end
@@ -104,11 +112,9 @@ module TurbotRunner
104
112
 
105
113
  def build_command(script, input_file=nil)
106
114
  raise "Could not run #{script} with #{language}" unless script_extension == File.extname(script)
107
- path_to_script = File.join(@directory, script)
108
- command = "#{full_interpreter_path} #{additional_args} #{path_to_script} >#{output_file(script)}"
115
+ command = "#{full_interpreter_path} #{additional_args} #{script} >#{output_file(script)}"
109
116
  command << " 2>#{output_file(script, '.err')}" if @log_to_file
110
117
  command << " <#{input_file}" unless input_file.nil?
111
-
112
118
  command
113
119
  end
114
120
 
@@ -133,6 +139,7 @@ module TurbotRunner
133
139
 
134
140
  def scraper_config
135
141
  {
142
+ :base_directory => @base_directory,
136
143
  :file => scraper_script,
137
144
  :data_type => scraper_data_type,
138
145
  :identifying_fields => scraper_identifying_fields
@@ -162,5 +169,11 @@ module TurbotRunner
162
169
  def scraper_identifying_fields
163
170
  @config[:identifying_fields]
164
171
  end
172
+
173
+ def assert_absolute_path(path)
174
+ unless Pathname.new(path).absolute?
175
+ raise "#{path} must be an absolute path"
176
+ end
177
+ end
165
178
  end
166
179
  end
@@ -9,65 +9,67 @@ module TurbotRunner
9
9
  def initialize(command, output_file, script_config, options={})
10
10
  @command = command
11
11
  @output_file = output_file
12
-
12
+ @script_config = script_config
13
13
  record_handler = options[:record_handler] || BaseHandler.new # A BaseHandler does nothing
14
14
  @processor = Processor.new(self, script_config, record_handler)
15
-
16
15
  @timeout = options[:timeout] || 3600
17
16
  end
18
17
 
19
18
  def run
20
- @interrupted = false
21
- @failed = false
22
-
23
- # Start a thread that spawns a subprocess that runs the script and
24
- # redirects the script's output to a file at a known location.
25
- script_thread = Thread.new { run_command(@command) }
19
+ Dir.chdir(@script_config[:base_directory]) do
26
20
 
27
- # Wait for the output file to be created, so that we can start to read
28
- # from it.
29
- begin
30
- f = File.open(@output_file)
31
- rescue Errno::ENOENT
32
- sleep 0.1
33
- retry
34
- end
35
-
36
- # Read from output file buildling up lines byte by byte byte by byte
37
- # until either we reach the end of the file and the script has exited, or
38
- # @interrupted becomes true. We cannot use IO#readline here because if
39
- # only half a line has been synced to the file by the time we read it,
40
- # then the incomplete line will be read, causing chaos down the line.
41
- line = ''
21
+ begin
22
+ @interrupted = false
23
+ @failed = false
42
24
 
43
- time_of_last_read = Time.now
25
+ # Start a thread that spawns a subprocess that runs the script and
26
+ # redirects the script's output to a file at a known location.
27
+ script_thread = Thread.new { run_command(@command) }
44
28
 
45
- until @interrupted do
46
- byte = f.read(1)
47
- if byte.nil?
48
- if script_thread.alive?
29
+ # Wait for the output file to be created, so that we can start to read
30
+ # from it.
31
+ begin
32
+ f = File.open(@output_file, "r")
33
+ rescue Errno::ENOENT
49
34
  sleep 0.1
50
- interrupt_and_mark_as_failed if (Time.now - time_of_last_read) > @timeout
51
- else
52
- break
35
+ retry
53
36
  end
54
- elsif byte == "\n"
55
- @processor.process(line)
56
- time_of_last_read = Time.now
37
+ # Read from output file buildling up lines byte by byte byte by byte
38
+ # until either we reach the end of the file and the script has exited, or
39
+ # @interrupted becomes true. We cannot use IO#readline here because if
40
+ # only half a line has been synced to the file by the time we read it,
41
+ # then the incomplete line will be read, causing chaos down the line.
57
42
  line = ''
58
- else
43
+
59
44
  time_of_last_read = Time.now
60
- line << byte
61
- end
62
- end
45
+ until @interrupted do
46
+ byte = f.read(1)
47
+ if byte.nil?
48
+ if script_thread.alive?
49
+ sleep 0.1
50
+ interrupt_and_mark_as_failed if (Time.now - time_of_last_read) > @timeout
51
+ else
52
+ break
53
+ end
54
+ elsif byte == "\n"
55
+ @processor.process(line)
56
+ time_of_last_read = Time.now
57
+ line = ''
58
+ else
59
+ time_of_last_read = Time.now
60
+ line << byte
61
+ end
62
+ end
63
63
 
64
- # script_thread may still be alive if we exited the loop above becuase
65
- # @interrupted became true, and so we must kill it.
66
- kill_running_processes if script_thread.alive?
64
+ # script_thread may still be alive if we exited the loop above becuase
65
+ # @interrupted became true, and so we must kill it.
66
+ kill_running_processes if script_thread.alive?
67
67
 
68
- @failed ? false : script_thread.join.value
69
- ensure
70
- f.close if f
68
+ @failed ? false : script_thread.join.value
69
+ ensure
70
+ f.close if f
71
+ end
72
+ end
71
73
  end
72
74
 
73
75
  def interrupt
@@ -82,7 +84,6 @@ module TurbotRunner
82
84
  private
83
85
  def run_command(command)
84
86
  system(command)
85
-
86
87
  # A nil exitstatus indicates that the script was interrupted. A
87
88
  # termsig of 2 indicates that the script was interrupted by a SIGINT.
88
89
  $?.exitstatus == 0 || ($?.exitstatus.nil? && $?.termsig == 2)
@@ -0,0 +1,12 @@
1
+ require 'date'
2
+ require 'json-schema'
3
+
4
+ date_format_validator = -> value {
5
+ begin
6
+ Date.strptime(value, '%Y-%m-%d')
7
+ rescue ArgumentError
8
+ raise JSON::Schema::CustomFormatError.new('must be of format yyyy-mm-dd')
9
+ end
10
+ }
11
+
12
+ JSON::Validator.register_format_validator('date', date_format_validator)
@@ -1,3 +1,3 @@
1
1
  module TurbotRunner
2
- VERSION = '0.1.18'
2
+ VERSION = '0.1.19'
3
3
  end
data/lib/turbot_runner.rb CHANGED
@@ -3,4 +3,5 @@ require 'turbot_runner/exceptions'
3
3
  require 'turbot_runner/processor'
4
4
  require 'turbot_runner/runner'
5
5
  require 'turbot_runner/script_runner'
6
+ require 'turbot_runner/validators'
6
7
  require 'turbot_runner/version'
@@ -0,0 +1,8 @@
1
+ {
2
+ "bot_id": "bot-that-expects-file",
3
+ "description": "This bot depends on being able to open a file in the current directory",
4
+ "language": "ruby",
5
+ "data_type": "primary data",
6
+ "identifying_fields": ["licence_number"],
7
+ "files": ["scraper.rb", "something.txt"]
8
+ }
@@ -0,0 +1,11 @@
1
+ require 'json'
2
+
3
+ asd = open("something.txt", "r")
4
+ 0.upto(2) do |n|
5
+ record = {
6
+ :licence_number => "XYZ#{n}",
7
+ :source_url => 'http://example.com',
8
+ :sample_date => '2014-06-01'
9
+ }
10
+ puts(record.to_json)
11
+ end
@@ -0,0 +1,8 @@
1
+ {
2
+ "bot_id": "bot-with-invalid-sample-date",
3
+ "description": "This bot produces data with an invalid sample date",
4
+ "language": "ruby",
5
+ "data_type": "primary data",
6
+ "identifying_fields": ["licence_number"],
7
+ "files": ["scraper.rb"]
8
+ }
@@ -0,0 +1,10 @@
1
+ require 'json'
2
+
3
+ 0.upto(9) do |n|
4
+ record = {
5
+ :licence_number => "XYZ#{n}",
6
+ :source_url => 'http://example.com',
7
+ :sample_date => '01/06/2014'
8
+ }
9
+ puts(record.to_json)
10
+ end
@@ -14,7 +14,7 @@ describe TurbotRunner::Runner do
14
14
  describe '#run' do
15
15
  context 'with a bot written in ruby' do
16
16
  before do
17
- @runner = TurbotRunner::Runner.new('spec/bots/ruby-bot')
17
+ @runner = test_runner('ruby-bot')
18
18
  end
19
19
 
20
20
  it 'produces expected output' do
@@ -29,7 +29,7 @@ describe TurbotRunner::Runner do
29
29
 
30
30
  context 'with a bot written in python' do
31
31
  before do
32
- @runner = TurbotRunner::Runner.new('spec/bots/python-bot')
32
+ @runner = test_runner('python-bot')
33
33
  end
34
34
 
35
35
  it 'produces expected output' do
@@ -40,7 +40,7 @@ describe TurbotRunner::Runner do
40
40
 
41
41
  context 'with a bot with a transformer' do
42
42
  before do
43
- @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformer')
43
+ @runner = test_runner('bot-with-transformer')
44
44
  end
45
45
 
46
46
  it 'produces expected outputs' do
@@ -56,7 +56,7 @@ describe TurbotRunner::Runner do
56
56
 
57
57
  context 'with a bot with multiple transformers' do
58
58
  before do
59
- @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformers')
59
+ @runner = test_runner('bot-with-transformers')
60
60
  end
61
61
 
62
62
  it 'produces expected outputs' do
@@ -75,10 +75,7 @@ describe TurbotRunner::Runner do
75
75
  context 'when logging to file enabled' do
76
76
  it 'logs to file' do
77
77
  expected_log = "doing...\ndone\n"
78
- runner = TurbotRunner::Runner.new(
79
- 'spec/bots/logging-bot',
80
- :log_to_file => true
81
- )
78
+ runner = test_runner('logging-bot', :log_to_file => true)
82
79
  runner.run
83
80
  expect([runner, 'scraper']).to have_error_output_matching(expected_log)
84
81
  end
@@ -93,10 +90,7 @@ describe TurbotRunner::Runner do
93
90
 
94
91
  context 'with a bot that outputs RUN ENDED' do
95
92
  before do
96
- @runner = TurbotRunner::Runner.new(
97
- 'spec/bots/bot-that-emits-run-ended',
98
- :log_to_file => true
99
- )
93
+ @runner = test_runner('bot-that-emits-run-ended', :log_to_file => true)
100
94
  end
101
95
  it 'calls handle_run_ended on the handler' do
102
96
  expect_any_instance_of(TurbotRunner::BaseHandler).to receive(:handle_run_ended)
@@ -112,10 +106,7 @@ describe TurbotRunner::Runner do
112
106
 
113
107
  context 'with a bot that crashes in scraper' do
114
108
  before do
115
- @runner = TurbotRunner::Runner.new(
116
- 'spec/bots/bot-that-crashes-in-scraper',
117
- :log_to_file => true
118
- )
109
+ @runner = test_runner('bot-that-crashes-in-scraper', :log_to_file => true)
119
110
  end
120
111
 
121
112
  it 'returns false' do
@@ -136,12 +127,19 @@ describe TurbotRunner::Runner do
136
127
  end
137
128
  end
138
129
 
130
+ context 'with a bot that expects a file to be present in the working directory' do
131
+ before do
132
+ @runner = test_runner('bot-that-expects-file')
133
+ end
134
+
135
+ it 'returns true' do
136
+ expect(@runner.run).to be(true)
137
+ end
138
+ end
139
+
139
140
  context 'with a bot that crashes in transformer' do
140
141
  before do
141
- @runner = TurbotRunner::Runner.new(
142
- 'spec/bots/bot-that-crashes-in-transformer',
143
- :log_to_file => true
144
- )
142
+ @runner = test_runner('bot-that-crashes-in-transformer', :log_to_file => true)
145
143
  end
146
144
 
147
145
  it 'returns false' do
@@ -174,8 +172,7 @@ describe TurbotRunner::Runner do
174
172
  end
175
173
  end
176
174
 
177
- @runner = TurbotRunner::Runner.new(
178
- 'spec/bots/slow-bot',
175
+ @runner = test_runner('slow-bot',
179
176
  :record_handler => Handler.new,
180
177
  :log_to_file => true
181
178
  )
@@ -193,28 +190,42 @@ describe TurbotRunner::Runner do
193
190
 
194
191
  context 'with a scraper that produces an invalid record' do
195
192
  it 'returns false' do
196
- @runner = TurbotRunner::Runner.new('spec/bots/invalid-record-bot')
193
+ @runner = test_runner('invalid-record-bot')
197
194
  expect(@runner.run).to be(false)
198
195
  end
199
196
  end
200
197
 
201
198
  context 'with a scraper that produces invalid JSON' do
202
199
  it 'returns false' do
203
- @runner = TurbotRunner::Runner.new('spec/bots/invalid-json-bot')
200
+ @runner = test_runner('invalid-json-bot')
204
201
  expect(@runner.run).to be(false)
205
202
  end
206
203
  end
207
204
 
208
205
  context 'with a scraper that hangs' do
206
+ # XXX This spec fails because the loop in ScriptRunner#run that
207
+ # reads lines from the output file doesn't start until the
208
+ # output file is created; however, the way we're redirecting
209
+ # stdout using the shell means the file doesn't get created
210
+ # until
209
211
  it 'returns false' do
210
- @runner = TurbotRunner::Runner.new(
211
- 'spec/bots/bot-with-pause',
212
+ @runner = test_runner('bot-with-pause',
212
213
  :timeout => 1,
213
214
  :log_to_file => true
214
215
  )
215
216
  expect(@runner.run).to be(false)
216
217
  end
217
218
  end
219
+
220
+ context 'with a bot that emits an invalid sample date' do
221
+ before do
222
+ @runner = TurbotRunner::Runner.new('spec/bots/bot-with-invalid-sample-date')
223
+ end
224
+
225
+ it 'returns false' do
226
+ expect(@runner.run).to be(true)
227
+ end
228
+ end
218
229
  end
219
230
 
220
231
  describe '#process_output' do
@@ -236,10 +247,9 @@ describe TurbotRunner::Runner do
236
247
  end
237
248
 
238
249
  it 'calls handler once for each line of output' do
239
- TurbotRunner::Runner.new('spec/bots/bot-with-transformer').run
250
+ test_runner('bot-with-transformer').run
240
251
 
241
- runner = TurbotRunner::Runner.new(
242
- 'spec/bots/bot-with-transformer',
252
+ runner = test_runner('bot-with-transformer',
243
253
  :record_handler => @handler
244
254
  )
245
255
 
@@ -249,10 +259,9 @@ describe TurbotRunner::Runner do
249
259
  end
250
260
 
251
261
  it 'can cope when scraper has failed immediately' do
252
- TurbotRunner::Runner.new('spec/bots/bot-that-crashes-immediately').run
262
+ test_runner('bot-that-crashes-immediately').run
253
263
 
254
- runner = TurbotRunner::Runner.new(
255
- 'spec/bots/bot-with-transformer',
264
+ runner = test_runner('bot-with-transformer',
256
265
  :record_handler => @handler
257
266
  )
258
267
 
@@ -262,18 +271,18 @@ describe TurbotRunner::Runner do
262
271
 
263
272
  describe '#set_up_output_directory' do
264
273
  before do
265
- @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformer')
274
+ @runner = test_runner('bot-with-transformer')
266
275
  end
267
276
 
268
277
  it 'clears existing output' do
269
- path = File.join(@runner.directory, 'output', 'scraper.out')
278
+ path = File.join(@runner.base_directory, 'output', 'scraper.out')
270
279
  FileUtils.touch(path)
271
280
  @runner.set_up_output_directory
272
281
  expect(File.exist?(path)).to be(false)
273
282
  end
274
283
 
275
284
  it 'does not clear existing files that are not output files' do
276
- path = File.join(@runner.directory, 'output', 'stdout')
285
+ path = File.join(@runner.base_directory, 'output', 'stdout')
277
286
  FileUtils.touch(path)
278
287
  @runner.set_up_output_directory
279
288
  expect(File.exist?(path)).to be(true)
@@ -288,7 +297,7 @@ RSpec::Matchers.define :have_output do |expected|
288
297
 
289
298
  expected_path = File.join('spec', 'outputs', expected)
290
299
  expected_output = File.readlines(expected_path).map {|line| JSON.parse(line)}
291
- actual_path = File.join(runner.directory, 'output', "#{script}.out")
300
+ actual_path = File.join(runner.base_directory, 'output', "#{script}.out")
292
301
  actual_output = File.readlines(actual_path).map {|line| JSON.parse(line)}
293
302
  expect(expected_output).to eq(actual_output)
294
303
  end
@@ -299,8 +308,13 @@ RSpec::Matchers.define :have_error_output_matching do |expected|
299
308
  match do |actual|
300
309
  runner, script = actual
301
310
 
302
- actual_path = File.join(runner.directory, 'output', "#{script}.err")
311
+ actual_path = File.join(runner.base_directory, 'output', "#{script}.err")
303
312
  actual_output = File.read(actual_path)
304
313
  expect(actual_output).to match(expected)
305
314
  end
306
315
  end
316
+
317
+ def test_runner(name, opts={})
318
+ test_bot_location = File.join(File.dirname(__FILE__), '../bots', name)
319
+ TurbotRunner::Runner.new(test_bot_location, opts)
320
+ end
data/spec/manual_spec.rb CHANGED
@@ -18,7 +18,8 @@ eos
18
18
 
19
19
  gets
20
20
 
21
- runner = TurbotRunner::Runner.new('spec/bots/logging-bot').run
21
+ bot_location = File.join(File.dirname(__FILE__), 'bots/logging-bot')
22
+ runner = TurbotRunner::Runner.new(bot_location).run
22
23
 
23
24
  puts
24
25
  puts 'Did you see the expected lines? [y]/n'
@@ -38,7 +39,8 @@ eos
38
39
 
39
40
  gets
40
41
 
41
- runner = TurbotRunner::Runner.new('spec/bots/bot-with-pause').run
42
+ bot_location = File.join(File.dirname(__FILE__), 'bots/bot-with-pause')
43
+ runner = TurbotRunner::Runner.new(bot_location).run
42
44
 
43
45
  expected_output = File.readlines('spec/outputs/truncated-scraper.out').map {|line| JSON.parse(line)}
44
46
  actual_output = File.readlines('spec/bots/bot-with-pause/output/scraper.out').map {|line| JSON.parse(line)}
metadata CHANGED
@@ -1,35 +1,46 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: turbot-runner
3
- version: !ruby/object:Gem::Version
4
- version: 0.1.18
3
+ version: !ruby/object:Gem::Version
4
+ hash: 61
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 1
9
+ - 19
10
+ version: 0.1.19
5
11
  platform: ruby
6
- authors:
12
+ authors:
7
13
  - OpenCorporates
8
14
  autorequire:
9
15
  bindir: bin
10
16
  cert_chain: []
11
- date: 2015-01-06 00:00:00.000000000 Z
12
- dependencies:
13
- - !ruby/object:Gem::Dependency
17
+
18
+ date: 2015-01-07 00:00:00 Z
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
14
21
  name: json-schema
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - '='
18
- - !ruby/object:Gem::Version
19
- version: 2.2.2
20
- type: :runtime
21
22
  prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - '='
25
- - !ruby/object:Gem::Version
26
- version: 2.2.2
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - ~>
27
+ - !ruby/object:Gem::Version
28
+ hash: 11
29
+ segments:
30
+ - 2
31
+ - 4
32
+ version: "2.4"
33
+ type: :runtime
34
+ version_requirements: *id001
27
35
  description:
28
36
  email: bots@opencorporates.com
29
37
  executables: []
38
+
30
39
  extensions: []
40
+
31
41
  extra_rdoc_files: []
32
- files:
42
+
43
+ files:
33
44
  - bin/rspec
34
45
  - lib/turbot_runner.rb
35
46
  - lib/turbot_runner/base_handler.rb
@@ -38,36 +49,8 @@ files:
38
49
  - lib/turbot_runner/processor.rb
39
50
  - lib/turbot_runner/runner.rb
40
51
  - lib/turbot_runner/script_runner.rb
52
+ - lib/turbot_runner/validators.rb
41
53
  - lib/turbot_runner/version.rb
42
- - schema/schemas/company-schema.json
43
- - schema/schemas/financial-payment-schema.json
44
- - schema/schemas/includes/address.json
45
- - schema/schemas/includes/alternative_name.json
46
- - schema/schemas/includes/company.json
47
- - schema/schemas/includes/filing.json
48
- - schema/schemas/includes/financial-payment-data-object.json
49
- - schema/schemas/includes/identifier.json
50
- - schema/schemas/includes/industry_code.json
51
- - schema/schemas/includes/licence-data-object.json
52
- - schema/schemas/includes/officer.json
53
- - schema/schemas/includes/organisation.json
54
- - schema/schemas/includes/permission.json
55
- - schema/schemas/includes/person.json
56
- - schema/schemas/includes/person_name.json
57
- - schema/schemas/includes/previous_name.json
58
- - schema/schemas/includes/share-parcel-data.json
59
- - schema/schemas/includes/share-parcel.json
60
- - schema/schemas/includes/subsidiary-relationship-data.json
61
- - schema/schemas/includes/total-shares.json
62
- - schema/schemas/includes/unknown_entity_type.json
63
- - schema/schemas/licence-schema.json
64
- - schema/schemas/primary-data-schema.json
65
- - schema/schemas/rich-licence-schema.json
66
- - schema/schemas/share-parcel-schema.json
67
- - schema/schemas/simple-financial-payment-schema.json
68
- - schema/schemas/simple-licence-schema.json
69
- - schema/schemas/simple-subsidiary-schema.json
70
- - schema/schemas/subsidiary-relationship-schema.json
71
54
  - spec/bots/bot-that-crashes-immediately/manifest.json
72
55
  - spec/bots/bot-that-crashes-immediately/scraper.rb
73
56
  - spec/bots/bot-that-crashes-immediately/transformer1.rb
@@ -80,6 +63,11 @@ files:
80
63
  - spec/bots/bot-that-crashes-in-transformer/transformer2.rb
81
64
  - spec/bots/bot-that-emits-run-ended/manifest.json
82
65
  - spec/bots/bot-that-emits-run-ended/scraper.rb
66
+ - spec/bots/bot-that-expects-file/manifest.json
67
+ - spec/bots/bot-that-expects-file/scraper.rb
68
+ - spec/bots/bot-that-expects-file/something.txt
69
+ - spec/bots/bot-with-invalid-sample-date/manifest.json
70
+ - spec/bots/bot-with-invalid-sample-date/scraper.rb
83
71
  - spec/bots/bot-with-pause/manifest.json
84
72
  - spec/bots/bot-with-pause/scraper.rb
85
73
  - spec/bots/bot-with-transformer/manifest.json
@@ -107,29 +95,69 @@ files:
107
95
  - spec/outputs/full-scraper.out
108
96
  - spec/outputs/full-transformer.out
109
97
  - spec/outputs/truncated-scraper.out
98
+ - schema/schemas/company-schema.json
99
+ - schema/schemas/financial-payment-schema.json
100
+ - schema/schemas/includes/address.json
101
+ - schema/schemas/includes/alternative_name.json
102
+ - schema/schemas/includes/company.json
103
+ - schema/schemas/includes/filing.json
104
+ - schema/schemas/includes/financial-payment-data-object.json
105
+ - schema/schemas/includes/identifier.json
106
+ - schema/schemas/includes/industry_code.json
107
+ - schema/schemas/includes/licence-data-object.json
108
+ - schema/schemas/includes/officer.json
109
+ - schema/schemas/includes/organisation.json
110
+ - schema/schemas/includes/permission.json
111
+ - schema/schemas/includes/person.json
112
+ - schema/schemas/includes/person_name.json
113
+ - schema/schemas/includes/previous_name.json
114
+ - schema/schemas/includes/share-parcel-data.json
115
+ - schema/schemas/includes/share-parcel.json
116
+ - schema/schemas/includes/subsidiary-relationship-data.json
117
+ - schema/schemas/includes/total-shares.json
118
+ - schema/schemas/includes/unknown_entity_type.json
119
+ - schema/schemas/licence-schema.json
120
+ - schema/schemas/primary-data-schema.json
121
+ - schema/schemas/rich-licence-schema.json
122
+ - schema/schemas/share-parcel-schema.json
123
+ - schema/schemas/simple-financial-payment-schema.json
124
+ - schema/schemas/simple-licence-schema.json
125
+ - schema/schemas/simple-subsidiary-schema.json
126
+ - schema/schemas/subsidiary-relationship-schema.json
110
127
  homepage: http://turbot.opencorporates.com/
111
- licenses:
128
+ licenses:
112
129
  - MIT
113
- metadata: {}
114
130
  post_install_message:
115
131
  rdoc_options: []
116
- require_paths:
132
+
133
+ require_paths:
117
134
  - lib
118
- required_ruby_version: !ruby/object:Gem::Requirement
119
- requirements:
120
- - - ! '>='
121
- - !ruby/object:Gem::Version
135
+ required_ruby_version: !ruby/object:Gem::Requirement
136
+ none: false
137
+ requirements:
138
+ - - ">="
139
+ - !ruby/object:Gem::Version
140
+ hash: 55
141
+ segments:
142
+ - 1
143
+ - 9
144
+ - 2
122
145
  version: 1.9.2
123
- required_rubygems_version: !ruby/object:Gem::Requirement
124
- requirements:
125
- - - ! '>='
126
- - !ruby/object:Gem::Version
127
- version: '0'
146
+ required_rubygems_version: !ruby/object:Gem::Requirement
147
+ none: false
148
+ requirements:
149
+ - - ">="
150
+ - !ruby/object:Gem::Version
151
+ hash: 3
152
+ segments:
153
+ - 0
154
+ version: "0"
128
155
  requirements: []
156
+
129
157
  rubyforge_project:
130
- rubygems_version: 2.2.2
158
+ rubygems_version: 1.8.15
131
159
  signing_key:
132
- specification_version: 4
160
+ specification_version: 3
133
161
  summary: Utilities for running bots with Turbot
134
162
  test_files: []
135
- has_rdoc:
163
+
checksums.yaml DELETED
@@ -1,15 +0,0 @@
1
- ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- NDlhMjNhODQ5ZDI1NmVlMWUxMTJkNDMwOTQ4MDY0MzljYjZhZTRkMg==
5
- data.tar.gz: !binary |-
6
- M2FjMGQ0ODAxYTUwZjc5MDBmY2ViN2RlMDlmZmY0MzZlODA2MTU2MQ==
7
- SHA512:
8
- metadata.gz: !binary |-
9
- YmJkMjNjOThlNjJmNDM0MmRjYjIxNDQ1YjI5MDMxNGQxNDhmOTRiOGNiMTU0
10
- MGVlZTRhOGU4OTE2YjVjN2I3ZTU4ODhhZDdhZWJhMDg5NGMxOTcyOGQ4ZmFk
11
- MzA2YmViNmM1YmQwOTgwODBlYWM0OGEzNDc3YzYxYjM0YjIzOTQ=
12
- data.tar.gz: !binary |-
13
- MjZkNTQ4MGY4OGNlYTAwOWMzMTZlNDM2MmJkZDFhZTgxZDExOWVhN2QxOWEy
14
- MjQzMGJiOWY2MTJiNzAxZWQ3ZGRkODA1ZjdmNzk0YjY2OTc3MDFlNzk3ZDIx
15
- YTExNDViYTU5MjZiNzY3OThkMDBkNzg3MmRmNWY5NDAwMzM3MWQ=