turbot-runner 0.1.18 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,24 @@
1
1
  require 'json'
2
2
  require 'fileutils'
3
+ require 'pathname'
3
4
 
4
5
  module TurbotRunner
5
6
  class Runner
6
- attr_reader :directory
7
+ attr_reader :base_directory
7
8
 
8
9
  def initialize(directory, options={})
9
- @directory = directory
10
+ assert_absolute_path(directory)
11
+ @base_directory = directory
10
12
  @config = load_config(directory)
11
13
  @record_handler = options[:record_handler]
12
14
  @log_to_file = options[:log_to_file]
13
15
  @timeout = options[:timeout]
14
- @output_directory = options[:output_directory] || File.join(@directory, 'output')
16
+ if options[:output_directory]
17
+ assert_absolute_path(options[:output_directory])
18
+ @output_directory = options[:output_directory]
19
+ else
20
+ @output_directory = File.join(@base_directory, 'output')
21
+ end
15
22
  end
16
23
 
17
24
  def run
@@ -20,14 +27,15 @@ module TurbotRunner
20
27
  succeeded = run_script(scraper_config)
21
28
  # Run the transformers even if the scraper fails
22
29
  transformers.each do |transformer_config|
23
- succeeded = run_script(transformer_config, input_file=scraper_output_file) && succeeded
30
+ succeeded = run_script(
31
+ transformer_config.merge(:base_directory => @base_directory),
32
+ input_file=scraper_output_file) && succeeded
24
33
  end
25
34
  succeeded
26
35
  end
27
36
 
28
37
  def set_up_output_directory
29
38
  FileUtils.mkdir_p(@output_directory)
30
-
31
39
  FileUtils.rm_f(File.join(@output_directory, 'scraper.out'))
32
40
  FileUtils.rm_f(File.join(@output_directory, 'scraper.err'))
33
41
 
@@ -41,7 +49,7 @@ module TurbotRunner
41
49
  process_script_output(scraper_config)
42
50
 
43
51
  transformers.each do |transformer_config|
44
- process_script_output(transformer_config)
52
+ process_script_output(transformer_config.merge(:base_directory => @base_directory))
45
53
  end
46
54
  end
47
55
 
@@ -70,9 +78,9 @@ module TurbotRunner
70
78
  end
71
79
  end
72
80
 
81
+
73
82
  def run_script(script_config, input_file=nil)
74
83
  command = build_command(script_config[:file], input_file)
75
-
76
84
  script_runner = ScriptRunner.new(
77
85
  command,
78
86
  output_file(script_config[:file]),
@@ -90,8 +98,8 @@ module TurbotRunner
90
98
  # (e.g. interruptions etc) is required; we just want to do
91
99
  # record handling.
92
100
  processor = Processor.new(nil, script_config, @record_handler)
93
-
94
- File.open(output_file(script_config[:file])) do |f|
101
+ file = output_file(script_config[:file])
102
+ File.open(file) do |f|
95
103
  f.each_line do |line|
96
104
  processor.process(line)
97
105
  end
@@ -104,11 +112,9 @@ module TurbotRunner
104
112
 
105
113
  def build_command(script, input_file=nil)
106
114
  raise "Could not run #{script} with #{language}" unless script_extension == File.extname(script)
107
- path_to_script = File.join(@directory, script)
108
- command = "#{full_interpreter_path} #{additional_args} #{path_to_script} >#{output_file(script)}"
115
+ command = "#{full_interpreter_path} #{additional_args} #{script} >#{output_file(script)}"
109
116
  command << " 2>#{output_file(script, '.err')}" if @log_to_file
110
117
  command << " <#{input_file}" unless input_file.nil?
111
-
112
118
  command
113
119
  end
114
120
 
@@ -133,6 +139,7 @@ module TurbotRunner
133
139
 
134
140
  def scraper_config
135
141
  {
142
+ :base_directory => @base_directory,
136
143
  :file => scraper_script,
137
144
  :data_type => scraper_data_type,
138
145
  :identifying_fields => scraper_identifying_fields
@@ -162,5 +169,11 @@ module TurbotRunner
162
169
  def scraper_identifying_fields
163
170
  @config[:identifying_fields]
164
171
  end
172
+
173
+ def assert_absolute_path(path)
174
+ unless Pathname.new(path).absolute?
175
+ raise "#{path} must be an absolute path"
176
+ end
177
+ end
165
178
  end
166
179
  end
@@ -9,65 +9,67 @@ module TurbotRunner
9
9
  def initialize(command, output_file, script_config, options={})
10
10
  @command = command
11
11
  @output_file = output_file
12
-
12
+ @script_config = script_config
13
13
  record_handler = options[:record_handler] || BaseHandler.new # A BaseHandler does nothing
14
14
  @processor = Processor.new(self, script_config, record_handler)
15
-
16
15
  @timeout = options[:timeout] || 3600
17
16
  end
18
17
 
19
18
  def run
20
- @interrupted = false
21
- @failed = false
22
-
23
- # Start a thread that spawns a subprocess that runs the script and
24
- # redirects the script's output to a file at a known location.
25
- script_thread = Thread.new { run_command(@command) }
19
+ Dir.chdir(@script_config[:base_directory]) do
26
20
 
27
- # Wait for the output file to be created, so that we can start to read
28
- # from it.
29
- begin
30
- f = File.open(@output_file)
31
- rescue Errno::ENOENT
32
- sleep 0.1
33
- retry
34
- end
35
-
36
- # Read from output file buildling up lines byte by byte byte by byte
37
- # until either we reach the end of the file and the script has exited, or
38
- # @interrupted becomes true. We cannot use IO#readline here because if
39
- # only half a line has been synced to the file by the time we read it,
40
- # then the incomplete line will be read, causing chaos down the line.
41
- line = ''
21
+ begin
22
+ @interrupted = false
23
+ @failed = false
42
24
 
43
- time_of_last_read = Time.now
25
+ # Start a thread that spawns a subprocess that runs the script and
26
+ # redirects the script's output to a file at a known location.
27
+ script_thread = Thread.new { run_command(@command) }
44
28
 
45
- until @interrupted do
46
- byte = f.read(1)
47
- if byte.nil?
48
- if script_thread.alive?
29
+ # Wait for the output file to be created, so that we can start to read
30
+ # from it.
31
+ begin
32
+ f = File.open(@output_file, "r")
33
+ rescue Errno::ENOENT
49
34
  sleep 0.1
50
- interrupt_and_mark_as_failed if (Time.now - time_of_last_read) > @timeout
51
- else
52
- break
35
+ retry
53
36
  end
54
- elsif byte == "\n"
55
- @processor.process(line)
56
- time_of_last_read = Time.now
37
+ # Read from output file buildling up lines byte by byte byte by byte
38
+ # until either we reach the end of the file and the script has exited, or
39
+ # @interrupted becomes true. We cannot use IO#readline here because if
40
+ # only half a line has been synced to the file by the time we read it,
41
+ # then the incomplete line will be read, causing chaos down the line.
57
42
  line = ''
58
- else
43
+
59
44
  time_of_last_read = Time.now
60
- line << byte
61
- end
62
- end
45
+ until @interrupted do
46
+ byte = f.read(1)
47
+ if byte.nil?
48
+ if script_thread.alive?
49
+ sleep 0.1
50
+ interrupt_and_mark_as_failed if (Time.now - time_of_last_read) > @timeout
51
+ else
52
+ break
53
+ end
54
+ elsif byte == "\n"
55
+ @processor.process(line)
56
+ time_of_last_read = Time.now
57
+ line = ''
58
+ else
59
+ time_of_last_read = Time.now
60
+ line << byte
61
+ end
62
+ end
63
63
 
64
- # script_thread may still be alive if we exited the loop above becuase
65
- # @interrupted became true, and so we must kill it.
66
- kill_running_processes if script_thread.alive?
64
+ # script_thread may still be alive if we exited the loop above becuase
65
+ # @interrupted became true, and so we must kill it.
66
+ kill_running_processes if script_thread.alive?
67
67
 
68
- @failed ? false : script_thread.join.value
69
- ensure
70
- f.close if f
68
+ @failed ? false : script_thread.join.value
69
+ ensure
70
+ f.close if f
71
+ end
72
+ end
71
73
  end
72
74
 
73
75
  def interrupt
@@ -82,7 +84,6 @@ module TurbotRunner
82
84
  private
83
85
  def run_command(command)
84
86
  system(command)
85
-
86
87
  # A nil exitstatus indicates that the script was interrupted. A
87
88
  # termsig of 2 indicates that the script was interrupted by a SIGINT.
88
89
  $?.exitstatus == 0 || ($?.exitstatus.nil? && $?.termsig == 2)
@@ -0,0 +1,12 @@
1
+ require 'date'
2
+ require 'json-schema'
3
+
4
+ date_format_validator = -> value {
5
+ begin
6
+ Date.strptime(value, '%Y-%m-%d')
7
+ rescue ArgumentError
8
+ raise JSON::Schema::CustomFormatError.new('must be of format yyyy-mm-dd')
9
+ end
10
+ }
11
+
12
+ JSON::Validator.register_format_validator('date', date_format_validator)
@@ -1,3 +1,3 @@
1
1
  module TurbotRunner
2
- VERSION = '0.1.18'
2
+ VERSION = '0.1.19'
3
3
  end
data/lib/turbot_runner.rb CHANGED
@@ -3,4 +3,5 @@ require 'turbot_runner/exceptions'
3
3
  require 'turbot_runner/processor'
4
4
  require 'turbot_runner/runner'
5
5
  require 'turbot_runner/script_runner'
6
+ require 'turbot_runner/validators'
6
7
  require 'turbot_runner/version'
@@ -0,0 +1,8 @@
1
+ {
2
+ "bot_id": "bot-that-expects-file",
3
+ "description": "This bot depends on being able to open a file in the current directory",
4
+ "language": "ruby",
5
+ "data_type": "primary data",
6
+ "identifying_fields": ["licence_number"],
7
+ "files": ["scraper.rb", "something.txt"]
8
+ }
@@ -0,0 +1,11 @@
1
+ require 'json'
2
+
3
+ asd = open("something.txt", "r")
4
+ 0.upto(2) do |n|
5
+ record = {
6
+ :licence_number => "XYZ#{n}",
7
+ :source_url => 'http://example.com',
8
+ :sample_date => '2014-06-01'
9
+ }
10
+ puts(record.to_json)
11
+ end
@@ -0,0 +1,8 @@
1
+ {
2
+ "bot_id": "bot-with-invalid-sample-date",
3
+ "description": "This bot produces data with an invalid sample date",
4
+ "language": "ruby",
5
+ "data_type": "primary data",
6
+ "identifying_fields": ["licence_number"],
7
+ "files": ["scraper.rb"]
8
+ }
@@ -0,0 +1,10 @@
1
+ require 'json'
2
+
3
+ 0.upto(9) do |n|
4
+ record = {
5
+ :licence_number => "XYZ#{n}",
6
+ :source_url => 'http://example.com',
7
+ :sample_date => '01/06/2014'
8
+ }
9
+ puts(record.to_json)
10
+ end
@@ -14,7 +14,7 @@ describe TurbotRunner::Runner do
14
14
  describe '#run' do
15
15
  context 'with a bot written in ruby' do
16
16
  before do
17
- @runner = TurbotRunner::Runner.new('spec/bots/ruby-bot')
17
+ @runner = test_runner('ruby-bot')
18
18
  end
19
19
 
20
20
  it 'produces expected output' do
@@ -29,7 +29,7 @@ describe TurbotRunner::Runner do
29
29
 
30
30
  context 'with a bot written in python' do
31
31
  before do
32
- @runner = TurbotRunner::Runner.new('spec/bots/python-bot')
32
+ @runner = test_runner('python-bot')
33
33
  end
34
34
 
35
35
  it 'produces expected output' do
@@ -40,7 +40,7 @@ describe TurbotRunner::Runner do
40
40
 
41
41
  context 'with a bot with a transformer' do
42
42
  before do
43
- @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformer')
43
+ @runner = test_runner('bot-with-transformer')
44
44
  end
45
45
 
46
46
  it 'produces expected outputs' do
@@ -56,7 +56,7 @@ describe TurbotRunner::Runner do
56
56
 
57
57
  context 'with a bot with multiple transformers' do
58
58
  before do
59
- @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformers')
59
+ @runner = test_runner('bot-with-transformers')
60
60
  end
61
61
 
62
62
  it 'produces expected outputs' do
@@ -75,10 +75,7 @@ describe TurbotRunner::Runner do
75
75
  context 'when logging to file enabled' do
76
76
  it 'logs to file' do
77
77
  expected_log = "doing...\ndone\n"
78
- runner = TurbotRunner::Runner.new(
79
- 'spec/bots/logging-bot',
80
- :log_to_file => true
81
- )
78
+ runner = test_runner('logging-bot', :log_to_file => true)
82
79
  runner.run
83
80
  expect([runner, 'scraper']).to have_error_output_matching(expected_log)
84
81
  end
@@ -93,10 +90,7 @@ describe TurbotRunner::Runner do
93
90
 
94
91
  context 'with a bot that outputs RUN ENDED' do
95
92
  before do
96
- @runner = TurbotRunner::Runner.new(
97
- 'spec/bots/bot-that-emits-run-ended',
98
- :log_to_file => true
99
- )
93
+ @runner = test_runner('bot-that-emits-run-ended', :log_to_file => true)
100
94
  end
101
95
  it 'calls handle_run_ended on the handler' do
102
96
  expect_any_instance_of(TurbotRunner::BaseHandler).to receive(:handle_run_ended)
@@ -112,10 +106,7 @@ describe TurbotRunner::Runner do
112
106
 
113
107
  context 'with a bot that crashes in scraper' do
114
108
  before do
115
- @runner = TurbotRunner::Runner.new(
116
- 'spec/bots/bot-that-crashes-in-scraper',
117
- :log_to_file => true
118
- )
109
+ @runner = test_runner('bot-that-crashes-in-scraper', :log_to_file => true)
119
110
  end
120
111
 
121
112
  it 'returns false' do
@@ -136,12 +127,19 @@ describe TurbotRunner::Runner do
136
127
  end
137
128
  end
138
129
 
130
+ context 'with a bot that expects a file to be present in the working directory' do
131
+ before do
132
+ @runner = test_runner('bot-that-expects-file')
133
+ end
134
+
135
+ it 'returns true' do
136
+ expect(@runner.run).to be(true)
137
+ end
138
+ end
139
+
139
140
  context 'with a bot that crashes in transformer' do
140
141
  before do
141
- @runner = TurbotRunner::Runner.new(
142
- 'spec/bots/bot-that-crashes-in-transformer',
143
- :log_to_file => true
144
- )
142
+ @runner = test_runner('bot-that-crashes-in-transformer', :log_to_file => true)
145
143
  end
146
144
 
147
145
  it 'returns false' do
@@ -174,8 +172,7 @@ describe TurbotRunner::Runner do
174
172
  end
175
173
  end
176
174
 
177
- @runner = TurbotRunner::Runner.new(
178
- 'spec/bots/slow-bot',
175
+ @runner = test_runner('slow-bot',
179
176
  :record_handler => Handler.new,
180
177
  :log_to_file => true
181
178
  )
@@ -193,28 +190,42 @@ describe TurbotRunner::Runner do
193
190
 
194
191
  context 'with a scraper that produces an invalid record' do
195
192
  it 'returns false' do
196
- @runner = TurbotRunner::Runner.new('spec/bots/invalid-record-bot')
193
+ @runner = test_runner('invalid-record-bot')
197
194
  expect(@runner.run).to be(false)
198
195
  end
199
196
  end
200
197
 
201
198
  context 'with a scraper that produces invalid JSON' do
202
199
  it 'returns false' do
203
- @runner = TurbotRunner::Runner.new('spec/bots/invalid-json-bot')
200
+ @runner = test_runner('invalid-json-bot')
204
201
  expect(@runner.run).to be(false)
205
202
  end
206
203
  end
207
204
 
208
205
  context 'with a scraper that hangs' do
206
+ # XXX This spec fails because the loop in ScriptRunner#run that
207
+ # reads lines from the output file doesn't start until the
208
+ # output file is created; however, the way we're redirecting
209
+ # stdout using the shell means the file doesn't get created
210
+ # until
209
211
  it 'returns false' do
210
- @runner = TurbotRunner::Runner.new(
211
- 'spec/bots/bot-with-pause',
212
+ @runner = test_runner('bot-with-pause',
212
213
  :timeout => 1,
213
214
  :log_to_file => true
214
215
  )
215
216
  expect(@runner.run).to be(false)
216
217
  end
217
218
  end
219
+
220
+ context 'with a bot that emits an invalid sample date' do
221
+ before do
222
+ @runner = TurbotRunner::Runner.new('spec/bots/bot-with-invalid-sample-date')
223
+ end
224
+
225
+ it 'returns false' do
226
+ expect(@runner.run).to be(true)
227
+ end
228
+ end
218
229
  end
219
230
 
220
231
  describe '#process_output' do
@@ -236,10 +247,9 @@ describe TurbotRunner::Runner do
236
247
  end
237
248
 
238
249
  it 'calls handler once for each line of output' do
239
- TurbotRunner::Runner.new('spec/bots/bot-with-transformer').run
250
+ test_runner('bot-with-transformer').run
240
251
 
241
- runner = TurbotRunner::Runner.new(
242
- 'spec/bots/bot-with-transformer',
252
+ runner = test_runner('bot-with-transformer',
243
253
  :record_handler => @handler
244
254
  )
245
255
 
@@ -249,10 +259,9 @@ describe TurbotRunner::Runner do
249
259
  end
250
260
 
251
261
  it 'can cope when scraper has failed immediately' do
252
- TurbotRunner::Runner.new('spec/bots/bot-that-crashes-immediately').run
262
+ test_runner('bot-that-crashes-immediately').run
253
263
 
254
- runner = TurbotRunner::Runner.new(
255
- 'spec/bots/bot-with-transformer',
264
+ runner = test_runner('bot-with-transformer',
256
265
  :record_handler => @handler
257
266
  )
258
267
 
@@ -262,18 +271,18 @@ describe TurbotRunner::Runner do
262
271
 
263
272
  describe '#set_up_output_directory' do
264
273
  before do
265
- @runner = TurbotRunner::Runner.new('spec/bots/bot-with-transformer')
274
+ @runner = test_runner('bot-with-transformer')
266
275
  end
267
276
 
268
277
  it 'clears existing output' do
269
- path = File.join(@runner.directory, 'output', 'scraper.out')
278
+ path = File.join(@runner.base_directory, 'output', 'scraper.out')
270
279
  FileUtils.touch(path)
271
280
  @runner.set_up_output_directory
272
281
  expect(File.exist?(path)).to be(false)
273
282
  end
274
283
 
275
284
  it 'does not clear existing files that are not output files' do
276
- path = File.join(@runner.directory, 'output', 'stdout')
285
+ path = File.join(@runner.base_directory, 'output', 'stdout')
277
286
  FileUtils.touch(path)
278
287
  @runner.set_up_output_directory
279
288
  expect(File.exist?(path)).to be(true)
@@ -288,7 +297,7 @@ RSpec::Matchers.define :have_output do |expected|
288
297
 
289
298
  expected_path = File.join('spec', 'outputs', expected)
290
299
  expected_output = File.readlines(expected_path).map {|line| JSON.parse(line)}
291
- actual_path = File.join(runner.directory, 'output', "#{script}.out")
300
+ actual_path = File.join(runner.base_directory, 'output', "#{script}.out")
292
301
  actual_output = File.readlines(actual_path).map {|line| JSON.parse(line)}
293
302
  expect(expected_output).to eq(actual_output)
294
303
  end
@@ -299,8 +308,13 @@ RSpec::Matchers.define :have_error_output_matching do |expected|
299
308
  match do |actual|
300
309
  runner, script = actual
301
310
 
302
- actual_path = File.join(runner.directory, 'output', "#{script}.err")
311
+ actual_path = File.join(runner.base_directory, 'output', "#{script}.err")
303
312
  actual_output = File.read(actual_path)
304
313
  expect(actual_output).to match(expected)
305
314
  end
306
315
  end
316
+
317
+ def test_runner(name, opts={})
318
+ test_bot_location = File.join(File.dirname(__FILE__), '../bots', name)
319
+ TurbotRunner::Runner.new(test_bot_location, opts)
320
+ end
data/spec/manual_spec.rb CHANGED
@@ -18,7 +18,8 @@ eos
18
18
 
19
19
  gets
20
20
 
21
- runner = TurbotRunner::Runner.new('spec/bots/logging-bot').run
21
+ bot_location = File.join(File.dirname(__FILE__), 'bots/logging-bot')
22
+ runner = TurbotRunner::Runner.new(bot_location).run
22
23
 
23
24
  puts
24
25
  puts 'Did you see the expected lines? [y]/n'
@@ -38,7 +39,8 @@ eos
38
39
 
39
40
  gets
40
41
 
41
- runner = TurbotRunner::Runner.new('spec/bots/bot-with-pause').run
42
+ bot_location = File.join(File.dirname(__FILE__), 'bots/bot-with-pause')
43
+ runner = TurbotRunner::Runner.new(bot_location).run
42
44
 
43
45
  expected_output = File.readlines('spec/outputs/truncated-scraper.out').map {|line| JSON.parse(line)}
44
46
  actual_output = File.readlines('spec/bots/bot-with-pause/output/scraper.out').map {|line| JSON.parse(line)}
metadata CHANGED
@@ -1,35 +1,46 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: turbot-runner
3
- version: !ruby/object:Gem::Version
4
- version: 0.1.18
3
+ version: !ruby/object:Gem::Version
4
+ hash: 61
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 1
9
+ - 19
10
+ version: 0.1.19
5
11
  platform: ruby
6
- authors:
12
+ authors:
7
13
  - OpenCorporates
8
14
  autorequire:
9
15
  bindir: bin
10
16
  cert_chain: []
11
- date: 2015-01-06 00:00:00.000000000 Z
12
- dependencies:
13
- - !ruby/object:Gem::Dependency
17
+
18
+ date: 2015-01-07 00:00:00 Z
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
14
21
  name: json-schema
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - '='
18
- - !ruby/object:Gem::Version
19
- version: 2.2.2
20
- type: :runtime
21
22
  prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - '='
25
- - !ruby/object:Gem::Version
26
- version: 2.2.2
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - ~>
27
+ - !ruby/object:Gem::Version
28
+ hash: 11
29
+ segments:
30
+ - 2
31
+ - 4
32
+ version: "2.4"
33
+ type: :runtime
34
+ version_requirements: *id001
27
35
  description:
28
36
  email: bots@opencorporates.com
29
37
  executables: []
38
+
30
39
  extensions: []
40
+
31
41
  extra_rdoc_files: []
32
- files:
42
+
43
+ files:
33
44
  - bin/rspec
34
45
  - lib/turbot_runner.rb
35
46
  - lib/turbot_runner/base_handler.rb
@@ -38,36 +49,8 @@ files:
38
49
  - lib/turbot_runner/processor.rb
39
50
  - lib/turbot_runner/runner.rb
40
51
  - lib/turbot_runner/script_runner.rb
52
+ - lib/turbot_runner/validators.rb
41
53
  - lib/turbot_runner/version.rb
42
- - schema/schemas/company-schema.json
43
- - schema/schemas/financial-payment-schema.json
44
- - schema/schemas/includes/address.json
45
- - schema/schemas/includes/alternative_name.json
46
- - schema/schemas/includes/company.json
47
- - schema/schemas/includes/filing.json
48
- - schema/schemas/includes/financial-payment-data-object.json
49
- - schema/schemas/includes/identifier.json
50
- - schema/schemas/includes/industry_code.json
51
- - schema/schemas/includes/licence-data-object.json
52
- - schema/schemas/includes/officer.json
53
- - schema/schemas/includes/organisation.json
54
- - schema/schemas/includes/permission.json
55
- - schema/schemas/includes/person.json
56
- - schema/schemas/includes/person_name.json
57
- - schema/schemas/includes/previous_name.json
58
- - schema/schemas/includes/share-parcel-data.json
59
- - schema/schemas/includes/share-parcel.json
60
- - schema/schemas/includes/subsidiary-relationship-data.json
61
- - schema/schemas/includes/total-shares.json
62
- - schema/schemas/includes/unknown_entity_type.json
63
- - schema/schemas/licence-schema.json
64
- - schema/schemas/primary-data-schema.json
65
- - schema/schemas/rich-licence-schema.json
66
- - schema/schemas/share-parcel-schema.json
67
- - schema/schemas/simple-financial-payment-schema.json
68
- - schema/schemas/simple-licence-schema.json
69
- - schema/schemas/simple-subsidiary-schema.json
70
- - schema/schemas/subsidiary-relationship-schema.json
71
54
  - spec/bots/bot-that-crashes-immediately/manifest.json
72
55
  - spec/bots/bot-that-crashes-immediately/scraper.rb
73
56
  - spec/bots/bot-that-crashes-immediately/transformer1.rb
@@ -80,6 +63,11 @@ files:
80
63
  - spec/bots/bot-that-crashes-in-transformer/transformer2.rb
81
64
  - spec/bots/bot-that-emits-run-ended/manifest.json
82
65
  - spec/bots/bot-that-emits-run-ended/scraper.rb
66
+ - spec/bots/bot-that-expects-file/manifest.json
67
+ - spec/bots/bot-that-expects-file/scraper.rb
68
+ - spec/bots/bot-that-expects-file/something.txt
69
+ - spec/bots/bot-with-invalid-sample-date/manifest.json
70
+ - spec/bots/bot-with-invalid-sample-date/scraper.rb
83
71
  - spec/bots/bot-with-pause/manifest.json
84
72
  - spec/bots/bot-with-pause/scraper.rb
85
73
  - spec/bots/bot-with-transformer/manifest.json
@@ -107,29 +95,69 @@ files:
107
95
  - spec/outputs/full-scraper.out
108
96
  - spec/outputs/full-transformer.out
109
97
  - spec/outputs/truncated-scraper.out
98
+ - schema/schemas/company-schema.json
99
+ - schema/schemas/financial-payment-schema.json
100
+ - schema/schemas/includes/address.json
101
+ - schema/schemas/includes/alternative_name.json
102
+ - schema/schemas/includes/company.json
103
+ - schema/schemas/includes/filing.json
104
+ - schema/schemas/includes/financial-payment-data-object.json
105
+ - schema/schemas/includes/identifier.json
106
+ - schema/schemas/includes/industry_code.json
107
+ - schema/schemas/includes/licence-data-object.json
108
+ - schema/schemas/includes/officer.json
109
+ - schema/schemas/includes/organisation.json
110
+ - schema/schemas/includes/permission.json
111
+ - schema/schemas/includes/person.json
112
+ - schema/schemas/includes/person_name.json
113
+ - schema/schemas/includes/previous_name.json
114
+ - schema/schemas/includes/share-parcel-data.json
115
+ - schema/schemas/includes/share-parcel.json
116
+ - schema/schemas/includes/subsidiary-relationship-data.json
117
+ - schema/schemas/includes/total-shares.json
118
+ - schema/schemas/includes/unknown_entity_type.json
119
+ - schema/schemas/licence-schema.json
120
+ - schema/schemas/primary-data-schema.json
121
+ - schema/schemas/rich-licence-schema.json
122
+ - schema/schemas/share-parcel-schema.json
123
+ - schema/schemas/simple-financial-payment-schema.json
124
+ - schema/schemas/simple-licence-schema.json
125
+ - schema/schemas/simple-subsidiary-schema.json
126
+ - schema/schemas/subsidiary-relationship-schema.json
110
127
  homepage: http://turbot.opencorporates.com/
111
- licenses:
128
+ licenses:
112
129
  - MIT
113
- metadata: {}
114
130
  post_install_message:
115
131
  rdoc_options: []
116
- require_paths:
132
+
133
+ require_paths:
117
134
  - lib
118
- required_ruby_version: !ruby/object:Gem::Requirement
119
- requirements:
120
- - - ! '>='
121
- - !ruby/object:Gem::Version
135
+ required_ruby_version: !ruby/object:Gem::Requirement
136
+ none: false
137
+ requirements:
138
+ - - ">="
139
+ - !ruby/object:Gem::Version
140
+ hash: 55
141
+ segments:
142
+ - 1
143
+ - 9
144
+ - 2
122
145
  version: 1.9.2
123
- required_rubygems_version: !ruby/object:Gem::Requirement
124
- requirements:
125
- - - ! '>='
126
- - !ruby/object:Gem::Version
127
- version: '0'
146
+ required_rubygems_version: !ruby/object:Gem::Requirement
147
+ none: false
148
+ requirements:
149
+ - - ">="
150
+ - !ruby/object:Gem::Version
151
+ hash: 3
152
+ segments:
153
+ - 0
154
+ version: "0"
128
155
  requirements: []
156
+
129
157
  rubyforge_project:
130
- rubygems_version: 2.2.2
158
+ rubygems_version: 1.8.15
131
159
  signing_key:
132
- specification_version: 4
160
+ specification_version: 3
133
161
  summary: Utilities for running bots with Turbot
134
162
  test_files: []
135
- has_rdoc:
163
+
checksums.yaml DELETED
@@ -1,15 +0,0 @@
1
- ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- NDlhMjNhODQ5ZDI1NmVlMWUxMTJkNDMwOTQ4MDY0MzljYjZhZTRkMg==
5
- data.tar.gz: !binary |-
6
- M2FjMGQ0ODAxYTUwZjc5MDBmY2ViN2RlMDlmZmY0MzZlODA2MTU2MQ==
7
- SHA512:
8
- metadata.gz: !binary |-
9
- YmJkMjNjOThlNjJmNDM0MmRjYjIxNDQ1YjI5MDMxNGQxNDhmOTRiOGNiMTU0
10
- MGVlZTRhOGU4OTE2YjVjN2I3ZTU4ODhhZDdhZWJhMDg5NGMxOTcyOGQ4ZmFk
11
- MzA2YmViNmM1YmQwOTgwODBlYWM0OGEzNDc3YzYxYjM0YjIzOTQ=
12
- data.tar.gz: !binary |-
13
- MjZkNTQ4MGY4OGNlYTAwOWMzMTZlNDM2MmJkZDFhZTgxZDExOWVhN2QxOWEy
14
- MjQzMGJiOWY2MTJiNzAxZWQ3ZGRkODA1ZjdmNzk0YjY2OTc3MDFlNzk3ZDIx
15
- YTExNDViYTU5MjZiNzY3OThkMDBkNzg3MmRmNWY5NDAwMzM3MWQ=