turbot-runner 0.0.23 → 0.0.24
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/turbot_runner/version.rb +1 -1
- data/lib/turbot_runner.rb +28 -5
- data/spec/dummy-bot-python/scraper.py +9 -4
- data/spec/dummy-bot-ruby/scraper.rb +2 -0
- data/spec/dummy-broken-bot-ruby/scraper.rb +2 -0
- data/spec/turbot_runner_spec.rb +28 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YTNmMmJmZDNkNDllMTk0MjFmYTkxOTMxZWI3NzNiYjdlYzQ2NzgzMg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MThhNmM4ZGM3OTU2NDAyZGRkM2EzZTE2ZTA4ZDQzMjFmNTE3ODFmNw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
Yjc0NjdiZTc2NWE1NzAwMGIzMTBhNTIxNGE5YmQ4ZjFmMWMwZjcyMWJlYWFk
|
10
|
+
ZDM0YjcyNDRiYTE2MjMzOTQ4MDA4YzkwNjAzOGUyOGNjNmI3Y2U3YzEwODdk
|
11
|
+
YmQ1ZjYzYjdmNGY5ODIwZjZhMjQ5ODQ5OThlNWU4MDg1M2I5ODM=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YjNkMDAyMTBkYjY4YTg5ZWM5NDdlOGE2OWE5OGNkMzRkMDI1MjA4OWJjMDQ3
|
14
|
+
NTJiODM4MzIzNDQ2ZDg2ZTNlZDJkYTNmMzk4ODExODlhOTdjNTU0NDc5MmI1
|
15
|
+
NWIyMTY2MTVjOGI1OWYxNmNjMzJlOTdlZDU1Yzk0NzQxODc1ZmQ=
|
data/lib/turbot_runner.rb
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
#require 'active_support/core_ext/hash/slice'
|
2
2
|
#require 'active_support/core_ext/object/to_query'
|
3
3
|
require 'json'
|
4
|
+
require 'json-schema'
|
4
5
|
require 'open3'
|
5
6
|
require 'set'
|
6
7
|
require 'timeout'
|
8
|
+
require 'io/wait'
|
7
9
|
|
8
10
|
module TurbotRunner
|
9
11
|
class ScriptError < StandardError; end
|
@@ -128,20 +130,33 @@ module TurbotRunner
|
|
128
130
|
@status = :successful
|
129
131
|
handle_successful_run
|
130
132
|
end
|
133
|
+
|
134
|
+
handle_stderr(scraper_runner.drain_stderr)
|
135
|
+
|
136
|
+
transformers.each do |transformer|
|
137
|
+
runner = transformer['runner']
|
138
|
+
handle_stderr(runner.drain_stderr)
|
139
|
+
end
|
140
|
+
|
131
141
|
rescue ScriptError => e
|
132
142
|
if @interrupted
|
133
143
|
@status = :interrupted
|
134
144
|
handle_interrupted_run
|
135
145
|
else
|
136
146
|
@status = :failed
|
137
|
-
@error = e
|
138
147
|
handle_failed_run
|
139
148
|
end
|
140
149
|
end
|
141
150
|
ensure
|
151
|
+
handle_stderr(scraper_runner.drain_stderr)
|
142
152
|
scraper_runner.close unless scraper_runner.nil?
|
153
|
+
|
143
154
|
transformers.each do |transformer|
|
144
|
-
|
155
|
+
runner = transformer['runner']
|
156
|
+
if !runner.nil?
|
157
|
+
handle_stderr(runner.drain_stderr)
|
158
|
+
runner.close
|
159
|
+
end
|
145
160
|
end
|
146
161
|
end
|
147
162
|
|
@@ -226,8 +241,8 @@ module TurbotRunner
|
|
226
241
|
def handle_interrupted_run
|
227
242
|
end
|
228
243
|
|
229
|
-
def
|
230
|
-
|
244
|
+
def handle_stderr(data)
|
245
|
+
$stderr.write(data)
|
231
246
|
end
|
232
247
|
|
233
248
|
def scraper_file
|
@@ -278,6 +293,14 @@ module TurbotRunner
|
|
278
293
|
end
|
279
294
|
end
|
280
295
|
|
296
|
+
def drain_stderr
|
297
|
+
output = ''
|
298
|
+
while @stderr.ready?
|
299
|
+
output += @stderr.read(256)
|
300
|
+
end
|
301
|
+
output
|
302
|
+
end
|
303
|
+
|
281
304
|
def success?
|
282
305
|
if finished?
|
283
306
|
@wait_thread.value.success?
|
@@ -291,7 +314,7 @@ module TurbotRunner
|
|
291
314
|
end
|
292
315
|
|
293
316
|
def raise_if_failed!
|
294
|
-
raise TurbotRunner::ScriptError
|
317
|
+
raise TurbotRunner::ScriptError if failed?
|
295
318
|
end
|
296
319
|
|
297
320
|
def finished?
|
@@ -1,6 +1,11 @@
|
|
1
|
+
from __future__ import print_function
|
2
|
+
|
1
3
|
import json
|
4
|
+
import sys
|
5
|
+
|
6
|
+
print('hello from python', file=sys.stderr)
|
2
7
|
|
3
|
-
print
|
4
|
-
print
|
5
|
-
print
|
6
|
-
print
|
8
|
+
print(json.dumps({'n': 5, 'hello': 'hello, 5'}))
|
9
|
+
print(json.dumps({'n': 6, 'hello': 'hello, 6'}))
|
10
|
+
print(json.dumps({'n': 7}))
|
11
|
+
print(json.dumps({'n': 8, 'hello': 'hello, 8'}))
|
data/spec/turbot_runner_spec.rb
CHANGED
@@ -9,6 +9,9 @@ class SpecRunner < TurbotRunner::BaseRunner
|
|
9
9
|
[]
|
10
10
|
end
|
11
11
|
end
|
12
|
+
|
13
|
+
def handle_failed_run
|
14
|
+
end
|
12
15
|
end
|
13
16
|
|
14
17
|
class BrokenRunner < TurbotRunner::BaseRunner
|
@@ -25,6 +28,14 @@ end
|
|
25
28
|
|
26
29
|
|
27
30
|
describe TurbotRunner::BaseRunner do
|
31
|
+
before do
|
32
|
+
$stderr = StringIO.new
|
33
|
+
end
|
34
|
+
|
35
|
+
after do
|
36
|
+
$stderr = STDERR
|
37
|
+
end
|
38
|
+
|
28
39
|
it 'can run a ruby bot' do
|
29
40
|
runner = SpecRunner.new('spec/dummy-bot-ruby')
|
30
41
|
|
@@ -37,6 +48,7 @@ describe TurbotRunner::BaseRunner do
|
|
37
48
|
expect(runner).to receive(:handle_valid_record).with({'n' => 4, 'goodbye' => 'goodbye, 4'}, 'goodbye')
|
38
49
|
expect(runner).to receive(:handle_successful_run)
|
39
50
|
runner.run
|
51
|
+
expect($stderr.string).to eq("hello from ruby\n")
|
40
52
|
end
|
41
53
|
|
42
54
|
it 'can run a python bot' do
|
@@ -51,6 +63,7 @@ describe TurbotRunner::BaseRunner do
|
|
51
63
|
expect(runner).to receive(:handle_valid_record).with({'n' => 8, 'goodbye' => 'goodbye, 8'}, 'goodbye')
|
52
64
|
expect(runner).to receive(:handle_successful_run)
|
53
65
|
runner.run
|
66
|
+
expect($stderr.string).to eq("hello from python\n")
|
54
67
|
end
|
55
68
|
|
56
69
|
describe "broken bots" do
|
@@ -62,10 +75,11 @@ describe TurbotRunner::BaseRunner do
|
|
62
75
|
runner.run
|
63
76
|
end
|
64
77
|
|
65
|
-
it 'should
|
78
|
+
it 'should write exception to stderr' do
|
66
79
|
runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
|
67
80
|
runner.run
|
68
|
-
expect(
|
81
|
+
expect($stderr.string).to match(/^hello/)
|
82
|
+
expect($stderr.string).to match(/oops/)
|
69
83
|
end
|
70
84
|
end
|
71
85
|
|
@@ -77,6 +91,12 @@ describe TurbotRunner::BaseRunner do
|
|
77
91
|
expect(runner).to receive(:handle_failed_run)
|
78
92
|
runner.run
|
79
93
|
end
|
94
|
+
|
95
|
+
it 'should write exception to stderr' do
|
96
|
+
runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
|
97
|
+
runner.run
|
98
|
+
expect($stderr.string).to match(/oops/)
|
99
|
+
end
|
80
100
|
end
|
81
101
|
|
82
102
|
describe "sucessful bot with failing transformer" do
|
@@ -86,6 +106,12 @@ describe TurbotRunner::BaseRunner do
|
|
86
106
|
expect(runner).to receive(:handle_failed_run) # the transformer breaks immediately
|
87
107
|
runner.run
|
88
108
|
end
|
109
|
+
|
110
|
+
it 'should write exception to stderr' do
|
111
|
+
runner = BrokenRunner.new('spec/dummy-broken-bot-ruby')
|
112
|
+
runner.run
|
113
|
+
expect($stderr.string).to match(/oops/)
|
114
|
+
end
|
89
115
|
end
|
90
116
|
end
|
91
117
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: turbot-runner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.24
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- OpenCorporates
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-07-
|
11
|
+
date: 2014-07-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json-schema
|