rbbt-rest 1.8.9 → 1.8.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/rest/workflow/stream_task.rb +153 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1bbeba3d4ef508ff35ecfe2210b1d3b12d03a9d9
|
4
|
+
data.tar.gz: 809cd9b359f9658d5090a3a1ecdefbdcec706cde
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4b72a79a9580da0803d84881d085861a47760be7d7cfce50a4e6e4322c1d3a3696a8cf0d4589fdc810fea0b76b6f4234ff41ed39f4b63c11e332876578e6bd81
|
7
|
+
data.tar.gz: 81f55b8dfeefd7294f41a9ea695e80c692fd35c6a2f9634f3f6bee3ccc84497a7ed9bd1ba313002242f94a6b574c9c58503a97b70d3460c7a1061e3bdacbd4ac
|
@@ -48,7 +48,9 @@ class StreamWorkflowTask
|
|
48
48
|
EOL = "\r\n"
|
49
49
|
def read_chunk(sin, rest = "")
|
50
50
|
parts = []
|
51
|
-
c = sin.
|
51
|
+
c = sin.read(1024)
|
52
|
+
#c = sin.gets
|
53
|
+
raise "Early connection close" if c.nil?
|
52
54
|
c = rest << c unless rest.empty?
|
53
55
|
c = c[2..-1] if c[0..1] == EOL
|
54
56
|
index = c.index EOL
|
@@ -62,12 +64,11 @@ class StreamWorkflowTask
|
|
62
64
|
[parts, rest]
|
63
65
|
end
|
64
66
|
|
65
|
-
|
66
67
|
def copy_chunked_stream(sin, sout, boundary)
|
67
68
|
|
68
69
|
rest = ""
|
69
70
|
done = false
|
70
|
-
content =
|
71
|
+
content = false
|
71
72
|
|
72
73
|
while not done
|
73
74
|
parts, rest = read_chunk(sin, rest)
|
@@ -92,6 +93,38 @@ class StreamWorkflowTask
|
|
92
93
|
sout.close
|
93
94
|
end
|
94
95
|
|
96
|
+
def merge_chunks(sin, sout)
|
97
|
+
|
98
|
+
rest = ""
|
99
|
+
done = false
|
100
|
+
content = true
|
101
|
+
|
102
|
+
while not done
|
103
|
+
chunk_size_str = ""
|
104
|
+
while chunk_size_str.strip.empty?
|
105
|
+
chunk_size_str = sin.gets
|
106
|
+
raise "Error reading chunks" if chunk_size_str.nil?
|
107
|
+
end
|
108
|
+
size = chunk_size_str.strip.to_i(16)
|
109
|
+
break if size == 0
|
110
|
+
chunk = sin.read(size)
|
111
|
+
bound = sin.read(2)
|
112
|
+
raise "Size does not match" if false and chunk.length != size
|
113
|
+
sout.write chunk
|
114
|
+
end
|
115
|
+
|
116
|
+
sout.write rest
|
117
|
+
sout.close
|
118
|
+
end
|
119
|
+
|
120
|
+
def copy_until_boundary(sin, sout, boundary)
|
121
|
+
while line = sin.gets
|
122
|
+
break if line.include? boundary
|
123
|
+
sout.write line
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
|
95
128
|
def call(env)
|
96
129
|
if env["REQUEST_METHOD"] == "POST" and env["rack.hijack"] and env["CONTENT_TYPE"] and env["CONTENT_TYPE"].include? "Rbbt_Param_Stream" and env["HTTP_TRANSFER_ENCODING"] == 'chunked'
|
97
130
|
Log.high "Hijacking post data"
|
@@ -99,8 +132,14 @@ class StreamWorkflowTask
|
|
99
132
|
content_type = env["CONTENT_TYPE"]
|
100
133
|
boundary = content_type.match(/boundary=([^\s;]*)/)[1]
|
101
134
|
stream_input = content_type.match(/stream=([^\s;]*)/)[1]
|
102
|
-
|
135
|
+
post_stream_chunked = env["rack.hijack"].call
|
136
|
+
|
137
|
+
job_url = nil
|
103
138
|
begin
|
139
|
+
post_stream = Misc.open_pipe do |sin|
|
140
|
+
merge_chunks(post_stream_chunked, sin)
|
141
|
+
end
|
142
|
+
|
104
143
|
inputs, filename = read_normal_inputs(post_stream, boundary, stream_input)
|
105
144
|
|
106
145
|
input_stream_out, input_stream_in = Misc.pipe
|
@@ -110,6 +149,9 @@ class StreamWorkflowTask
|
|
110
149
|
workflow, task = parse_uri(env)
|
111
150
|
name = inputs.delete "jobname"
|
112
151
|
job = workflow.job(task, name, inputs)
|
152
|
+
Log.high "Run job #{job.path} with inputs #{Misc.fingerprint(inputs)}"
|
153
|
+
|
154
|
+
job_url = File.join("/", workflow.to_s, task, job.name)
|
113
155
|
|
114
156
|
task = task.to_sym
|
115
157
|
execution_type = case
|
@@ -130,28 +172,40 @@ class StreamWorkflowTask
|
|
130
172
|
when "exec", nil
|
131
173
|
job.exec(:stream)
|
132
174
|
when "sync", "synchronous", "async", "asynchronous"
|
133
|
-
job.
|
175
|
+
if job.done?
|
176
|
+
done_consumer = Thread.new do
|
177
|
+
while c = post_stream.read(1024)
|
178
|
+
end
|
179
|
+
end
|
180
|
+
else
|
181
|
+
job.run(:stream)
|
182
|
+
end
|
134
183
|
else
|
135
184
|
raise "Unknown execution_type: #{execution_type}"
|
136
185
|
end
|
137
186
|
|
138
187
|
t_in = Thread.new do
|
139
188
|
begin
|
140
|
-
|
189
|
+
copy_until_boundary(post_stream, input_stream_in, boundary)
|
190
|
+
input_stream_in.close
|
191
|
+
post_stream.close_read
|
141
192
|
rescue
|
142
193
|
Log.exception $!
|
143
194
|
end
|
144
|
-
end
|
195
|
+
end unless job.done?
|
145
196
|
|
146
197
|
job_output = TSV.get_stream job
|
147
198
|
t_out = Thread.new do
|
148
199
|
begin
|
149
|
-
|
200
|
+
post_stream_chunked.write "HTTP/1.1 200\r\n"
|
201
|
+
post_stream_chunked.write "RBBT-STREAMING-JOB-URL: #{ job_url }\r\n"
|
202
|
+
post_stream_chunked.write "\r\n"
|
150
203
|
while c = job_output.read(1024)
|
151
|
-
|
204
|
+
post_stream_chunked.write c
|
152
205
|
end
|
153
206
|
job_output.join if job_output.respond_to? :join
|
154
|
-
|
207
|
+
post_stream_chunked.close_write
|
208
|
+
done_consumer.join if done_consumer
|
155
209
|
rescue
|
156
210
|
Log.exception $!
|
157
211
|
job.abort
|
@@ -159,11 +213,100 @@ class StreamWorkflowTask
|
|
159
213
|
end
|
160
214
|
|
161
215
|
end
|
216
|
+
|
162
217
|
[200, {}, nil]
|
163
218
|
else
|
164
219
|
Log.high "NOT Hijacking post data"
|
165
220
|
@app.call(env)
|
166
221
|
end
|
167
222
|
end
|
223
|
+
|
224
|
+
#def call_old(env)
|
225
|
+
# if env["REQUEST_METHOD"] == "POST" and env["rack.hijack"] and env["CONTENT_TYPE"] and env["CONTENT_TYPE"].include? "Rbbt_Param_Stream" and env["HTTP_TRANSFER_ENCODING"] == 'chunked'
|
226
|
+
# Log.high "Hijacking post data"
|
227
|
+
# inputs = {}
|
228
|
+
# content_type = env["CONTENT_TYPE"]
|
229
|
+
# boundary = content_type.match(/boundary=([^\s;]*)/)[1]
|
230
|
+
# stream_input = content_type.match(/stream=([^\s;]*)/)[1]
|
231
|
+
# post_stream = env["rack.hijack"].call
|
232
|
+
# job_url = nil
|
233
|
+
# begin
|
234
|
+
# inputs, filename = read_normal_inputs(post_stream, boundary, stream_input)
|
235
|
+
|
236
|
+
# input_stream_out, input_stream_in = Misc.pipe
|
237
|
+
# Misc.add_stream_filename(input_stream_out, filename) if filename
|
238
|
+
# inputs[stream_input] = input_stream_out
|
239
|
+
|
240
|
+
# workflow, task = parse_uri(env)
|
241
|
+
# name = inputs.delete "jobname"
|
242
|
+
# job = workflow.job(task, name, inputs)
|
243
|
+
# Log.high "Run job #{job.path} with inputs #{Misc.fingerprint(inputs)}"
|
244
|
+
|
245
|
+
# job_url = File.join("/", workflow.to_s, task, job.name)
|
246
|
+
|
247
|
+
# task = task.to_sym
|
248
|
+
# execution_type = case
|
249
|
+
# when workflow.exec_exports.include?(task)
|
250
|
+
# "exec"
|
251
|
+
# when workflow.synchronous_exports.include?(task)
|
252
|
+
# "synchronous"
|
253
|
+
# when workflow.asynchronous_exports.include?(task)
|
254
|
+
# "asynchronous"
|
255
|
+
# else
|
256
|
+
# raise "No known export type for #{ workflow } #{ task }. Accesses denied"
|
257
|
+
# end
|
258
|
+
|
259
|
+
# execution_type = "exec" if inputs["_cache_type"] == 'exec'
|
260
|
+
# Log.info "Streaming task with execution_type: #{ execution_type }"
|
261
|
+
|
262
|
+
# case execution_type
|
263
|
+
# when "exec", nil
|
264
|
+
# job.exec(:stream)
|
265
|
+
# when "sync", "synchronous", "async", "asynchronous"
|
266
|
+
# if job.done?
|
267
|
+
# done_consumer = Thread.new do
|
268
|
+
# while c = post_stream.read(1024)
|
269
|
+
# end
|
270
|
+
# end
|
271
|
+
# else
|
272
|
+
# job.run(:stream)
|
273
|
+
# end
|
274
|
+
# else
|
275
|
+
# raise "Unknown execution_type: #{execution_type}"
|
276
|
+
# end
|
277
|
+
|
278
|
+
# t_in = Thread.new do
|
279
|
+
# begin
|
280
|
+
# copy_chunked_stream(post_stream, input_stream_in, boundary)
|
281
|
+
# rescue
|
282
|
+
# Log.exception $!
|
283
|
+
# end
|
284
|
+
# end unless job.done?
|
285
|
+
|
286
|
+
# job_output = TSV.get_stream job
|
287
|
+
# t_out = Thread.new do
|
288
|
+
# begin
|
289
|
+
# post_stream.write "HTTP/1.1 200\r\n"
|
290
|
+
# post_stream.write "RBBT-STREAMING-JOB-URL: #{ job_url }\r\n"
|
291
|
+
# post_stream.write "\r\n"
|
292
|
+
# while c = job_output.read(1024)
|
293
|
+
# post_stream.write c
|
294
|
+
# end
|
295
|
+
# job_output.join if job_output.respond_to? :join
|
296
|
+
# post_stream.close_write
|
297
|
+
# done_consumer.join if done_consumer
|
298
|
+
# rescue
|
299
|
+
# Log.exception $!
|
300
|
+
# job.abort
|
301
|
+
# end
|
302
|
+
# end
|
303
|
+
|
304
|
+
# end
|
305
|
+
# [200, {}, nil]
|
306
|
+
# else
|
307
|
+
# Log.high "NOT Hijacking post data"
|
308
|
+
# @app.call(env)
|
309
|
+
# end
|
310
|
+
#end
|
168
311
|
end
|
169
312
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-rest
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.8.
|
4
|
+
version: 1.8.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|