zillabyte-cli 0.9.0 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 90127131598b38dedc88b0903dc0b984537442a1
|
4
|
+
data.tar.gz: acf6b8bb47b8476621ea36834310ed48d479463a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d99663ba95dfbc8e76ef06656bf55621812f71c400c2ed170f00c5528cf04225b188b4b47e003fab82d8ac6f8f48f262261a47bd4f3750ef6d7715431ce85eee
|
7
|
+
data.tar.gz: 32ab54910d7b01b6171123ec1b7312a8fb8bafcb612c5465b3af5dfeaf77a0eda9e6c957bc179487bba29ea18550d63828b20e091a3640b03283448151d54d68
|
@@ -37,11 +37,13 @@ class Zillabyte::Runner::ComponentOperation
|
|
37
37
|
end
|
38
38
|
begin
|
39
39
|
case @__type
|
40
|
-
when "
|
40
|
+
when "input"
|
41
41
|
self.run_input()
|
42
42
|
when "component"
|
43
43
|
self.run_rpc_component()
|
44
|
-
|
44
|
+
when "output"
|
45
|
+
node["type"] = "sink"
|
46
|
+
Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
|
45
47
|
# Component outputs act in same manner as sinks
|
46
48
|
else
|
47
49
|
Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
|
@@ -50,6 +52,7 @@ class Zillabyte::Runner::ComponentOperation
|
|
50
52
|
cdisplay e.message
|
51
53
|
cdisplay e.backtrace
|
52
54
|
end
|
55
|
+
cdisplay "EXIT"
|
53
56
|
end
|
54
57
|
|
55
58
|
|
@@ -142,9 +145,8 @@ class Zillabyte::Runner::ComponentOperation
|
|
142
145
|
loop do
|
143
146
|
|
144
147
|
msg = stdin.gets
|
145
|
-
if msg ==
|
146
|
-
send_to_consumers(
|
147
|
-
send_to_consumers(DONE_MESSAGE)
|
148
|
+
if msg == KILL_CYCLE_MESSAGE
|
149
|
+
send_to_consumers(KILL_CYCLE_MESSAGE)
|
148
150
|
return
|
149
151
|
else
|
150
152
|
# Build tuples
|
@@ -271,13 +273,6 @@ class Zillabyte::Runner::ComponentOperation
|
|
271
273
|
# Handle Tuple in here
|
272
274
|
data_streams = hash['data']
|
273
275
|
|
274
|
-
# if stream_hash.nil?
|
275
|
-
# cdisplay("error : no results!")
|
276
|
-
# next
|
277
|
-
# else
|
278
|
-
# stream_hash.each_pair
|
279
|
-
# end
|
280
|
-
|
281
276
|
# Handle each resulting tuple
|
282
277
|
data_streams.each_pair do |stream, data_tuples|
|
283
278
|
data_tuples.each do |data_tuple|
|
@@ -334,7 +329,7 @@ class Zillabyte::Runner::ComponentOperation
|
|
334
329
|
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
335
330
|
consumers_running -= 1
|
336
331
|
if consumers_running == 0
|
337
|
-
|
332
|
+
return
|
338
333
|
end
|
339
334
|
|
340
335
|
# TODO break if last consumer
|
@@ -356,7 +351,7 @@ class Zillabyte::Runner::ComponentOperation
|
|
356
351
|
|
357
352
|
# Exit after ending consumer cycles
|
358
353
|
if consumers_running == 0
|
359
|
-
|
354
|
+
return
|
360
355
|
end
|
361
356
|
end
|
362
357
|
|
@@ -5,7 +5,7 @@ require "zillabyte/runner/component_operation"
|
|
5
5
|
class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
6
6
|
include Zillabyte::Helpers
|
7
7
|
|
8
|
-
|
8
|
+
KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
|
9
9
|
|
10
10
|
def run (meta, dir = Dir.pwd, session = nil, options = {})
|
11
11
|
|
@@ -154,7 +154,7 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
|
154
154
|
out_pipes = {}
|
155
155
|
|
156
156
|
# Check if you are the consumee for a downstream join in order to select the correct pipe
|
157
|
-
if type != "
|
157
|
+
if type != "output"
|
158
158
|
@arc_map[name].each_pair do |stream, destinations|
|
159
159
|
out_pipes[stream] ||= {}
|
160
160
|
|
@@ -216,7 +216,7 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
|
216
216
|
name = n["name"]
|
217
217
|
type = n["type"]
|
218
218
|
|
219
|
-
if type == "
|
219
|
+
if type == "input"
|
220
220
|
if source == ""
|
221
221
|
source = name
|
222
222
|
else
|
@@ -235,8 +235,13 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
|
235
235
|
fields = @node_map[source]['fields'].map {|h| h.keys[0].upcase }
|
236
236
|
display "Enter an input tuple in the form : #{fields.join(' ')}"
|
237
237
|
msg = ask
|
238
|
+
|
239
|
+
# Kill the cycle
|
238
240
|
if msg == 'end'
|
239
|
-
|
241
|
+
@operation_pipes[source]["wr_parent_1"].puts KILL_CYCLE_MESSAGE
|
242
|
+
break
|
243
|
+
|
244
|
+
# Check arguments
|
240
245
|
else
|
241
246
|
args = msg.scan(/(?:\w|"[^"]*")+/)
|
242
247
|
if (args.length % fields.length != 0)
|
@@ -244,11 +249,12 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
|
244
249
|
next
|
245
250
|
end
|
246
251
|
end
|
247
|
-
|
248
252
|
# Send tuple to source
|
249
253
|
@operation_pipes[source]["wr_parent_1"].puts msg
|
254
|
+
|
250
255
|
end
|
251
256
|
end
|
257
|
+
|
252
258
|
rescue => e
|
253
259
|
display e.message
|
254
260
|
display e.backtrace
|
@@ -311,12 +317,12 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
|
311
317
|
display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
|
312
318
|
|
313
319
|
# Convert metadata typing to that of components
|
314
|
-
if node['type'] == "
|
320
|
+
if node['type'] == "input"
|
315
321
|
type = "input"
|
316
322
|
display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
|
317
323
|
display "#{"fields".rjust(rjust)}: #{node['fields'].to_s.colorize(color)}"
|
318
324
|
display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
|
319
|
-
elsif node['type'] == "
|
325
|
+
elsif node['type'] == "output"
|
320
326
|
type = "output"
|
321
327
|
display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
|
322
328
|
display "#{"columns".rjust(rjust)}: #{node['columns'].to_s.colorize(color)}"
|
@@ -3,6 +3,7 @@
|
|
3
3
|
module Zillabyte; module Runner; class MultilangOperation
|
4
4
|
|
5
5
|
HANDSHAKE_MESSAGE = "{\"pidDir\": \"/tmp\"}\n"
|
6
|
+
PREPARE_MESSAGE = " {\"command\": \"prepare\"}\n"
|
6
7
|
DONE_MESSAGE = "{\"command\": \"done\"}\n"
|
7
8
|
NEXT_MESSAGE = "{\"command\": \"next\"}\n"
|
8
9
|
BEGIN_CYCLE_MESSAGE = "{\"command\": \"begin_cycle\"}\n"
|
@@ -77,9 +78,8 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
77
78
|
|
78
79
|
msg = @__consumee_pipes["rd_child_1"].gets
|
79
80
|
|
80
|
-
if msg ==
|
81
|
-
send_to_consumers(
|
82
|
-
send_to_consumers(DONE_MESSAGE)
|
81
|
+
if msg == KILL_CYCLE_MESSAGE
|
82
|
+
send_to_consumers(KILL_CYCLE_MESSAGE)
|
83
83
|
return
|
84
84
|
else
|
85
85
|
# Build tuple
|
@@ -193,217 +193,211 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
193
193
|
# Kill the cycle on error
|
194
194
|
cycle_killed = false
|
195
195
|
|
196
|
-
# Setup multilang
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
196
|
+
# Setup multilang socket
|
197
|
+
require 'socket'
|
198
|
+
host = "0.0.0.0"
|
199
|
+
server = TCPServer.new(0)
|
200
|
+
port = server.addr[1]
|
201
|
+
|
204
202
|
# Spawn multilang process
|
205
|
-
cmd = command("--execute_live --name #{@__name} --
|
203
|
+
cmd = command("--execute_live --name #{@__name} --host \"#{host}\" --port #{port}")
|
204
|
+
|
206
205
|
begin
|
207
206
|
|
208
|
-
Open3.popen3(cmd) do |
|
207
|
+
Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
|
209
208
|
begin
|
209
|
+
server_thread = Thread.new do
|
210
|
+
ml_socket = server.accept()
|
210
211
|
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
read_streams = consumer_hash.keys.concat [stdout, ml_output]
|
216
|
-
|
217
|
-
|
218
|
-
# Handshake
|
219
|
-
handshake(ml_input, ml_output)
|
220
|
-
|
221
|
-
# Begin cycle
|
222
|
-
begin_cycle(ml_input, ml_output)
|
223
|
-
emitted = false
|
224
|
-
write_message(ml_input, NEXT_MESSAGE)
|
225
|
-
multilang_queue = []
|
226
|
-
end_cycle_policy = @__options[:end_cycle_policy]
|
227
|
-
end_cycle_received = false
|
228
|
-
|
229
|
-
# Receive and handle messages
|
230
|
-
loop do
|
231
|
-
|
232
|
-
# Read from a stream
|
233
|
-
rs = select_read_streams(read_streams)
|
234
|
-
rs.each do |r|
|
235
|
-
|
236
|
-
# Read stdout straight to user
|
237
|
-
if r == stdout && consumers_running > 0
|
238
|
-
msg = r.gets
|
239
|
-
msg = msg.sub(/\n/, "")
|
240
|
-
cdisplay("log: #{msg}")
|
241
|
-
next
|
242
|
-
end
|
243
|
-
|
244
|
-
obj = read_message(r)
|
245
|
-
|
246
|
-
if obj.nil?
|
247
|
-
next
|
248
|
-
end
|
212
|
+
# RUN SOURCE
|
213
|
+
begin
|
214
|
+
# Setup streams from consumers and multilang(stdout and socket communication)
|
215
|
+
read_streams = consumer_hash.keys.concat [stdout, ml_socket]
|
249
216
|
|
250
|
-
|
217
|
+
# Handshake
|
218
|
+
handshake(ml_socket, ml_socket)
|
219
|
+
prepare(ml_socket, ml_socket)
|
251
220
|
|
252
|
-
|
221
|
+
# Begin cycle
|
222
|
+
begin_cycle(ml_socket, ml_socket)
|
223
|
+
emitted = false
|
224
|
+
write_message(ml_socket, NEXT_MESSAGE)
|
225
|
+
multilang_queue = []
|
226
|
+
end_cycle_policy = @__options[:end_cycle_policy]
|
227
|
+
end_cycle_received = false
|
228
|
+
|
229
|
+
# Receive and handle messages
|
230
|
+
loop do
|
231
|
+
# Read from a stream
|
232
|
+
rs = select_read_streams(read_streams)
|
233
|
+
rs.each do |r|
|
234
|
+
# Read stdout straight to user
|
235
|
+
if r == stdout && consumers_running > 0
|
236
|
+
msg = r.gets
|
237
|
+
msg = msg.sub(/\n/, "")
|
238
|
+
cdisplay("log: #{msg}")
|
239
|
+
next
|
240
|
+
end
|
253
241
|
|
254
|
-
|
255
|
-
when "emit"
|
256
|
-
stream = obj['stream']
|
257
|
-
# Check for null emit
|
258
|
-
if end_cycle_policy != "explicit"
|
242
|
+
obj = read_message(r)
|
259
243
|
|
260
|
-
|
261
|
-
|
262
|
-
else
|
263
|
-
nil_values = false
|
264
|
-
obj['tuple'].each_value do |v|
|
265
|
-
if v.nil?
|
266
|
-
nil_values = true
|
267
|
-
break
|
268
|
-
end
|
269
|
-
end
|
270
|
-
end_cycle_received = nil_values
|
271
|
-
next unless !end_cycle_received
|
272
|
-
end
|
244
|
+
if obj.nil?
|
245
|
+
next
|
273
246
|
end
|
274
247
|
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
248
|
+
if obj["command"]
|
249
|
+
case obj["command"]
|
250
|
+
|
251
|
+
# Multilang emitted a tuple
|
252
|
+
when "emit"
|
253
|
+
stream = obj['stream']
|
254
|
+
# Check for null emit
|
255
|
+
if end_cycle_policy != "explicit"
|
256
|
+
|
257
|
+
if obj['tuple'].nil?
|
258
|
+
end_cycle_received = true
|
259
|
+
else
|
260
|
+
nil_values = false
|
261
|
+
obj['tuple'].each_value do |v|
|
262
|
+
if v.nil?
|
263
|
+
nil_values = true
|
264
|
+
break
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end_cycle_received = nil_values
|
268
|
+
next unless !end_cycle_received
|
269
|
+
end
|
270
|
+
end
|
280
271
|
|
281
|
-
|
282
|
-
|
283
|
-
emit_consumer_tuple(stream, consumer, tuple_json)
|
272
|
+
# Valid emit
|
273
|
+
emitted = true
|
284
274
|
|
285
|
-
|
286
|
-
|
287
|
-
end
|
288
|
-
end
|
275
|
+
# Send or enqueue tuple for each consumer
|
276
|
+
tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
|
289
277
|
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
consumer = consumer_hash[r][:consumer]
|
294
|
-
@__emit_queues[stream][consumer][:ready] = true
|
295
|
-
tuple_json = get_consumer_tuple(stream, consumer)
|
278
|
+
@__emit_queues[stream].each_pair do |consumer, emitter|
|
279
|
+
if emitter[:ready]
|
280
|
+
emit_consumer_tuple(stream, consumer, tuple_json)
|
296
281
|
|
297
|
-
|
298
|
-
|
282
|
+
else
|
283
|
+
@__emit_queues[stream][consumer][:write_queue] << tuple_json
|
284
|
+
end
|
285
|
+
end
|
299
286
|
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
287
|
+
# Consumer is ready for a message
|
288
|
+
when "next"
|
289
|
+
stream = consumer_hash[r][:stream]
|
290
|
+
consumer = consumer_hash[r][:consumer]
|
291
|
+
@__emit_queues[stream][consumer][:ready] = true
|
292
|
+
tuple_json = get_consumer_tuple(stream, consumer)
|
305
293
|
|
306
|
-
|
307
|
-
|
308
|
-
# Emit tuple to consumer
|
309
|
-
emit_consumer_tuple(stream, consumer, tuple_json)
|
310
|
-
emitted = true
|
311
|
-
end
|
294
|
+
# End cycle for consumer if it has processed all tuples
|
295
|
+
if tuple_json.nil? && end_cycle_received
|
312
296
|
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
else
|
319
|
-
emitted = false
|
320
|
-
end
|
321
|
-
|
322
|
-
# Send the next tuple to multilang
|
323
|
-
if !multilang_queue.empty?
|
324
|
-
write_message(ml_input, multilang_queue.shift)
|
297
|
+
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
298
|
+
consumers_running -= 1
|
299
|
+
if consumers_running == 0
|
300
|
+
exit(0)
|
301
|
+
end
|
325
302
|
|
326
|
-
|
327
|
-
|
328
|
-
|
303
|
+
# TODO break if last consumer
|
304
|
+
elsif !tuple_json.nil?
|
305
|
+
# Emit tuple to consumer
|
306
|
+
emit_consumer_tuple(stream, consumer, tuple_json)
|
307
|
+
emitted = true
|
308
|
+
end
|
329
309
|
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
310
|
+
# Multilang is done emitting a group of tuples
|
311
|
+
when "done"
|
312
|
+
# End cycle if no tuples were emitted
|
313
|
+
if !emitted && end_cycle_policy == "null_emit"
|
314
|
+
end_cycle_received = true
|
315
|
+
else
|
316
|
+
emitted = false
|
317
|
+
end
|
318
|
+
|
319
|
+
# Send the next tuple to multilang
|
320
|
+
if !multilang_queue.empty?
|
321
|
+
write_message(ml_socket, multilang_queue.shift)
|
322
|
+
|
323
|
+
# Request next tuple from mutilang
|
324
|
+
elsif !end_cycle_received
|
325
|
+
write_message(ml_socket, NEXT_MESSAGE)
|
326
|
+
|
327
|
+
# If there are no more messages to send, we are done
|
328
|
+
else end_cycle_received
|
329
|
+
finished = true
|
330
|
+
# End cycle for ready consumers
|
331
|
+
@__emit_queues.each_pair do |stream, consumers|
|
332
|
+
consumers.each_pair do |consumer, emitter|
|
333
|
+
if emitter[:ready]
|
334
|
+
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
335
|
+
consumers_running -= 1
|
336
|
+
if consumers_running == 0
|
337
|
+
exit(0)
|
338
|
+
end
|
339
|
+
end
|
341
340
|
end
|
341
|
+
|
342
342
|
end
|
343
343
|
end
|
344
344
|
|
345
|
+
# Multilang sent an error message
|
346
|
+
when "fail"
|
347
|
+
cdisplay("ERROR : #{obj['msg']}")
|
348
|
+
cycle_killed = true
|
349
|
+
exit(0)
|
350
|
+
|
351
|
+
# Multilang sent a log message
|
352
|
+
when "log"
|
353
|
+
cdisplay "LOG: #{obj['msg']}"
|
354
|
+
|
355
|
+
# Multilang sent signal to end the cycle
|
356
|
+
when "end_cycle"
|
357
|
+
if end_cycle_policy != "explicit"
|
358
|
+
cdisplay "received end_cycle command for non explicit policy"
|
359
|
+
next
|
360
|
+
end
|
361
|
+
end_cycle_received = true
|
362
|
+
when "kill_cycle"
|
363
|
+
cycle_killed = true
|
364
|
+
exit(0)
|
345
365
|
end
|
346
|
-
end
|
347
|
-
|
348
|
-
# Multilang sent an error message
|
349
|
-
when "fail"
|
350
|
-
cdisplay("ERROR : #{obj['msg']}")
|
351
366
|
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
# Multilang sent signal to end the cycle
|
357
|
-
when "end_cycle"
|
358
|
-
if end_cycle_policy != "explicit"
|
359
|
-
cdisplay "received end_cycle command for non explicit policy"
|
360
|
-
next
|
367
|
+
# Multilang sent a ping
|
368
|
+
elsif obj['ping']
|
369
|
+
write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
|
361
370
|
end
|
362
|
-
end_cycle_received = true
|
363
|
-
when "kill_cycle"
|
364
|
-
cycle_killed = true
|
365
|
-
return
|
366
371
|
end
|
367
|
-
|
368
|
-
# Multilang sent a ping
|
369
|
-
elsif obj['ping']
|
370
|
-
write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
|
371
372
|
end
|
372
|
-
end
|
373
373
|
|
374
|
-
|
375
|
-
|
376
|
-
|
374
|
+
# Exit after ending consumer cycles
|
375
|
+
if consumers_running == 0
|
376
|
+
exit(0)
|
377
|
+
end
|
378
|
+
rescue => e
|
379
|
+
cycle_killed = true
|
380
|
+
cdisplay e.message
|
381
|
+
cdisplay e.backtrace
|
382
|
+
ensure
|
383
|
+
ml_socket.close()
|
377
384
|
end
|
378
|
-
|
379
385
|
end
|
386
|
+
server_thread.join()
|
380
387
|
rescue => e
|
381
|
-
cycle_killed = true
|
382
|
-
cdisplay e.message
|
383
|
-
cdisplay e.backtrace
|
384
388
|
ensure
|
385
|
-
|
386
389
|
# cleanup
|
387
390
|
if cycle_killed
|
388
391
|
send_to_consumers(KILL_CYCLE_MESSAGE, false)
|
389
392
|
end
|
390
393
|
pid = wait_thread[:pid]
|
391
|
-
ml_input.close
|
392
|
-
if File.exists?("#{ml_pipe}.in")
|
393
|
-
ml_output.close
|
394
|
-
File.delete("#{ml_pipe}.in")
|
395
|
-
end
|
396
394
|
stdout.close
|
397
|
-
stderr.close
|
398
395
|
Process.kill('INT', pid)
|
399
396
|
Process.exit!(true)
|
400
397
|
end
|
401
398
|
end
|
402
399
|
|
403
400
|
rescue PTY::ChildExited
|
404
|
-
if File.exists?("#{ml_pipe}.in")
|
405
|
-
File.delete("#{ml_pipe}.in")
|
406
|
-
end
|
407
401
|
end
|
408
402
|
end
|
409
403
|
end
|
@@ -411,7 +405,6 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
411
405
|
|
412
406
|
def self.run_each()
|
413
407
|
|
414
|
-
|
415
408
|
# Index streams and consumers by their pipes for lookup
|
416
409
|
consumer_hash = build_consumer_hash()
|
417
410
|
|
@@ -421,182 +414,187 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
421
414
|
# Kill the cycle on error
|
422
415
|
cycle_killed = false
|
423
416
|
|
424
|
-
# Setup multilang
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
cmd = command("--execute_live --name #{@__name} --
|
417
|
+
# Setup multilang socket
|
418
|
+
require 'socket'
|
419
|
+
host = "0.0.0.0"
|
420
|
+
server = TCPServer.new(0)
|
421
|
+
port = server.addr[1]
|
422
|
+
|
423
|
+
# Spawn multilang process
|
424
|
+
cmd = command("--execute_live --name #{@__name} --host \"#{host}\" --port #{port}")
|
425
|
+
|
432
426
|
begin
|
433
427
|
# Start the operation...
|
434
|
-
Open3.popen3(cmd) do |
|
428
|
+
Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
|
435
429
|
begin
|
430
|
+
server_thread = Thread.new do
|
431
|
+
ml_socket = server.accept()
|
432
|
+
begin
|
433
|
+
# Setup streams from consumers, multilang, and the consumee
|
434
|
+
read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"], ml_socket, stdout]
|
436
435
|
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
# Setup streams from consumers, multilang, and the consumee
|
441
|
-
read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"], ml_output, stdout]
|
442
|
-
|
443
|
-
# Handshake
|
444
|
-
handshake(ml_input, ml_output)
|
436
|
+
# Handshake
|
437
|
+
handshake(ml_socket, ml_socket)
|
438
|
+
prepare(ml_socket, ml_socket)
|
445
439
|
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
440
|
+
# Begin cycle
|
441
|
+
multilang_queue = []
|
442
|
+
mutlilang_count = 0
|
443
|
+
end_cycle_received = false
|
444
|
+
column_aliases = nil
|
451
445
|
|
452
446
|
|
453
|
-
|
454
|
-
|
447
|
+
# Receive and handle messages
|
448
|
+
loop do
|
455
449
|
|
456
|
-
|
457
|
-
|
458
|
-
|
450
|
+
# Read from a stream
|
451
|
+
rs = select_read_streams(read_streams)
|
452
|
+
rs.each do |r|
|
459
453
|
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
454
|
+
# Read STDOUT from program straight to user
|
455
|
+
if r == stdout
|
456
|
+
msg = r.gets
|
457
|
+
if !msg.nil?
|
458
|
+
msg = msg.sub(/\n/, "")
|
459
|
+
cdisplay("LOG: #{msg}")
|
460
|
+
end
|
461
|
+
next
|
462
|
+
end
|
469
463
|
|
470
|
-
|
471
|
-
|
464
|
+
# Receive an object
|
465
|
+
obj = read_message(r)
|
472
466
|
|
473
|
-
|
474
|
-
|
467
|
+
if obj["command"]
|
468
|
+
case obj["command"]
|
475
469
|
|
476
|
-
|
477
|
-
|
470
|
+
# Multilang emitted a tuple
|
471
|
+
when "emit"
|
478
472
|
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
473
|
+
stream = obj["stream"]
|
474
|
+
|
475
|
+
# Send or enqueue tuple for each consumer
|
476
|
+
tuple_json = build_tuple_json(obj['tuple'], obj['meta'], column_aliases)
|
483
477
|
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
# Consumer is ready for a message
|
493
|
-
when "next"
|
494
|
-
stream = consumer_hash[r][:stream]
|
495
|
-
consumer = consumer_hash[r][:consumer]
|
478
|
+
@__emit_queues[stream].each_pair do |consumer, emitter|
|
479
|
+
if emitter[:ready]
|
480
|
+
emit_consumer_tuple(stream, consumer, tuple_json)
|
481
|
+
else
|
482
|
+
@__emit_queues[stream][consumer][:write_queue] << tuple_json
|
483
|
+
end
|
484
|
+
end
|
496
485
|
|
486
|
+
# Consumer is ready for a message
|
487
|
+
when "next"
|
488
|
+
stream = consumer_hash[r][:stream]
|
489
|
+
consumer = consumer_hash[r][:consumer]
|
497
490
|
|
498
|
-
@__emit_queues[stream][consumer][:ready] = true
|
499
|
-
tuple_json = get_consumer_tuple(stream, consumer)
|
500
491
|
|
501
|
-
|
502
|
-
|
503
|
-
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
504
|
-
consumers_running -= 1
|
505
|
-
if consumers_running == 0
|
506
|
-
break
|
507
|
-
end
|
492
|
+
@__emit_queues[stream][consumer][:ready] = true
|
493
|
+
tuple_json = get_consumer_tuple(stream, consumer)
|
508
494
|
|
495
|
+
# End cycle for consumer if it has processed all tuples
|
496
|
+
if tuple_json.nil? && end_cycle_received
|
497
|
+
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
498
|
+
consumers_running -= 1
|
499
|
+
if consumers_running == 0
|
500
|
+
break
|
501
|
+
end
|
509
502
|
|
510
|
-
# TODO break if last consumer
|
511
|
-
elsif !tuple_json.nil?
|
512
|
-
# Emit tuple to consumer
|
513
|
-
emit_consumer_tuple(stream, consumer, tuple_json)
|
514
|
-
emitted = true
|
515
|
-
end
|
516
503
|
|
517
|
-
|
518
|
-
|
519
|
-
|
504
|
+
# TODO break if last consumer
|
505
|
+
elsif !tuple_json.nil?
|
506
|
+
# Emit tuple to consumer
|
507
|
+
emit_consumer_tuple(stream, consumer, tuple_json)
|
508
|
+
emitted = true
|
509
|
+
end
|
520
510
|
|
521
|
-
|
522
|
-
|
523
|
-
|
511
|
+
# Multilang is done emitting a group of tuples
|
512
|
+
when "done"
|
513
|
+
mutlilang_count -= 1
|
524
514
|
|
525
|
-
|
526
|
-
|
527
|
-
|
515
|
+
# Send the next tuple to multilang
|
516
|
+
if !multilang_queue.empty?
|
517
|
+
write_message(ml_socket, multilang_queue.shift)
|
528
518
|
|
529
|
-
|
530
|
-
|
531
|
-
|
519
|
+
# If there are no more messages to send, we are done
|
520
|
+
elsif end_cycle_received && mutlilang_count == 0
|
521
|
+
finished = true
|
532
522
|
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
523
|
+
# End cycle for ready consumers
|
524
|
+
@__emit_queues.each_pair do |stream, consumers|
|
525
|
+
consumers.each_pair do |consumer, emitter|
|
526
|
+
if emitter[:ready]
|
527
|
+
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
528
|
+
consumers_running -= 1
|
529
|
+
if consumers_running == 0
|
530
|
+
break
|
531
|
+
end
|
532
|
+
end
|
541
533
|
end
|
542
534
|
end
|
543
535
|
end
|
544
|
-
end
|
545
|
-
end
|
546
536
|
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
537
|
+
# Multilang sent an error message
|
538
|
+
when "fail"
|
539
|
+
cdisplay("ERROR : #{obj['msg']}")
|
540
|
+
cycle_killed = true
|
541
|
+
exit(0)
|
542
|
+
|
543
|
+
# Multilang sent a log message
|
544
|
+
when "log"
|
545
|
+
cdisplay "LOG: #{obj['msg']}"
|
546
|
+
|
547
|
+
# Consumee operation sent signal to end_cycle
|
548
|
+
when "end_cycle"
|
549
|
+
end_cycle_received = true
|
550
|
+
if mutlilang_count == 0
|
551
|
+
@__emit_queues.each_pair do |stream, consumers|
|
552
|
+
consumers.each_pair do |consumer, emitter|
|
553
|
+
if emitter[:ready]
|
554
|
+
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
555
|
+
consumers_running -= 1
|
556
|
+
if consumers_running == 0
|
557
|
+
exit(0)
|
558
|
+
end
|
559
|
+
end
|
566
560
|
end
|
567
561
|
end
|
568
562
|
end
|
563
|
+
|
564
|
+
when "kill_cycle"
|
565
|
+
cycle_killed = true
|
566
|
+
exit(0)
|
569
567
|
end
|
568
|
+
|
569
|
+
# Received a tuple from consumee
|
570
|
+
elsif obj['tuple']
|
571
|
+
column_aliases = obj['column_aliases']
|
572
|
+
# Send or enqueue to multilang
|
573
|
+
mutlilang_count += 1
|
574
|
+
if multilang_queue.empty?
|
575
|
+
write_message(ml_socket, obj.to_json)
|
576
|
+
else
|
577
|
+
multilang_queue << obj.to_json
|
578
|
+
end
|
579
|
+
write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
|
580
|
+
|
581
|
+
# Multilang sent a ping
|
582
|
+
elsif obj['ping']
|
583
|
+
write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
|
570
584
|
end
|
571
|
-
|
572
|
-
when "kill_cycle"
|
573
|
-
cycle_killed = true
|
574
|
-
return
|
575
585
|
end
|
576
586
|
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
# Send or enqueue to multilang
|
581
|
-
mutlilang_count += 1
|
582
|
-
if multilang_queue.empty?
|
583
|
-
write_message(ml_input, obj.to_json)
|
584
|
-
else
|
585
|
-
multilang_queue << obj.to_json
|
587
|
+
# Exit after ending consumer cycles
|
588
|
+
if consumers_running == 0
|
589
|
+
exit(0)
|
586
590
|
end
|
587
|
-
|
588
|
-
# Multilang sent a ping
|
589
|
-
elsif obj['ping']
|
590
|
-
write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
|
591
|
-
end
|
592
|
-
end
|
593
591
|
|
594
|
-
|
595
|
-
|
596
|
-
|
592
|
+
end
|
593
|
+
ensure
|
594
|
+
ml_socket.close()
|
597
595
|
end
|
598
|
-
|
599
596
|
end
|
597
|
+
server_thread.join()
|
600
598
|
rescue => e
|
601
599
|
cdisplay e.message
|
602
600
|
cdisplay e.backtrace
|
@@ -604,28 +602,18 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
604
602
|
# cleanup
|
605
603
|
if cycle_killed
|
606
604
|
send_to_consumers(KILL_CYCLE_MESSAGE, false)
|
607
|
-
send_to_consumees(KILL_CYCLE_MESSAGE
|
605
|
+
send_to_consumees(KILL_CYCLE_MESSAGE)
|
608
606
|
end
|
609
607
|
pid = wait_thread[:pid]
|
610
|
-
|
611
|
-
if File.exists?("#{ml_pipe}.in")
|
612
|
-
ml_output.close
|
613
|
-
File.delete("#{ml_pipe}.in")
|
614
|
-
end
|
608
|
+
stdin.close
|
615
609
|
stdout.close
|
616
610
|
stderr.close
|
617
|
-
Process.kill('INT', pid)
|
618
|
-
Process.exit!(true)
|
619
611
|
end
|
620
612
|
end
|
621
613
|
rescue PTY::ChildExited
|
622
|
-
|
623
614
|
cdisplay("The child process exited!")
|
624
|
-
ensure
|
625
|
-
if File.exists?("#{ml_pipe}.in")
|
626
|
-
File.delete("#{ml_pipe}.in")
|
627
|
-
end
|
628
615
|
end
|
616
|
+
|
629
617
|
end
|
630
618
|
|
631
619
|
|
@@ -648,219 +636,223 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
648
636
|
tuple_queue = []
|
649
637
|
|
650
638
|
# Setup multilang pipe
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
cmd = command("--execute_live --name #{@__name} --
|
639
|
+
require 'socket'
|
640
|
+
host = "0.0.0.0"
|
641
|
+
server = TCPServer.new(0)
|
642
|
+
port = server.addr[1]
|
643
|
+
|
644
|
+
# Spawn multilang process
|
645
|
+
cmd = command("--execute_live --name #{@__name} --host \"#{host}\" --port #{port}")
|
646
|
+
|
658
647
|
begin
|
659
648
|
# Start the operation...
|
660
|
-
Open3.popen3(cmd) do |
|
649
|
+
Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
|
661
650
|
begin
|
651
|
+
server_thread = Thread.new do
|
652
|
+
ml_socket = server.accept()
|
653
|
+
begin
|
654
|
+
# Setup streams from consumers, multilang, and the consumee
|
655
|
+
read_streams = consumer_hash.keys.concat [stdout, ml_socket, @__consumee_pipes["rd_child_1"]]
|
662
656
|
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
read_streams = consumer_hash.keys.concat [stdout, ml_output, @__consumee_pipes["rd_child_1"]]
|
667
|
-
|
668
|
-
# Handshake
|
669
|
-
handshake(ml_input, ml_output)
|
657
|
+
# Handshake
|
658
|
+
handshake(ml_socket, ml_socket)
|
659
|
+
prepare(ml_socket, ml_socket)
|
670
660
|
|
671
|
-
|
672
|
-
|
673
|
-
|
661
|
+
# Begin cycle
|
662
|
+
end_cycle_received = false
|
663
|
+
finished_emitting = false
|
674
664
|
|
675
|
-
|
676
|
-
|
665
|
+
# Handle streams
|
666
|
+
loop do
|
667
|
+
# Read from a stream
|
668
|
+
rs = select_read_streams(read_streams)
|
669
|
+
rs.each do |r|
|
677
670
|
|
678
|
-
|
679
|
-
|
680
|
-
|
671
|
+
# Read STDOUT from program straight to user
|
672
|
+
if r == stdout
|
673
|
+
msg = r.gets
|
674
|
+
msg = msg.sub(/\n/, "")
|
675
|
+
cdisplay("log: #{msg}")
|
676
|
+
next
|
677
|
+
end
|
681
678
|
|
682
|
-
# Read STDOUT from program straight to user
|
683
|
-
if r == stdout
|
684
|
-
msg = r.gets
|
685
|
-
msg = msg.sub(/\n/, "")
|
686
|
-
cdisplay("log: #{msg}")
|
687
|
-
next
|
688
|
-
end
|
689
679
|
|
680
|
+
# Receive an object
|
681
|
+
obj = read_message(r)
|
682
|
+
|
683
|
+
if obj["command"]
|
684
|
+
case obj["command"]
|
690
685
|
|
691
|
-
|
692
|
-
obj = read_message(r)
|
693
|
-
|
694
|
-
if obj["command"]
|
695
|
-
case obj["command"]
|
686
|
+
when "done"
|
696
687
|
|
697
|
-
|
688
|
+
if end_cycle_received
|
689
|
+
tuple_json = tuple_queue.shift
|
690
|
+
if !tuple_json.nil?
|
691
|
+
write_message(ml_socket, tuple_json)
|
692
|
+
end
|
693
|
+
end
|
698
694
|
|
699
|
-
|
700
|
-
tuple_json = tuple_queue.shift
|
701
|
-
if !tuple_json.nil?
|
702
|
-
write_message(ml_input, tuple_json)
|
703
|
-
end
|
704
|
-
end
|
695
|
+
next
|
705
696
|
|
706
|
-
|
697
|
+
# Begin aggregation
|
698
|
+
when "end_cycle"
|
699
|
+
end_cycle_received = true
|
700
|
+
read_streams = [ml_socket]
|
707
701
|
|
708
|
-
|
709
|
-
|
710
|
-
|
711
|
-
|
702
|
+
group_tuples.each do |group_tuple, tuples|
|
703
|
+
tuple_queue << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
|
704
|
+
tuples.each do |t|
|
705
|
+
tuple_queue << "{\"command\": \"aggregate\", #{t}}\n"
|
706
|
+
end
|
707
|
+
tuple_queue << "{\"command\": \"end_group\"}\n"
|
712
708
|
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
tuple_queue << "{\"command\": \"aggregate\", #{t}}\n"
|
717
|
-
end
|
718
|
-
tuple_queue << "{\"command\": \"end_group\"}\n"
|
709
|
+
# keep track of how many emits are expected
|
710
|
+
emit_count += 1
|
711
|
+
end
|
719
712
|
|
720
|
-
|
721
|
-
|
722
|
-
|
713
|
+
tuple_json = tuple_queue.shift
|
714
|
+
if !tuple_json.nil?
|
715
|
+
write_message(ml_socket, tuple_json)
|
716
|
+
end
|
723
717
|
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
718
|
+
# Multilang has emitted a grouped tuple
|
719
|
+
when "emit"
|
720
|
+
stream = obj['stream']
|
721
|
+
emit_count -= 1
|
722
|
+
# Enqueue for consumers
|
723
|
+
tuple_json = build_tuple_json(obj['tuple'], obj['meta'], obj['column_aliases'])
|
724
|
+
@__emit_queues.each_pair do |stream, consumers|
|
725
|
+
consumers.each_key do |consumer|
|
726
|
+
@__emit_queues[stream][consumer][:write_queue] << tuple_json
|
727
|
+
end
|
728
|
+
end
|
728
729
|
|
729
|
-
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
|
734
|
-
|
735
|
-
|
736
|
-
|
737
|
-
|
730
|
+
# End cycle when done emitting
|
731
|
+
if end_cycle_received && emit_count == 0
|
732
|
+
finished_emitting = true
|
733
|
+
break
|
734
|
+
elsif end_cycle_received
|
735
|
+
tuple_json = tuple_queue.shift
|
736
|
+
if !tuple_json.nil?
|
737
|
+
write_message(ml_socket, tuple_json)
|
738
|
+
end
|
739
|
+
end
|
740
|
+
|
741
|
+
# An error has occured
|
742
|
+
when "kill_cycle"
|
743
|
+
cycle_killed = true
|
744
|
+
exit(0)
|
738
745
|
end
|
739
|
-
end
|
740
746
|
|
741
|
-
#
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
747
|
+
# Received a tuple from operation
|
748
|
+
elsif obj["tuple"]
|
749
|
+
tuple = obj["tuple"].to_json
|
750
|
+
meta = obj["meta"].to_json
|
751
|
+
column_aliases = obj["column_aliases"] || {}
|
752
|
+
aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
|
753
|
+
gt = {}
|
754
|
+
|
755
|
+
# Get the column names to group on
|
756
|
+
group_by.each do |field|
|
757
|
+
field_name = aliases[field] || field
|
758
|
+
gt[field] = obj["tuple"][field_name]
|
749
759
|
end
|
750
|
-
end
|
751
|
-
|
752
|
-
# An error has occured
|
753
|
-
when "kill_cycle"
|
754
|
-
cycle_killed = true
|
755
|
-
return
|
756
|
-
end
|
757
760
|
|
758
|
-
|
759
|
-
elsif obj["tuple"]
|
760
|
-
tuple = obj["tuple"].to_json
|
761
|
-
meta = obj["meta"].to_json
|
762
|
-
column_aliases = obj["column_aliases"] || {}
|
763
|
-
aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
|
764
|
-
gt = {}
|
765
|
-
|
766
|
-
# Get the column names to group on
|
767
|
-
group_by.each do |field|
|
768
|
-
field_name = aliases[field] || field
|
769
|
-
gt[field] = obj["tuple"][field_name]
|
770
|
-
end
|
761
|
+
msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
|
771
762
|
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
group_tuples[gt] = [msg_no_brackets]
|
779
|
-
end
|
763
|
+
# Group tuple into existing group or create new group
|
764
|
+
if group_tuples[gt]
|
765
|
+
group_tuples[gt] << msg_no_brackets
|
766
|
+
else
|
767
|
+
group_tuples[gt] = [msg_no_brackets]
|
768
|
+
end
|
780
769
|
|
781
|
-
|
782
|
-
|
770
|
+
# Ask operation for next tuple
|
771
|
+
write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
|
783
772
|
|
784
|
-
|
785
|
-
|
786
|
-
|
787
|
-
|
788
|
-
|
773
|
+
# Multilang sent a ping
|
774
|
+
elsif obj['ping']
|
775
|
+
write_message(ml_socket, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
|
776
|
+
end
|
777
|
+
end
|
789
778
|
|
790
|
-
|
791
|
-
|
779
|
+
# Send tuples to consumers
|
780
|
+
if finished_emitting && consumers_running > 0
|
792
781
|
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
782
|
+
# Send first tuple
|
783
|
+
@__emit_queues.each_pair do |stream, consumers|
|
784
|
+
consumers.each_key do |consumer|
|
785
|
+
tuple_json = get_consumer_tuple(stream, consumer)
|
786
|
+
emit_consumer_tuple(stream, consumer, tuple_json)
|
787
|
+
end
|
788
|
+
end
|
800
789
|
|
801
|
-
|
802
|
-
|
790
|
+
# Sent tuples to consumers as appropriate
|
791
|
+
loop do
|
803
792
|
|
804
|
-
|
805
|
-
|
793
|
+
# Retrieve messages from consumers
|
794
|
+
rs, ws, es = IO.select(consumer_hash.keys, [], [])
|
806
795
|
|
807
|
-
|
808
|
-
|
809
|
-
|
796
|
+
# Emit tuples to consumers
|
797
|
+
emitted = false
|
798
|
+
rs.each do |r|
|
810
799
|
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
|
800
|
+
# Read from consumer
|
801
|
+
msg = read_message(r)
|
802
|
+
consumer = consumer_hash[r][:consumer]
|
803
|
+
stream = consumer_hash[r][:stream]
|
815
804
|
|
816
|
-
|
817
|
-
|
805
|
+
# Consumer is ready for next message
|
806
|
+
if msg["command"] && msg["command"] == "next"
|
818
807
|
|
819
|
-
|
820
|
-
|
808
|
+
@__emit_queues[stream][consumer][:ready] = true
|
809
|
+
tuple_json = get_consumer_tuple(stream, consumer)
|
821
810
|
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
811
|
+
# If all messages have been sent to a consumer, end its cycle
|
812
|
+
if tuple_json.nil?
|
813
|
+
write_stream = get_write_stream(stream, consumer)
|
814
|
+
write_message(write_stream, END_CYCLE_MESSAGE)
|
815
|
+
consumers_running -= 1
|
816
|
+
if consumers_running == 0
|
817
|
+
break
|
818
|
+
end
|
819
|
+
else
|
820
|
+
# Emit tuple to consumer
|
821
|
+
emit_consumer_tuple(stream, consumer, tuple_json)
|
822
|
+
emitted = true
|
823
|
+
end
|
829
824
|
end
|
830
|
-
|
831
|
-
|
832
|
-
|
833
|
-
|
825
|
+
|
826
|
+
end
|
827
|
+
# Exit when done emitting
|
828
|
+
if !emitted
|
829
|
+
exit(0)
|
834
830
|
end
|
835
831
|
end
|
832
|
+
break
|
836
833
|
|
837
|
-
|
838
|
-
|
839
|
-
|
840
|
-
|
841
|
-
end
|
834
|
+
# Exit after ending all consumer cycles
|
835
|
+
elsif consumers_running == 0
|
836
|
+
break
|
837
|
+
end
|
842
838
|
end
|
843
|
-
|
844
|
-
|
845
|
-
# Exit after ending all consumer cycles
|
846
|
-
elsif consumers_running == 0
|
847
|
-
break
|
839
|
+
ensure
|
840
|
+
ml_socket.close()
|
848
841
|
end
|
849
|
-
|
850
842
|
end
|
843
|
+
server_thread.join()
|
844
|
+
|
851
845
|
rescue Errno::EIO
|
852
846
|
cdisplay("Errno:EIO error")
|
853
847
|
ensure
|
854
848
|
# cleanup
|
855
849
|
if cycle_killed
|
856
850
|
send_to_consumers(KILL_CYCLE_MESSAGE, false)
|
857
|
-
|
851
|
+
send_to_consumees(KILL_CYCLE_MESSAGE)
|
858
852
|
end
|
859
853
|
|
860
854
|
pid = wait_thread[:pid]
|
861
|
-
|
862
|
-
ml_output.close
|
863
|
-
File.delete("#{ml_pipe}.in")
|
855
|
+
stdin.close
|
864
856
|
stdout.close
|
865
857
|
stderr.close
|
866
858
|
Process.kill('INT', pid)
|
@@ -1226,7 +1218,9 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
1226
1218
|
"string" => String,
|
1227
1219
|
"double" => Float,
|
1228
1220
|
"integer" => Integer,
|
1229
|
-
"float" => Float
|
1221
|
+
"float" => Float,
|
1222
|
+
"array" => Array,
|
1223
|
+
"map" => Hash
|
1230
1224
|
}
|
1231
1225
|
|
1232
1226
|
col_map = {}
|
@@ -1449,6 +1443,18 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
1449
1443
|
end
|
1450
1444
|
|
1451
1445
|
|
1446
|
+
# Instruct multilang to run prepare step
|
1447
|
+
def self.prepare(write_stream, read_stream)
|
1448
|
+
begin
|
1449
|
+
write_message write_stream, PREPARE_MESSAGE
|
1450
|
+
msg = read_message(read_stream)
|
1451
|
+
rescue => e
|
1452
|
+
cdisplay("Error running prepare")
|
1453
|
+
raise e
|
1454
|
+
end
|
1455
|
+
end
|
1456
|
+
|
1457
|
+
|
1452
1458
|
# Instruct multilang to begin cycle
|
1453
1459
|
def self.begin_cycle(write_stream, read_stream)
|
1454
1460
|
write_message(write_stream, BEGIN_CYCLE_MESSAGE)
|
@@ -1495,14 +1501,14 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
1495
1501
|
def self.send_to_consumees(json_obj)
|
1496
1502
|
pipes = @__consumee_pipes
|
1497
1503
|
# Left hand(or singular) input
|
1498
|
-
if (pipes.has_key? "
|
1499
|
-
write_stream =
|
1504
|
+
if (pipes.has_key? "wr_child_1")
|
1505
|
+
write_stream = pipes["wr_child_1"]
|
1500
1506
|
write_message(write_stream, json_obj)
|
1501
1507
|
end
|
1502
1508
|
|
1503
1509
|
# Right hand input
|
1504
|
-
if (pipes.has_key? "
|
1505
|
-
write_stream =
|
1510
|
+
if (pipes.has_key? "wr_child_2")
|
1511
|
+
write_stream = pipes["wr_child_2"]
|
1506
1512
|
write_message(write_stream, json_obj)
|
1507
1513
|
end
|
1508
1514
|
end
|
@@ -1511,7 +1517,6 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
1511
1517
|
def self.send_to_consumers(json_obj, display = false)
|
1512
1518
|
@__consumer_pipes.each_pair do |stream, consumers|
|
1513
1519
|
consumers.each_pair do |consumer, pipe|
|
1514
|
-
@__consumee_pipes["wr_child_1"]
|
1515
1520
|
# Single or Left hand pipe
|
1516
1521
|
if (pipe.has_key? "wr_parent_1")
|
1517
1522
|
write_stream = get_write_stream(stream, consumer, 1)
|
@@ -1626,7 +1631,6 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
1626
1631
|
# Construct a multilang command
|
1627
1632
|
def self.command(arg, ignore_stderr=false)
|
1628
1633
|
cdisplay("could not extract meta information. missing zillabyte.conf.yml?") if @__meta.nil?
|
1629
|
-
|
1630
1634
|
full_script = File.join(@__dir, @__meta["script"])
|
1631
1635
|
stderr_opt = "2> /dev/null" if ignore_stderr
|
1632
1636
|
|