zillabyte-cli 0.9.20 → 0.9.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +13 -5
- data/lib/zillabyte-cli/version.rb +1 -1
- data/lib/zillabyte/api/components.rb +2 -2
- data/lib/zillabyte/cli/base.rb +15 -0
- data/lib/zillabyte/cli/data.rb +31 -1
- data/lib/zillabyte/cli/download.rb +67 -0
- data/lib/zillabyte/cli/flows.rb +91 -37
- data/lib/zillabyte/helpers.rb +12 -1
- metadata +28 -43
- data/lib/#zillabyte-cli.rb# +0 -5
- data/lib/zillabyte/cli/#logs.rb# +0 -12
- data/lib/zillabyte/cli/#repl.rb# +0 -43
- data/lib/zillabyte/cli/templates/python/#simple_function.py# +0 -27
- data/lib/zillabyte/runner.rb +0 -6
- data/lib/zillabyte/runner/app_runner.rb +0 -320
- data/lib/zillabyte/runner/component_operation.rb +0 -636
- data/lib/zillabyte/runner/component_runner.rb +0 -337
- data/lib/zillabyte/runner/multilang_operation.rb +0 -1662
- data/lib/zillabyte/runner/operation.rb +0 -18
data/lib/#zillabyte-cli.rb#
DELETED
data/lib/zillabyte/cli/#logs.rb#
DELETED
data/lib/zillabyte/cli/#repl.rb#
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
require "zillabyte/cli/base"
|
2
|
-
require 'readline'
|
3
|
-
# REPL console for zillabyte commands
|
4
|
-
#
|
5
|
-
class Zillabyte::Command::Repl < Zillabyte::Command::Base
|
6
|
-
|
7
|
-
# repl
|
8
|
-
#
|
9
|
-
# start a console session for zillabyte
|
10
|
-
# --quiet # HIDDEN
|
11
|
-
# --history HISTORY# HIDDEN hack to allow history for readline
|
12
|
-
def index
|
13
|
-
if !options[:quiet]
|
14
|
-
v = `zillabyte version`
|
15
|
-
display "\n#{v}Type q,exit or Ctrl+D to quit\n\n"
|
16
|
-
end
|
17
|
-
server = `echo $ZILLABYTE_API_HOST` || ""
|
18
|
-
prompt = ""
|
19
|
-
if server && server.chomp.length > 0
|
20
|
-
prompt = "#{server.chomp} "
|
21
|
-
end
|
22
|
-
prompt += "zillabyte $ "
|
23
|
-
if options[:history]
|
24
|
-
#p options[:history]
|
25
|
-
history = JSON.parse(options[:history])
|
26
|
-
history.last(50).each do |his|
|
27
|
-
# TODO: Handle single quotes ??
|
28
|
-
Readline::HISTORY << his
|
29
|
-
end
|
30
|
-
end
|
31
|
-
# TODO: Add tab completion for basic commands, app/relation names etc.
|
32
|
-
while cmd = Readline.readline(prompt, true)
|
33
|
-
if cmd && cmd.length > 0
|
34
|
-
if cmd.downcase == "exit" || cmd.downcase == "q"
|
35
|
-
display "" # TODO Make Ctrl+D print a newline too
|
36
|
-
return
|
37
|
-
else
|
38
|
-
exec "zillabyte #{cmd}; zillabyte repl --quiet --history '#{Readline::HISTORY.to_a.to_json}'"
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
@@ -1,27 +0,0 @@
|
|
1
|
-
import zillabyte
|
2
|
-
|
3
|
-
def prep(controller):
|
4
|
-
return
|
5
|
-
|
6
|
-
# This is the heart of your algorithm. It's processed on every
|
7
|
-
# web page. This algorithm is run in parallel on possibly hundreds
|
8
|
-
# of machines.
|
9
|
-
def execute(controller, tup):
|
10
|
-
if("hello world" in tup.values["html"]):
|
11
|
-
controller.emit("has_hello_world",{"url":tup.values["url"]})
|
12
|
-
return
|
13
|
-
|
14
|
-
zillabyte.simple_function(\
|
15
|
-
# This directive instructs zillabyte to give your function every
|
16
|
-
# web page in our known universe. Your function will have access
|
17
|
-
# to two fields: URL and HTML
|
18
|
-
matches = "select * from web_pa", \
|
19
|
-
|
20
|
-
# This directive tells Zillabyte what kind of data your function
|
21
|
-
# produces. In this case, we're saying we will emit a tuple that
|
22
|
-
# is one-column wide and contains the field 'URL'
|
23
|
-
emits = [["has_hello_world", [{"url":"string"}]]], \
|
24
|
-
|
25
|
-
prepare = prep, \
|
26
|
-
execute = execute\
|
27
|
-
)
|
data/lib/zillabyte/runner.rb
DELETED
@@ -1,320 +0,0 @@
|
|
1
|
-
require "zillabyte/runner/multilang_operation"
|
2
|
-
|
3
|
-
# HIDDEN:
|
4
|
-
class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
|
5
|
-
include Zillabyte::Helpers
|
6
|
-
|
7
|
-
END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
|
8
|
-
|
9
|
-
def run (meta, dir = Dir.pwd, session = nil, options = {})
|
10
|
-
|
11
|
-
if meta.nil? or session.nil?
|
12
|
-
return
|
13
|
-
end
|
14
|
-
|
15
|
-
@session = session
|
16
|
-
|
17
|
-
@colors = {}
|
18
|
-
output = options[:output]
|
19
|
-
otype = options[:output_type]
|
20
|
-
interactive = options[:interactive]
|
21
|
-
cycles = (options[:cycles] || "1").to_i
|
22
|
-
|
23
|
-
# Show the user what we know about their app...
|
24
|
-
display "inferring your app details..."
|
25
|
-
describe_app(meta)
|
26
|
-
|
27
|
-
# Setup nodes and arcs
|
28
|
-
@nodes = meta["nodes"]
|
29
|
-
@node_map = {}
|
30
|
-
@nodes.each do |n|
|
31
|
-
@node_map[n["name"]] = n
|
32
|
-
end
|
33
|
-
|
34
|
-
@arcs = meta["arcs"]
|
35
|
-
|
36
|
-
# Organize component pipes
|
37
|
-
@operations = {}
|
38
|
-
@operation_pipes = {}
|
39
|
-
|
40
|
-
# On each cycle, setup and tear down a test harness
|
41
|
-
(1..cycles).each do |cycle|
|
42
|
-
|
43
|
-
display "starting cycle #{cycle}" unless cycles == 1
|
44
|
-
|
45
|
-
begin
|
46
|
-
# Setup component pipes
|
47
|
-
@nodes.each do |n|
|
48
|
-
|
49
|
-
name = n["name"]
|
50
|
-
type = n["type"]
|
51
|
-
emits = n["emits"]
|
52
|
-
if n["type"] == "source"
|
53
|
-
options[:end_cycle_policy] = n["end_cycle_policy"]
|
54
|
-
end
|
55
|
-
|
56
|
-
# Create two new pipes in the parent.
|
57
|
-
rd_child_1, wr_parent_1 = IO.pipe()
|
58
|
-
rd_parent_1, wr_child_1 = IO.pipe()
|
59
|
-
|
60
|
-
@operation_pipes[name] = {
|
61
|
-
"rd_child_1" => rd_child_1,
|
62
|
-
"wr_child_1" => wr_child_1,
|
63
|
-
"rd_parent_1" => rd_parent_1,
|
64
|
-
"wr_parent_1" => wr_parent_1
|
65
|
-
}
|
66
|
-
|
67
|
-
|
68
|
-
# Add a second(right hand side) set ofpipes for joins
|
69
|
-
if type == "join"
|
70
|
-
# Create two new pipes in the parent.
|
71
|
-
rd_child_2, wr_parent_2 = IO.pipe()
|
72
|
-
rd_parent_2, wr_child_2 = IO.pipe()
|
73
|
-
@operation_pipes[name]["rd_child_2"] = rd_child_2
|
74
|
-
@operation_pipes[name]["wr_child_2"] = wr_child_2
|
75
|
-
@operation_pipes[name]["rd_parent_2"] = rd_parent_2
|
76
|
-
@operation_pipes[name]["wr_parent_2"] = wr_parent_2
|
77
|
-
end
|
78
|
-
|
79
|
-
|
80
|
-
end
|
81
|
-
|
82
|
-
# Maps origin => {stream => [destinations]}
|
83
|
-
@arc_map = {}
|
84
|
-
@arcs.each do |a|
|
85
|
-
origin = a["origin"]
|
86
|
-
name = a["name"]
|
87
|
-
dest = a["dest"]
|
88
|
-
@arc_map[origin] ||= {}
|
89
|
-
@arc_map[origin][name] ||= []
|
90
|
-
@arc_map[origin][name] << a["dest"]
|
91
|
-
end
|
92
|
-
|
93
|
-
# # Spawn component threads
|
94
|
-
@nodes.each do |n|
|
95
|
-
|
96
|
-
name = n["name"]
|
97
|
-
type = n["type"]
|
98
|
-
emits = n["emits"]
|
99
|
-
|
100
|
-
|
101
|
-
pipes = @operation_pipes[name]
|
102
|
-
|
103
|
-
# Fork.
|
104
|
-
pid = fork()
|
105
|
-
if pid # In parent
|
106
|
-
# Close the reading end of the first child so we can write to the child.
|
107
|
-
pipes["rd_child_1"].close()
|
108
|
-
# Close the writing end of the first child so we can read from the child.
|
109
|
-
pipes["wr_child_1"].close()
|
110
|
-
|
111
|
-
if type == "join"
|
112
|
-
# Close the reading end of the second child so we can write to the child.
|
113
|
-
pipes["rd_child_2"].close()
|
114
|
-
# Close the writing end of the second child so we can read from the child.
|
115
|
-
pipes["wr_child_2"].close()
|
116
|
-
end
|
117
|
-
else # in child
|
118
|
-
# Close the writing end of the first parent so we can read from the parent.
|
119
|
-
pipes["wr_parent_1"].close()
|
120
|
-
# Close the reading end of the first parent so we can write to the parent.
|
121
|
-
pipes["rd_parent_1"].close()
|
122
|
-
|
123
|
-
if type == "join"
|
124
|
-
# Close the reading end of the second child so we can write to the child.
|
125
|
-
pipes["rd_parent_2"].close()
|
126
|
-
# Close the writing end of the second child so we can read from the child.
|
127
|
-
pipes["wr_parent_2"].close()
|
128
|
-
end
|
129
|
-
|
130
|
-
|
131
|
-
begin
|
132
|
-
|
133
|
-
# Setup reading and writing pipes for communicating with consumee component
|
134
|
-
if type != "join"
|
135
|
-
in_pipes = {"rd_child_1" => @operation_pipes[name]["rd_child_1"], "wr_child_1" => @operation_pipes[name]["wr_child_1"]}
|
136
|
-
|
137
|
-
# Add join specific options
|
138
|
-
else
|
139
|
-
options[:join_options] = {}
|
140
|
-
in_pipes = {}
|
141
|
-
@arcs.each do |a|
|
142
|
-
|
143
|
-
if (a["dest"] == name)
|
144
|
-
# Left Side
|
145
|
-
if (a["left"] == 1)
|
146
|
-
options[:join_options][:lhs] = a["origin"]
|
147
|
-
in_pipes["rd_child_1"] = @operation_pipes[name]["rd_child_1"]
|
148
|
-
in_pipes["wr_child_1"] = @operation_pipes[name]["wr_child_1"]
|
149
|
-
# Right Side
|
150
|
-
elsif (a["right"] == 1)
|
151
|
-
options[:join_options][:rhs] = a["origin"]
|
152
|
-
in_pipes["rd_child_2"] = @operation_pipes[name]["rd_child_2"]
|
153
|
-
in_pipes["wr_child_2"] = @operation_pipes[name]["wr_child_2"]
|
154
|
-
end
|
155
|
-
end
|
156
|
-
end
|
157
|
-
end
|
158
|
-
|
159
|
-
# Index consumer pipes by stream name, consumer_name
|
160
|
-
out_pipes = {}
|
161
|
-
|
162
|
-
# Check if you are the consumee for a downstream join in order to select the correct pipe
|
163
|
-
if type != "sink"
|
164
|
-
@arc_map[name].each_pair do |stream, destinations|
|
165
|
-
out_pipes[stream] ||= {}
|
166
|
-
|
167
|
-
|
168
|
-
destinations.each do |dest|
|
169
|
-
out_pipes[stream][dest] ||= {}
|
170
|
-
|
171
|
-
# Check for a join at the destination
|
172
|
-
if (@node_map[dest]["type"] == "join")
|
173
|
-
@arcs.each do |a|
|
174
|
-
if (a["dest"] == dest && a["origin"] == name)
|
175
|
-
# Left Side
|
176
|
-
if (a["left"] == 1)
|
177
|
-
out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"]
|
178
|
-
out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
|
179
|
-
break
|
180
|
-
elsif (a["right"] == 1)
|
181
|
-
out_pipes[stream][dest]["wr_parent_2"] = @operation_pipes[dest]["wr_parent_2"]
|
182
|
-
out_pipes[stream][dest]["rd_parent_2"] = @operation_pipes[dest]["rd_parent_2"]
|
183
|
-
break
|
184
|
-
end
|
185
|
-
end
|
186
|
-
end
|
187
|
-
else
|
188
|
-
out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"]
|
189
|
-
out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
|
190
|
-
end
|
191
|
-
end
|
192
|
-
end
|
193
|
-
end
|
194
|
-
|
195
|
-
# Run the child process
|
196
|
-
Zillabyte::Runner::MultilangOperation.run(n, dir, in_pipes, out_pipes, self, meta, options)
|
197
|
-
|
198
|
-
rescue => e
|
199
|
-
display e.message
|
200
|
-
display e.backtrace
|
201
|
-
ensure
|
202
|
-
|
203
|
-
# Close the reading end of the child
|
204
|
-
pipes["rd_child_1"].close()
|
205
|
-
# Close the writing end of the child
|
206
|
-
pipes["wr_child_1"].close()
|
207
|
-
|
208
|
-
# Close secondary join child
|
209
|
-
pipes["rd_child_2"].close() if pipes["rd_child_2"]
|
210
|
-
pipes["wr_child_2"].close() if pipes["wr_child_2"]
|
211
|
-
|
212
|
-
exit!(-1)
|
213
|
-
end
|
214
|
-
|
215
|
-
end #end child
|
216
|
-
end
|
217
|
-
|
218
|
-
if interactive
|
219
|
-
display "To view results: Enter 'end' "
|
220
|
-
display ""
|
221
|
-
|
222
|
-
|
223
|
-
while true
|
224
|
-
display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
|
225
|
-
msg = ask
|
226
|
-
|
227
|
-
|
228
|
-
if msg == "end"
|
229
|
-
msg = END_CYCLE_MESSAGE
|
230
|
-
else
|
231
|
-
begin
|
232
|
-
JSON.parse(msg)
|
233
|
-
rescue JSON::ParserError
|
234
|
-
display "Received invalid JSON object"
|
235
|
-
next
|
236
|
-
end
|
237
|
-
end
|
238
|
-
# Send tuple to source
|
239
|
-
@operation_pipes["source_1"]["wr_parent_1"].puts msg
|
240
|
-
end
|
241
|
-
end
|
242
|
-
rescue => e
|
243
|
-
display e.message
|
244
|
-
display e.backtrace
|
245
|
-
ensure
|
246
|
-
Process.waitall()
|
247
|
-
@operation_pipes.each do |name, pipes|
|
248
|
-
#Close the writing end of the parent
|
249
|
-
pipes["wr_parent_1"].close()
|
250
|
-
# Close the reading end of the parent
|
251
|
-
pipes["rd_parent_1"].close()
|
252
|
-
|
253
|
-
# Close secondary join parent
|
254
|
-
pipes["wr_parent_2"].close() if pipes["wr_parent_2"]
|
255
|
-
pipes["rd_parent_2"].close() if pipes["rd_parent_2"]
|
256
|
-
|
257
|
-
end
|
258
|
-
end
|
259
|
-
|
260
|
-
end
|
261
|
-
end
|
262
|
-
|
263
|
-
def session
|
264
|
-
@session
|
265
|
-
end
|
266
|
-
|
267
|
-
|
268
|
-
def cdisplay(name, message, useName=true)
|
269
|
-
|
270
|
-
color = @colors[name] || :default
|
271
|
-
if message.nil? || message == ""
|
272
|
-
return
|
273
|
-
else
|
274
|
-
|
275
|
-
if message.is_a?(Array)
|
276
|
-
lines = message
|
277
|
-
else
|
278
|
-
lines = message.split("\n")
|
279
|
-
end
|
280
|
-
|
281
|
-
prefix = useName ? "#{name} - " : ""
|
282
|
-
display "#{prefix}#{lines.first}".colorize(color)
|
283
|
-
lines[1..-1].each do |line|
|
284
|
-
display "#{' '*prefix.size}#{line}".colorize(color)
|
285
|
-
end
|
286
|
-
end
|
287
|
-
end
|
288
|
-
|
289
|
-
|
290
|
-
def query_agnostic(query)
|
291
|
-
require("zillabyte/api")
|
292
|
-
@session.api.query.agnostic(query)
|
293
|
-
end
|
294
|
-
|
295
|
-
def display(message, newline = true)
|
296
|
-
@session.display(message, newline)
|
297
|
-
end
|
298
|
-
|
299
|
-
|
300
|
-
def describe_app(meta)
|
301
|
-
require("colorize")
|
302
|
-
require("indentation")
|
303
|
-
colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
|
304
|
-
rjust = 20
|
305
|
-
display "#{'app name'.rjust(rjust)}: #{meta['name']}"
|
306
|
-
display "#{'app language'.rjust(rjust)}: #{meta['language']}"
|
307
|
-
meta['nodes'].each_with_index do |node, index|
|
308
|
-
@colors[node['name']] ||= colors[index % colors.length]
|
309
|
-
color = @colors[node['name']]
|
310
|
-
display (("="*rjust + " operation ##{index}").colorize(color))
|
311
|
-
display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
|
312
|
-
display "#{"type".rjust(rjust)}: #{node['type'].to_s.colorize(color)}"
|
313
|
-
display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
|
314
|
-
display "#{"consumes".rjust(rjust)}: #{node['consumes'].to_s.colorize(color)}" if node['consumes']
|
315
|
-
display "#{"emits".rjust(rjust)}: #{JSON.pretty_generate(node['emits']).indent(rjust+2).lstrip.colorize(color)}" if node['emits']
|
316
|
-
end
|
317
|
-
end
|
318
|
-
|
319
|
-
|
320
|
-
end
|
@@ -1,636 +0,0 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
# Emulate Component Operations
|
5
|
-
class Zillabyte::Runner::ComponentOperation
|
6
|
-
|
7
|
-
NEXT_MESSAGE = "{\"command\": \"next\"}\n"
|
8
|
-
DONE_MESSAGE = "{\"command\": \"done\"}\n"
|
9
|
-
KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
|
10
|
-
END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
|
11
|
-
ENDMARKER = "\nend\n"
|
12
|
-
|
13
|
-
# Run the operation
|
14
|
-
def self.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
|
15
|
-
require "zillabyte/runner/multilang_operation"
|
16
|
-
require("zillabyte/runner/operation")
|
17
|
-
|
18
|
-
@__node = node
|
19
|
-
@__name = node["name"]
|
20
|
-
@__type = node["type"]
|
21
|
-
@__dir = dir
|
22
|
-
@__consumee_pipes = consumee_pipes
|
23
|
-
@__consumer_pipes = consumer_pipes
|
24
|
-
@__tester = tester
|
25
|
-
|
26
|
-
@__meta = meta
|
27
|
-
@__options = options
|
28
|
-
@__output_type = options[:output_type]
|
29
|
-
|
30
|
-
# Each consumer of a stream gets its own queue and message passing
|
31
|
-
@__emit_queues = {}
|
32
|
-
@__consumer_pipes.each_pair do |stream, consumers|
|
33
|
-
consumers.each_key do |consumer|
|
34
|
-
@__emit_queues[stream] ||= {}
|
35
|
-
@__emit_queues[stream][consumer] = {:write_queue => [], :ready => true}
|
36
|
-
end
|
37
|
-
end
|
38
|
-
begin
|
39
|
-
case @__type
|
40
|
-
when "input"
|
41
|
-
self.run_input()
|
42
|
-
when "component"
|
43
|
-
self.run_rpc_component()
|
44
|
-
when "output"
|
45
|
-
node["type"] = "sink"
|
46
|
-
Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
|
47
|
-
# Component outputs act in same manner as sinks
|
48
|
-
else
|
49
|
-
Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
|
50
|
-
end
|
51
|
-
rescue => e
|
52
|
-
cdisplay e.message
|
53
|
-
cdisplay e.backtrace
|
54
|
-
end
|
55
|
-
cdisplay "EXIT"
|
56
|
-
end
|
57
|
-
|
58
|
-
|
59
|
-
# Run a component input
|
60
|
-
def self.run_input()
|
61
|
-
input = @__options[:input]
|
62
|
-
|
63
|
-
fields = @__node["fields"].map {|f| f.keys[0]}
|
64
|
-
|
65
|
-
# Read input from file
|
66
|
-
if input
|
67
|
-
messages = []
|
68
|
-
cdisplay "reading from file...."
|
69
|
-
csv_rows = CSV.read("#{input}")
|
70
|
-
csv_rows.each do |row|
|
71
|
-
tuple = {}
|
72
|
-
fields.each {|f| tuple[f] = row.shift}
|
73
|
-
|
74
|
-
tuple_json = build_tuple_json(tuple)
|
75
|
-
|
76
|
-
@__emit_queues.each_pair do |stream, consumers|
|
77
|
-
consumers.each_pair do |consumer, emitter|
|
78
|
-
emitter[:write_queue] << tuple_json
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
# Index streams and consumers by their pipes for lookup
|
83
|
-
consumer_hash = build_consumer_hash()
|
84
|
-
|
85
|
-
# Send first tuple
|
86
|
-
@__emit_queues.each_pair do |stream, consumers|
|
87
|
-
consumers.each_key do |consumer|
|
88
|
-
tuple_json = get_consumer_tuple(stream, consumer)
|
89
|
-
emit_consumer_tuple(stream, consumer, tuple_json)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
# Sent tuples to consumers as appropriate
|
94
|
-
loop do
|
95
|
-
|
96
|
-
# Retrieve messages from consumers
|
97
|
-
rs, ws, es = IO.select(consumer_hash.keys, [], [])
|
98
|
-
|
99
|
-
# Emit tuples to consumers
|
100
|
-
emitted = false
|
101
|
-
rs.each do |r|
|
102
|
-
|
103
|
-
# Read from consumer
|
104
|
-
msg = read_message(r)
|
105
|
-
|
106
|
-
stream = consumer_hash[r][:stream]
|
107
|
-
consumer = consumer_hash[r][:consumer]
|
108
|
-
|
109
|
-
# Consumer is ready for next message
|
110
|
-
if msg["command"]
|
111
|
-
case msg["command"]
|
112
|
-
when "next"
|
113
|
-
|
114
|
-
@__emit_queues[stream][consumer][:ready] = true
|
115
|
-
tuple_json = get_consumer_tuple(stream, consumer)
|
116
|
-
|
117
|
-
# If all messages have been sent to consumer, end their cycle
|
118
|
-
if tuple_json.nil?
|
119
|
-
write_stream = get_write_stream(stream, consumer)
|
120
|
-
cdisplay "ending cycle for #{consumer}"
|
121
|
-
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
122
|
-
send_command_tuple(stream, consumer, DONE_MESSAGE)
|
123
|
-
|
124
|
-
else
|
125
|
-
# Emit tuple to consumer
|
126
|
-
emit_consumer_tuple(stream, consumer, tuple_json)
|
127
|
-
emitted = true
|
128
|
-
end
|
129
|
-
|
130
|
-
when "kill_cycle"
|
131
|
-
send_to_consumers(KILL_CYCLE_MESSAGE)
|
132
|
-
end
|
133
|
-
end
|
134
|
-
end
|
135
|
-
|
136
|
-
# Exit when done emitting
|
137
|
-
if !emitted
|
138
|
-
return
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
|
143
|
-
else
|
144
|
-
stdin = @__consumee_pipes["rd_child_1"]
|
145
|
-
loop do
|
146
|
-
|
147
|
-
msg = stdin.gets
|
148
|
-
if msg == KILL_CYCLE_MESSAGE
|
149
|
-
send_to_consumers(KILL_CYCLE_MESSAGE)
|
150
|
-
return
|
151
|
-
else
|
152
|
-
# Build tuples
|
153
|
-
args = msg.scan(/(?:\w|"[^"]*")+/).map {|s| s.gsub(/[\'\"]/, '')}
|
154
|
-
component_args = []
|
155
|
-
|
156
|
-
while(true) do
|
157
|
-
break if args.empty?
|
158
|
-
tuple = {}
|
159
|
-
fields.each {|f| tuple[f] = args.shift}
|
160
|
-
tuple_json = build_tuple_json(tuple)
|
161
|
-
display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
|
162
|
-
send_to_consumers(tuple_json)
|
163
|
-
end
|
164
|
-
end
|
165
|
-
end
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
|
170
|
-
# Send to and manage an RPC component
|
171
|
-
def self.run_rpc_component()
|
172
|
-
|
173
|
-
# Index streams and consumers by their pipes for lookup
|
174
|
-
consumer_hash = build_consumer_hash()
|
175
|
-
|
176
|
-
# Keep track of how many consumers to handle before exiting
|
177
|
-
consumers_running = consumer_hash.keys.length
|
178
|
-
|
179
|
-
# Kill the cycle on error
|
180
|
-
cycle_killed = false
|
181
|
-
|
182
|
-
# Begin cycle
|
183
|
-
end_cycle_received = false
|
184
|
-
|
185
|
-
# The input component(singular at the moment)
|
186
|
-
read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"]]
|
187
|
-
|
188
|
-
# Start communication with API
|
189
|
-
api = @__tester.session.api
|
190
|
-
require("zillabyte/api/components")
|
191
|
-
caller = Zillabyte::API::Components.new(api)
|
192
|
-
component_id = @__node["id"]
|
193
|
-
output_format = @__node["output_format"]
|
194
|
-
component_info = api.request(
|
195
|
-
:expects => 200,
|
196
|
-
:method => :get,
|
197
|
-
:path => "/flows/#{component_id}"
|
198
|
-
)
|
199
|
-
component_schema = component_info.body["schema"]
|
200
|
-
cdisplay("error: The requested component is not properly registered.") if component_info.nil?
|
201
|
-
|
202
|
-
component_nodes = component_schema["nodes"]
|
203
|
-
source_nodes = []
|
204
|
-
component_nodes.each do |node|
|
205
|
-
source_nodes << node if node["type"] == "source"
|
206
|
-
end
|
207
|
-
cdisplay("error: This component has multiple input streams. Currently we only support single input streams, sorry!") if source_nodes.size > 1
|
208
|
-
|
209
|
-
fields = source_nodes[0]["fields"]
|
210
|
-
|
211
|
-
begin
|
212
|
-
|
213
|
-
# Receive and handle messages
|
214
|
-
loop do
|
215
|
-
|
216
|
-
# Read from a stream
|
217
|
-
rs = select_read_streams(read_streams)
|
218
|
-
rs.each do |r|
|
219
|
-
# Read a message
|
220
|
-
obj = read_message(r)
|
221
|
-
|
222
|
-
# Handle tuple through RPC
|
223
|
-
if obj['tuple']
|
224
|
-
cdisplay("error: The number of inputs to the component does not match the declared number for stream #{source_nodes[0]["name"]}.") if obj['tuple'].size != fields.size
|
225
|
-
rpc_inputs = []
|
226
|
-
fields.each do |field|
|
227
|
-
rpc_inputs << obj['tuple'][field.keys[0]]
|
228
|
-
end
|
229
|
-
|
230
|
-
display_json = Hash[obj['tuple'].map{|k, v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
|
231
|
-
|
232
|
-
|
233
|
-
# Send RPC call
|
234
|
-
cdisplay "sending RPC call..."
|
235
|
-
call_options = {
|
236
|
-
:rpc_inputs => [rpc_inputs],
|
237
|
-
:output_format => output_format
|
238
|
-
}
|
239
|
-
|
240
|
-
res = caller.rpc(component_id, call_options)
|
241
|
-
cdisplay "received API response : #{res}"
|
242
|
-
|
243
|
-
#TODO handle errors
|
244
|
-
if res['error']
|
245
|
-
cdisplay("error: #{res['error']}")
|
246
|
-
next
|
247
|
-
end
|
248
|
-
run_ids = []
|
249
|
-
res['execute_ids'].each do |q, qid|
|
250
|
-
run_ids << qid
|
251
|
-
end
|
252
|
-
status = res['status']
|
253
|
-
|
254
|
-
|
255
|
-
# Send result call
|
256
|
-
loop do
|
257
|
-
|
258
|
-
# If a call for results is not ready, save the execution ID for the next cycle
|
259
|
-
ids_in_progress = []
|
260
|
-
res = caller.get_rpc_results(component_id, {:execute_ids => run_ids})
|
261
|
-
if res['error']
|
262
|
-
cdisplay("error: #{res['error']}")
|
263
|
-
next
|
264
|
-
end
|
265
|
-
|
266
|
-
|
267
|
-
# check results
|
268
|
-
res['results'].each do |run_id, hash|
|
269
|
-
|
270
|
-
# We have results
|
271
|
-
if hash['status'] == "complete"
|
272
|
-
|
273
|
-
# Handle Tuple in here
|
274
|
-
data_streams = hash['data']
|
275
|
-
|
276
|
-
# Handle each resulting tuple
|
277
|
-
data_streams.each_pair do |stream, data_tuples|
|
278
|
-
data_tuples.each do |data_tuple|
|
279
|
-
tuple_json = build_tuple_json(data_tuple)
|
280
|
-
|
281
|
-
#send or enqueue the tuple to all consumers of the stream
|
282
|
-
@__emit_queues.each_pair do |stream, consumers|
|
283
|
-
consumers.each_pair do |consumer, emitter|
|
284
|
-
if emitter[:ready]
|
285
|
-
emit_consumer_tuple(stream, consumer, tuple_json)
|
286
|
-
else
|
287
|
-
@__emit_queues[stream][consumer][:write_queue] << tuple_json
|
288
|
-
end
|
289
|
-
end
|
290
|
-
end
|
291
|
-
|
292
|
-
end
|
293
|
-
end
|
294
|
-
|
295
|
-
else
|
296
|
-
cdisplay "RPC #{run_id} has not completed... "
|
297
|
-
ids_in_progress << run_id
|
298
|
-
end
|
299
|
-
end
|
300
|
-
run_ids = ids_in_progress
|
301
|
-
|
302
|
-
# If no more IDs to run, we are done
|
303
|
-
if run_ids.empty?
|
304
|
-
break
|
305
|
-
end
|
306
|
-
|
307
|
-
# Dont spam the API
|
308
|
-
sleep(2)
|
309
|
-
end
|
310
|
-
|
311
|
-
# Ask for next tuple
|
312
|
-
write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
|
313
|
-
|
314
|
-
|
315
|
-
# End cycle
|
316
|
-
elsif obj['command']
|
317
|
-
case obj["command"]
|
318
|
-
|
319
|
-
# Consumer is ready for a message
|
320
|
-
when "next"
|
321
|
-
stream = consumer_hash[r][:stream]
|
322
|
-
consumer = consumer_hash[r][:consumer]
|
323
|
-
|
324
|
-
@__emit_queues[stream][consumer][:ready] = true
|
325
|
-
tuple_json = get_consumer_tuple(stream, consumer)
|
326
|
-
|
327
|
-
# End cycle for consumer if it has processed all tuples
|
328
|
-
if tuple_json.nil? && end_cycle_received
|
329
|
-
send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
|
330
|
-
consumers_running -= 1
|
331
|
-
if consumers_running == 0
|
332
|
-
return
|
333
|
-
end
|
334
|
-
|
335
|
-
# TODO break if last consumer
|
336
|
-
elsif !tuple_json.nil?
|
337
|
-
# Emit tuple to consumer
|
338
|
-
emit_consumer_tuple(stream, consumer, tuple_json)
|
339
|
-
emitted = true
|
340
|
-
end
|
341
|
-
|
342
|
-
when "end_cycle"
|
343
|
-
end_cycle_received = true
|
344
|
-
when "kill_cycle"
|
345
|
-
cycle_killed = true
|
346
|
-
return
|
347
|
-
end
|
348
|
-
end
|
349
|
-
end
|
350
|
-
|
351
|
-
|
352
|
-
# Exit after ending consumer cycles
|
353
|
-
if consumers_running == 0
|
354
|
-
return
|
355
|
-
end
|
356
|
-
end
|
357
|
-
|
358
|
-
rescue => e
|
359
|
-
cdisplay e.message
|
360
|
-
cdisplay e.backtrace
|
361
|
-
ensure
|
362
|
-
# cleanup
|
363
|
-
if cycle_killed
|
364
|
-
send_to_consumers(KILL_CYCLE_MESSAGE, false)
|
365
|
-
send_to_consumees(KILL_CYCLE_MESSAGE, false)
|
366
|
-
end
|
367
|
-
end
|
368
|
-
|
369
|
-
|
370
|
-
end
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
private
|
375
|
-
|
376
|
-
BUFSIZE = 8192
|
377
|
-
|
378
|
-
# Each reading pipe has a read buffer and message queue
|
379
|
-
@__read_buffers = {}
|
380
|
-
@__read_buffered_messages = {}
|
381
|
-
|
382
|
-
|
383
|
-
# Return availible reading streams
|
384
|
-
def self.select_read_streams(read_streams)
|
385
|
-
|
386
|
-
rs = []
|
387
|
-
read_streams.each do |read_stream|
|
388
|
-
@__read_buffered_messages[read_stream] ||= []
|
389
|
-
if !@__read_buffered_messages[read_stream].empty?
|
390
|
-
rs << read_stream
|
391
|
-
end
|
392
|
-
end
|
393
|
-
return rs unless rs.empty?
|
394
|
-
|
395
|
-
rs, ws, es = IO.select(read_streams, [], [])
|
396
|
-
return rs
|
397
|
-
end
|
398
|
-
|
399
|
-
|
400
|
-
# Read a JSON message
|
401
|
-
def self.read_message(read_stream)
|
402
|
-
|
403
|
-
@__read_buffers[read_stream] ||= ""
|
404
|
-
@__read_buffered_messages[read_stream] ||= []
|
405
|
-
if !@__read_buffered_messages[read_stream].empty?
|
406
|
-
obj = @__read_buffered_messages[read_stream].shift
|
407
|
-
return obj
|
408
|
-
end
|
409
|
-
|
410
|
-
# read message from stream
|
411
|
-
loop do
|
412
|
-
|
413
|
-
while !@__read_buffers[read_stream].include? ENDMARKER
|
414
|
-
segment = read_stream.sysread(BUFSIZE)
|
415
|
-
@__read_buffers[read_stream] << segment
|
416
|
-
end
|
417
|
-
|
418
|
-
read_buffer = @__read_buffers[read_stream]
|
419
|
-
if read_buffer.include? ENDMARKER
|
420
|
-
objs = read_buffer.split(ENDMARKER)
|
421
|
-
ends = read_buffer.scan(ENDMARKER)
|
422
|
-
if objs.count == ends.count # We have a full number of messages
|
423
|
-
objs.each do |obj|
|
424
|
-
begin
|
425
|
-
@__read_buffered_messages[read_stream] << JSON.parse(obj)
|
426
|
-
rescue JSON::ParserError
|
427
|
-
cdisplay "READMESSAGE: invalid JSON #{obj}"
|
428
|
-
end
|
429
|
-
end
|
430
|
-
@__read_buffers[read_stream] = ""
|
431
|
-
return @__read_buffered_messages[read_stream].shift
|
432
|
-
else
|
433
|
-
|
434
|
-
(0..ends.count-1).each do |i|
|
435
|
-
obj = objs[i]
|
436
|
-
begin
|
437
|
-
@__read_buffered_messages[read_stream] << JSON.parse(obj)
|
438
|
-
rescue JSON::ParserError
|
439
|
-
cdisplay "READMESSAGE: invalid JSON #{obj}"
|
440
|
-
end
|
441
|
-
end
|
442
|
-
|
443
|
-
@__read_buffers[read_stream] = objs[ends.count..-1].join(ENDMARKER)
|
444
|
-
return @__read_buffered_messages[read_stream].shift
|
445
|
-
end
|
446
|
-
end
|
447
|
-
end
|
448
|
-
end
|
449
|
-
|
450
|
-
|
451
|
-
# Write JSON message
|
452
|
-
def self.write_message(write_stream, msg)
|
453
|
-
write_msg = msg.strip + ENDMARKER
|
454
|
-
write_stream.write write_msg
|
455
|
-
write_stream.flush
|
456
|
-
end
|
457
|
-
|
458
|
-
# Handshake connection to multilang
|
459
|
-
def self.handshake(write_stream, read_stream)
|
460
|
-
begin
|
461
|
-
write_message write_stream, HANDSHAKE_MESSAGE
|
462
|
-
msg = read_message(read_stream)
|
463
|
-
rescue Exception => e
|
464
|
-
cdisplay(e)
|
465
|
-
cdisplay("Error handshaking node")
|
466
|
-
raise e
|
467
|
-
end
|
468
|
-
end
|
469
|
-
|
470
|
-
|
471
|
-
# Build the hash of consumer streams for lookup when receiving responses
|
472
|
-
def self.build_consumer_hash()
|
473
|
-
consumer_hash = {}
|
474
|
-
@__emit_queues.each_pair do |stream, consumers|
|
475
|
-
consumers.each_key do |consumer|
|
476
|
-
|
477
|
-
pipes = @__consumer_pipes[stream][consumer]
|
478
|
-
if pipes.has_key? "rd_parent_1"
|
479
|
-
read_stream = pipes["rd_parent_1"]
|
480
|
-
consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
|
481
|
-
|
482
|
-
elsif pipes.has_key? "rd_parent_2"
|
483
|
-
read_stream = pipes["rd_parent_2"]
|
484
|
-
consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
|
485
|
-
end
|
486
|
-
end
|
487
|
-
end
|
488
|
-
|
489
|
-
return consumer_hash
|
490
|
-
end
|
491
|
-
|
492
|
-
|
493
|
-
# Send object to every consumer of the operation, regardless of stream
|
494
|
-
def self.send_to_consumers(json_obj)
|
495
|
-
@__consumer_pipes.each_pair do |stream, consumers|
|
496
|
-
consumers.each_pair do |consumer, pipe|
|
497
|
-
|
498
|
-
# Single or Left hand pipe
|
499
|
-
if (pipe.has_key? "wr_parent_1")
|
500
|
-
write_stream = get_write_stream(stream, consumer, 1)
|
501
|
-
write_message(write_stream, json_obj)
|
502
|
-
elsif (pipe.has_key? "wr_parent_2")
|
503
|
-
write_stream = get_write_stream(stream, consumer, 2)
|
504
|
-
write_message(write_stream, json_obj)
|
505
|
-
end
|
506
|
-
|
507
|
-
cdisplay "emitted #{json_obj.chomp} to #{consumer}"
|
508
|
-
end
|
509
|
-
end
|
510
|
-
end
|
511
|
-
|
512
|
-
|
513
|
-
# Send object to every consumer of the operation, regardless of stream
|
514
|
-
def self.send_to_consumees(json_obj)
|
515
|
-
pipes = @__consumee_pipes
|
516
|
-
# Left hand(or singular) input
|
517
|
-
if (pipes.has_key? "wr_parent_1")
|
518
|
-
write_stream = get_write_stream(stream, consumer, 1)
|
519
|
-
write_message(write_stream, json_obj)
|
520
|
-
end
|
521
|
-
|
522
|
-
# Right hand input
|
523
|
-
if (pipes.has_key? "wr_parent_2")
|
524
|
-
write_stream = get_write_stream(stream, consumer, 2)
|
525
|
-
write_message(write_stream, json_obj)
|
526
|
-
end
|
527
|
-
end
|
528
|
-
|
529
|
-
|
530
|
-
# Get the write pipe of the stream consumer
|
531
|
-
def self.get_write_stream(stream, consumer, number=1)
|
532
|
-
wr_pipe = "wr_parent_" + number.to_s
|
533
|
-
@__consumer_pipes[stream][consumer][wr_pipe]
|
534
|
-
end
|
535
|
-
|
536
|
-
|
537
|
-
# Get tuple for sending to consumer of stream
|
538
|
-
def self.get_consumer_tuple(stream, consumer)
|
539
|
-
@__emit_queues[stream][consumer][:write_queue].shift
|
540
|
-
end
|
541
|
-
|
542
|
-
|
543
|
-
def self.send_command_tuple(stream, consumer, json_obj)
|
544
|
-
pipe = @__consumer_pipes[stream][consumer]
|
545
|
-
# Single or Left hand pipe
|
546
|
-
if (pipe.has_key? "wr_parent_1")
|
547
|
-
write_stream = get_write_stream(stream, consumer, 1)
|
548
|
-
write_message(write_stream, json_obj)
|
549
|
-
|
550
|
-
# Right hand pipe
|
551
|
-
elsif (pipe.has_key? "wr_parent_2")
|
552
|
-
write_stream = get_write_stream(stream, consumer, 2)
|
553
|
-
write_message(write_stream, json_obj)
|
554
|
-
end
|
555
|
-
@__emit_queues[stream][consumer][:ready] = false
|
556
|
-
end
|
557
|
-
|
558
|
-
|
559
|
-
# Emit tuple_json to the consumer of a stream
|
560
|
-
def self.emit_consumer_tuple(stream, consumer, tuple_json)
|
561
|
-
begin
|
562
|
-
display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
|
563
|
-
rescue JSON::ParserError
|
564
|
-
cdisplay "Error: invalid JSON"
|
565
|
-
end
|
566
|
-
|
567
|
-
pipe = @__consumer_pipes[stream][consumer]
|
568
|
-
# Single or Left hand pipe
|
569
|
-
if (pipe.has_key? "wr_parent_1")
|
570
|
-
write_stream = get_write_stream(stream, consumer, 1)
|
571
|
-
write_message(write_stream, tuple_json)
|
572
|
-
|
573
|
-
# Right hand pipe
|
574
|
-
elsif (pipe.has_key? "wr_parent_2")
|
575
|
-
write_stream = get_write_stream(stream, consumer, 2)
|
576
|
-
write_message(write_stream, tuple_json)
|
577
|
-
end
|
578
|
-
|
579
|
-
@__emit_queues[stream][consumer][:ready] = false
|
580
|
-
cdisplay "emitted tuple #{display_json} to #{consumer} "
|
581
|
-
end
|
582
|
-
|
583
|
-
|
584
|
-
# Build a tuple and format into JSON
|
585
|
-
def self.build_tuple_json(tuple, meta = nil, column_aliases = nil)
|
586
|
-
meta ||= {}
|
587
|
-
column_aliases ||= {}
|
588
|
-
values = {}
|
589
|
-
tuple.each do |k, v|
|
590
|
-
if(k == "id")
|
591
|
-
nextx
|
592
|
-
elsif(k == "confidence" or k == "since" or k == "source")
|
593
|
-
meta[k] = v
|
594
|
-
else
|
595
|
-
values[k] = v
|
596
|
-
end
|
597
|
-
end
|
598
|
-
tuple_json = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
|
599
|
-
return tuple_json
|
600
|
-
end
|
601
|
-
|
602
|
-
|
603
|
-
# Construct a multilang command
|
604
|
-
def self.command(arg, ignore_stderr=false)
|
605
|
-
cdisplay("could not extract meta information. missing zillabyte.conf.yml?") if @__meta.nil?
|
606
|
-
|
607
|
-
full_script = File.join(@__dir, @__meta["script"])
|
608
|
-
stderr_opt = "2> /dev/null" if ignore_stderr
|
609
|
-
|
610
|
-
case @__meta["language"]
|
611
|
-
when "ruby"
|
612
|
-
# Execute in the bundler context
|
613
|
-
cmd = "cd \"#{@__dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr_opt}"
|
614
|
-
when "python"#{
|
615
|
-
if(File.directory?("#{@__dir}/vEnv"))
|
616
|
-
cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{@__dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr_opt}"
|
617
|
-
else
|
618
|
-
cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} #{stderr_opt}"
|
619
|
-
end
|
620
|
-
when "js"
|
621
|
-
require("zillabyte/api/settings")
|
622
|
-
cmd = "cd \"#{@__dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
|
623
|
-
else
|
624
|
-
cdisplay("no language specified")
|
625
|
-
end
|
626
|
-
return cmd
|
627
|
-
end
|
628
|
-
|
629
|
-
|
630
|
-
# Display a colored, formatted message
|
631
|
-
def self.cdisplay(msg, useName=true)
|
632
|
-
@__tester.cdisplay(@__name, msg, useName)
|
633
|
-
end
|
634
|
-
|
635
|
-
|
636
|
-
end
|