zillabyte-cli 0.9.20 → 0.9.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +0,0 @@
1
- require "zillabyte-cli/version"
2
- require "zillabyte/api"
3
- require "zillabyte/cli"
4
- require "zillabyte/common"
5
- require "zillabyte/queries"
@@ -1,12 +0,0 @@
1
- require "zillabyte/cli/base"
2
-
3
- # manage custom logs
4
- #
5
- class Zillabyte::Command::Logs < Zillabyte::Command::Base
6
-
7
-
8
-
9
-
10
-
11
-
12
- end
@@ -1,43 +0,0 @@
1
- require "zillabyte/cli/base"
2
- require 'readline'
3
- # REPL console for zillabyte commands
4
- #
5
- class Zillabyte::Command::Repl < Zillabyte::Command::Base
6
-
7
- # repl
8
- #
9
- # start a console session for zillabyte
10
- # --quiet # HIDDEN
11
- # --history HISTORY# HIDDEN hack to allow history for readline
12
- def index
13
- if !options[:quiet]
14
- v = `zillabyte version`
15
- display "\n#{v}Type q,exit or Ctrl+D to quit\n\n"
16
- end
17
- server = `echo $ZILLABYTE_API_HOST` || ""
18
- prompt = ""
19
- if server && server.chomp.length > 0
20
- prompt = "#{server.chomp} "
21
- end
22
- prompt += "zillabyte $ "
23
- if options[:history]
24
- #p options[:history]
25
- history = JSON.parse(options[:history])
26
- history.last(50).each do |his|
27
- # TODO: Handle single quotes ??
28
- Readline::HISTORY << his
29
- end
30
- end
31
- # TODO: Add tab completion for basic commands, app/relation names etc.
32
- while cmd = Readline.readline(prompt, true)
33
- if cmd && cmd.length > 0
34
- if cmd.downcase == "exit" || cmd.downcase == "q"
35
- display "" # TODO Make Ctrl+D print a newline too
36
- return
37
- else
38
- exec "zillabyte #{cmd}; zillabyte repl --quiet --history '#{Readline::HISTORY.to_a.to_json}'"
39
- end
40
- end
41
- end
42
- end
43
- end
@@ -1,27 +0,0 @@
1
- import zillabyte
2
-
3
- def prep(controller):
4
- return
5
-
6
- # This is the heart of your algorithm. It's processed on every
7
- # web page. This algorithm is run in parallel on possibly hundreds
8
- # of machines.
9
- def execute(controller, tup):
10
- if("hello world" in tup.values["html"]):
11
- controller.emit("has_hello_world",{"url":tup.values["url"]})
12
- return
13
-
14
- zillabyte.simple_function(\
15
- # This directive instructs zillabyte to give your function every
16
- # web page in our known universe. Your function will have access
17
- # to two fields: URL and HTML
18
- matches = "select * from web_pa", \
19
-
20
- # This directive tells Zillabyte what kind of data your function
21
- # produces. In this case, we're saying we will emit a tuple that
22
- # is one-column wide and contains the field 'URL'
23
- emits = [["has_hello_world", [{"url":"string"}]]], \
24
-
25
- prepare = prep, \
26
- execute = execute\
27
- )
@@ -1,6 +0,0 @@
1
- require "zillabyte/helpers"
2
-
3
-
4
- module Zillabyte::Runner
5
-
6
- end
@@ -1,320 +0,0 @@
1
- require "zillabyte/runner/multilang_operation"
2
-
3
- # HIDDEN:
4
- class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
5
- include Zillabyte::Helpers
6
-
7
- END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
8
-
9
- def run (meta, dir = Dir.pwd, session = nil, options = {})
10
-
11
- if meta.nil? or session.nil?
12
- return
13
- end
14
-
15
- @session = session
16
-
17
- @colors = {}
18
- output = options[:output]
19
- otype = options[:output_type]
20
- interactive = options[:interactive]
21
- cycles = (options[:cycles] || "1").to_i
22
-
23
- # Show the user what we know about their app...
24
- display "inferring your app details..."
25
- describe_app(meta)
26
-
27
- # Setup nodes and arcs
28
- @nodes = meta["nodes"]
29
- @node_map = {}
30
- @nodes.each do |n|
31
- @node_map[n["name"]] = n
32
- end
33
-
34
- @arcs = meta["arcs"]
35
-
36
- # Organize component pipes
37
- @operations = {}
38
- @operation_pipes = {}
39
-
40
- # On each cycle, setup and tear down a test harness
41
- (1..cycles).each do |cycle|
42
-
43
- display "starting cycle #{cycle}" unless cycles == 1
44
-
45
- begin
46
- # Setup component pipes
47
- @nodes.each do |n|
48
-
49
- name = n["name"]
50
- type = n["type"]
51
- emits = n["emits"]
52
- if n["type"] == "source"
53
- options[:end_cycle_policy] = n["end_cycle_policy"]
54
- end
55
-
56
- # Create two new pipes in the parent.
57
- rd_child_1, wr_parent_1 = IO.pipe()
58
- rd_parent_1, wr_child_1 = IO.pipe()
59
-
60
- @operation_pipes[name] = {
61
- "rd_child_1" => rd_child_1,
62
- "wr_child_1" => wr_child_1,
63
- "rd_parent_1" => rd_parent_1,
64
- "wr_parent_1" => wr_parent_1
65
- }
66
-
67
-
68
- # Add a second(right hand side) set ofpipes for joins
69
- if type == "join"
70
- # Create two new pipes in the parent.
71
- rd_child_2, wr_parent_2 = IO.pipe()
72
- rd_parent_2, wr_child_2 = IO.pipe()
73
- @operation_pipes[name]["rd_child_2"] = rd_child_2
74
- @operation_pipes[name]["wr_child_2"] = wr_child_2
75
- @operation_pipes[name]["rd_parent_2"] = rd_parent_2
76
- @operation_pipes[name]["wr_parent_2"] = wr_parent_2
77
- end
78
-
79
-
80
- end
81
-
82
- # Maps origin => {stream => [destinations]}
83
- @arc_map = {}
84
- @arcs.each do |a|
85
- origin = a["origin"]
86
- name = a["name"]
87
- dest = a["dest"]
88
- @arc_map[origin] ||= {}
89
- @arc_map[origin][name] ||= []
90
- @arc_map[origin][name] << a["dest"]
91
- end
92
-
93
- # # Spawn component threads
94
- @nodes.each do |n|
95
-
96
- name = n["name"]
97
- type = n["type"]
98
- emits = n["emits"]
99
-
100
-
101
- pipes = @operation_pipes[name]
102
-
103
- # Fork.
104
- pid = fork()
105
- if pid # In parent
106
- # Close the reading end of the first child so we can write to the child.
107
- pipes["rd_child_1"].close()
108
- # Close the writing end of the first child so we can read from the child.
109
- pipes["wr_child_1"].close()
110
-
111
- if type == "join"
112
- # Close the reading end of the second child so we can write to the child.
113
- pipes["rd_child_2"].close()
114
- # Close the writing end of the second child so we can read from the child.
115
- pipes["wr_child_2"].close()
116
- end
117
- else # in child
118
- # Close the writing end of the first parent so we can read from the parent.
119
- pipes["wr_parent_1"].close()
120
- # Close the reading end of the first parent so we can write to the parent.
121
- pipes["rd_parent_1"].close()
122
-
123
- if type == "join"
124
- # Close the reading end of the second child so we can write to the child.
125
- pipes["rd_parent_2"].close()
126
- # Close the writing end of the second child so we can read from the child.
127
- pipes["wr_parent_2"].close()
128
- end
129
-
130
-
131
- begin
132
-
133
- # Setup reading and writing pipes for communicating with consumee component
134
- if type != "join"
135
- in_pipes = {"rd_child_1" => @operation_pipes[name]["rd_child_1"], "wr_child_1" => @operation_pipes[name]["wr_child_1"]}
136
-
137
- # Add join specific options
138
- else
139
- options[:join_options] = {}
140
- in_pipes = {}
141
- @arcs.each do |a|
142
-
143
- if (a["dest"] == name)
144
- # Left Side
145
- if (a["left"] == 1)
146
- options[:join_options][:lhs] = a["origin"]
147
- in_pipes["rd_child_1"] = @operation_pipes[name]["rd_child_1"]
148
- in_pipes["wr_child_1"] = @operation_pipes[name]["wr_child_1"]
149
- # Right Side
150
- elsif (a["right"] == 1)
151
- options[:join_options][:rhs] = a["origin"]
152
- in_pipes["rd_child_2"] = @operation_pipes[name]["rd_child_2"]
153
- in_pipes["wr_child_2"] = @operation_pipes[name]["wr_child_2"]
154
- end
155
- end
156
- end
157
- end
158
-
159
- # Index consumer pipes by stream name, consumer_name
160
- out_pipes = {}
161
-
162
- # Check if you are the consumee for a downstream join in order to select the correct pipe
163
- if type != "sink"
164
- @arc_map[name].each_pair do |stream, destinations|
165
- out_pipes[stream] ||= {}
166
-
167
-
168
- destinations.each do |dest|
169
- out_pipes[stream][dest] ||= {}
170
-
171
- # Check for a join at the destination
172
- if (@node_map[dest]["type"] == "join")
173
- @arcs.each do |a|
174
- if (a["dest"] == dest && a["origin"] == name)
175
- # Left Side
176
- if (a["left"] == 1)
177
- out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"]
178
- out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
179
- break
180
- elsif (a["right"] == 1)
181
- out_pipes[stream][dest]["wr_parent_2"] = @operation_pipes[dest]["wr_parent_2"]
182
- out_pipes[stream][dest]["rd_parent_2"] = @operation_pipes[dest]["rd_parent_2"]
183
- break
184
- end
185
- end
186
- end
187
- else
188
- out_pipes[stream][dest]["wr_parent_1"] = @operation_pipes[dest]["wr_parent_1"]
189
- out_pipes[stream][dest]["rd_parent_1"] = @operation_pipes[dest]["rd_parent_1"]
190
- end
191
- end
192
- end
193
- end
194
-
195
- # Run the child process
196
- Zillabyte::Runner::MultilangOperation.run(n, dir, in_pipes, out_pipes, self, meta, options)
197
-
198
- rescue => e
199
- display e.message
200
- display e.backtrace
201
- ensure
202
-
203
- # Close the reading end of the child
204
- pipes["rd_child_1"].close()
205
- # Close the writing end of the child
206
- pipes["wr_child_1"].close()
207
-
208
- # Close secondary join child
209
- pipes["rd_child_2"].close() if pipes["rd_child_2"]
210
- pipes["wr_child_2"].close() if pipes["wr_child_2"]
211
-
212
- exit!(-1)
213
- end
214
-
215
- end #end child
216
- end
217
-
218
- if interactive
219
- display "To view results: Enter 'end' "
220
- display ""
221
-
222
-
223
- while true
224
- display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
225
- msg = ask
226
-
227
-
228
- if msg == "end"
229
- msg = END_CYCLE_MESSAGE
230
- else
231
- begin
232
- JSON.parse(msg)
233
- rescue JSON::ParserError
234
- display "Received invalid JSON object"
235
- next
236
- end
237
- end
238
- # Send tuple to source
239
- @operation_pipes["source_1"]["wr_parent_1"].puts msg
240
- end
241
- end
242
- rescue => e
243
- display e.message
244
- display e.backtrace
245
- ensure
246
- Process.waitall()
247
- @operation_pipes.each do |name, pipes|
248
- #Close the writing end of the parent
249
- pipes["wr_parent_1"].close()
250
- # Close the reading end of the parent
251
- pipes["rd_parent_1"].close()
252
-
253
- # Close secondary join parent
254
- pipes["wr_parent_2"].close() if pipes["wr_parent_2"]
255
- pipes["rd_parent_2"].close() if pipes["rd_parent_2"]
256
-
257
- end
258
- end
259
-
260
- end
261
- end
262
-
263
- def session
264
- @session
265
- end
266
-
267
-
268
- def cdisplay(name, message, useName=true)
269
-
270
- color = @colors[name] || :default
271
- if message.nil? || message == ""
272
- return
273
- else
274
-
275
- if message.is_a?(Array)
276
- lines = message
277
- else
278
- lines = message.split("\n")
279
- end
280
-
281
- prefix = useName ? "#{name} - " : ""
282
- display "#{prefix}#{lines.first}".colorize(color)
283
- lines[1..-1].each do |line|
284
- display "#{' '*prefix.size}#{line}".colorize(color)
285
- end
286
- end
287
- end
288
-
289
-
290
- def query_agnostic(query)
291
- require("zillabyte/api")
292
- @session.api.query.agnostic(query)
293
- end
294
-
295
- def display(message, newline = true)
296
- @session.display(message, newline)
297
- end
298
-
299
-
300
- def describe_app(meta)
301
- require("colorize")
302
- require("indentation")
303
- colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
304
- rjust = 20
305
- display "#{'app name'.rjust(rjust)}: #{meta['name']}"
306
- display "#{'app language'.rjust(rjust)}: #{meta['language']}"
307
- meta['nodes'].each_with_index do |node, index|
308
- @colors[node['name']] ||= colors[index % colors.length]
309
- color = @colors[node['name']]
310
- display (("="*rjust + " operation ##{index}").colorize(color))
311
- display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
312
- display "#{"type".rjust(rjust)}: #{node['type'].to_s.colorize(color)}"
313
- display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
314
- display "#{"consumes".rjust(rjust)}: #{node['consumes'].to_s.colorize(color)}" if node['consumes']
315
- display "#{"emits".rjust(rjust)}: #{JSON.pretty_generate(node['emits']).indent(rjust+2).lstrip.colorize(color)}" if node['emits']
316
- end
317
- end
318
-
319
-
320
- end
@@ -1,636 +0,0 @@
1
-
2
-
3
-
4
- # Emulate Component Operations
5
- class Zillabyte::Runner::ComponentOperation
6
-
7
- NEXT_MESSAGE = "{\"command\": \"next\"}\n"
8
- DONE_MESSAGE = "{\"command\": \"done\"}\n"
9
- KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
10
- END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
11
- ENDMARKER = "\nend\n"
12
-
13
- # Run the operation
14
- def self.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options = {})
15
- require "zillabyte/runner/multilang_operation"
16
- require("zillabyte/runner/operation")
17
-
18
- @__node = node
19
- @__name = node["name"]
20
- @__type = node["type"]
21
- @__dir = dir
22
- @__consumee_pipes = consumee_pipes
23
- @__consumer_pipes = consumer_pipes
24
- @__tester = tester
25
-
26
- @__meta = meta
27
- @__options = options
28
- @__output_type = options[:output_type]
29
-
30
- # Each consumer of a stream gets its own queue and message passing
31
- @__emit_queues = {}
32
- @__consumer_pipes.each_pair do |stream, consumers|
33
- consumers.each_key do |consumer|
34
- @__emit_queues[stream] ||= {}
35
- @__emit_queues[stream][consumer] = {:write_queue => [], :ready => true}
36
- end
37
- end
38
- begin
39
- case @__type
40
- when "input"
41
- self.run_input()
42
- when "component"
43
- self.run_rpc_component()
44
- when "output"
45
- node["type"] = "sink"
46
- Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
47
- # Component outputs act in same manner as sinks
48
- else
49
- Zillabyte::Runner::MultilangOperation.run(node, dir, consumee_pipes, consumer_pipes, tester, meta, options)
50
- end
51
- rescue => e
52
- cdisplay e.message
53
- cdisplay e.backtrace
54
- end
55
- cdisplay "EXIT"
56
- end
57
-
58
-
59
- # Run a component input
60
- def self.run_input()
61
- input = @__options[:input]
62
-
63
- fields = @__node["fields"].map {|f| f.keys[0]}
64
-
65
- # Read input from file
66
- if input
67
- messages = []
68
- cdisplay "reading from file...."
69
- csv_rows = CSV.read("#{input}")
70
- csv_rows.each do |row|
71
- tuple = {}
72
- fields.each {|f| tuple[f] = row.shift}
73
-
74
- tuple_json = build_tuple_json(tuple)
75
-
76
- @__emit_queues.each_pair do |stream, consumers|
77
- consumers.each_pair do |consumer, emitter|
78
- emitter[:write_queue] << tuple_json
79
- end
80
- end
81
-
82
- # Index streams and consumers by their pipes for lookup
83
- consumer_hash = build_consumer_hash()
84
-
85
- # Send first tuple
86
- @__emit_queues.each_pair do |stream, consumers|
87
- consumers.each_key do |consumer|
88
- tuple_json = get_consumer_tuple(stream, consumer)
89
- emit_consumer_tuple(stream, consumer, tuple_json)
90
- end
91
- end
92
-
93
- # Sent tuples to consumers as appropriate
94
- loop do
95
-
96
- # Retrieve messages from consumers
97
- rs, ws, es = IO.select(consumer_hash.keys, [], [])
98
-
99
- # Emit tuples to consumers
100
- emitted = false
101
- rs.each do |r|
102
-
103
- # Read from consumer
104
- msg = read_message(r)
105
-
106
- stream = consumer_hash[r][:stream]
107
- consumer = consumer_hash[r][:consumer]
108
-
109
- # Consumer is ready for next message
110
- if msg["command"]
111
- case msg["command"]
112
- when "next"
113
-
114
- @__emit_queues[stream][consumer][:ready] = true
115
- tuple_json = get_consumer_tuple(stream, consumer)
116
-
117
- # If all messages have been sent to consumer, end their cycle
118
- if tuple_json.nil?
119
- write_stream = get_write_stream(stream, consumer)
120
- cdisplay "ending cycle for #{consumer}"
121
- send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
122
- send_command_tuple(stream, consumer, DONE_MESSAGE)
123
-
124
- else
125
- # Emit tuple to consumer
126
- emit_consumer_tuple(stream, consumer, tuple_json)
127
- emitted = true
128
- end
129
-
130
- when "kill_cycle"
131
- send_to_consumers(KILL_CYCLE_MESSAGE)
132
- end
133
- end
134
- end
135
-
136
- # Exit when done emitting
137
- if !emitted
138
- return
139
- end
140
- end
141
- end
142
-
143
- else
144
- stdin = @__consumee_pipes["rd_child_1"]
145
- loop do
146
-
147
- msg = stdin.gets
148
- if msg == KILL_CYCLE_MESSAGE
149
- send_to_consumers(KILL_CYCLE_MESSAGE)
150
- return
151
- else
152
- # Build tuples
153
- args = msg.scan(/(?:\w|"[^"]*")+/).map {|s| s.gsub(/[\'\"]/, '')}
154
- component_args = []
155
-
156
- while(true) do
157
- break if args.empty?
158
- tuple = {}
159
- fields.each {|f| tuple[f] = args.shift}
160
- tuple_json = build_tuple_json(tuple)
161
- display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
162
- send_to_consumers(tuple_json)
163
- end
164
- end
165
- end
166
- end
167
- end
168
-
169
-
170
- # Send to and manage an RPC component
171
- def self.run_rpc_component()
172
-
173
- # Index streams and consumers by their pipes for lookup
174
- consumer_hash = build_consumer_hash()
175
-
176
- # Keep track of how many consumers to handle before exiting
177
- consumers_running = consumer_hash.keys.length
178
-
179
- # Kill the cycle on error
180
- cycle_killed = false
181
-
182
- # Begin cycle
183
- end_cycle_received = false
184
-
185
- # The input component(singular at the moment)
186
- read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"]]
187
-
188
- # Start communication with API
189
- api = @__tester.session.api
190
- require("zillabyte/api/components")
191
- caller = Zillabyte::API::Components.new(api)
192
- component_id = @__node["id"]
193
- output_format = @__node["output_format"]
194
- component_info = api.request(
195
- :expects => 200,
196
- :method => :get,
197
- :path => "/flows/#{component_id}"
198
- )
199
- component_schema = component_info.body["schema"]
200
- cdisplay("error: The requested component is not properly registered.") if component_info.nil?
201
-
202
- component_nodes = component_schema["nodes"]
203
- source_nodes = []
204
- component_nodes.each do |node|
205
- source_nodes << node if node["type"] == "source"
206
- end
207
- cdisplay("error: This component has multiple input streams. Currently we only support single input streams, sorry!") if source_nodes.size > 1
208
-
209
- fields = source_nodes[0]["fields"]
210
-
211
- begin
212
-
213
- # Receive and handle messages
214
- loop do
215
-
216
- # Read from a stream
217
- rs = select_read_streams(read_streams)
218
- rs.each do |r|
219
- # Read a message
220
- obj = read_message(r)
221
-
222
- # Handle tuple through RPC
223
- if obj['tuple']
224
- cdisplay("error: The number of inputs to the component does not match the declared number for stream #{source_nodes[0]["name"]}.") if obj['tuple'].size != fields.size
225
- rpc_inputs = []
226
- fields.each do |field|
227
- rpc_inputs << obj['tuple'][field.keys[0]]
228
- end
229
-
230
- display_json = Hash[obj['tuple'].map{|k, v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
231
-
232
-
233
- # Send RPC call
234
- cdisplay "sending RPC call..."
235
- call_options = {
236
- :rpc_inputs => [rpc_inputs],
237
- :output_format => output_format
238
- }
239
-
240
- res = caller.rpc(component_id, call_options)
241
- cdisplay "received API response : #{res}"
242
-
243
- #TODO handle errors
244
- if res['error']
245
- cdisplay("error: #{res['error']}")
246
- next
247
- end
248
- run_ids = []
249
- res['execute_ids'].each do |q, qid|
250
- run_ids << qid
251
- end
252
- status = res['status']
253
-
254
-
255
- # Send result call
256
- loop do
257
-
258
- # If a call for results is not ready, save the execution ID for the next cycle
259
- ids_in_progress = []
260
- res = caller.get_rpc_results(component_id, {:execute_ids => run_ids})
261
- if res['error']
262
- cdisplay("error: #{res['error']}")
263
- next
264
- end
265
-
266
-
267
- # check results
268
- res['results'].each do |run_id, hash|
269
-
270
- # We have results
271
- if hash['status'] == "complete"
272
-
273
- # Handle Tuple in here
274
- data_streams = hash['data']
275
-
276
- # Handle each resulting tuple
277
- data_streams.each_pair do |stream, data_tuples|
278
- data_tuples.each do |data_tuple|
279
- tuple_json = build_tuple_json(data_tuple)
280
-
281
- #send or enqueue the tuple to all consumers of the stream
282
- @__emit_queues.each_pair do |stream, consumers|
283
- consumers.each_pair do |consumer, emitter|
284
- if emitter[:ready]
285
- emit_consumer_tuple(stream, consumer, tuple_json)
286
- else
287
- @__emit_queues[stream][consumer][:write_queue] << tuple_json
288
- end
289
- end
290
- end
291
-
292
- end
293
- end
294
-
295
- else
296
- cdisplay "RPC #{run_id} has not completed... "
297
- ids_in_progress << run_id
298
- end
299
- end
300
- run_ids = ids_in_progress
301
-
302
- # If no more IDs to run, we are done
303
- if run_ids.empty?
304
- break
305
- end
306
-
307
- # Dont spam the API
308
- sleep(2)
309
- end
310
-
311
- # Ask for next tuple
312
- write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
313
-
314
-
315
- # End cycle
316
- elsif obj['command']
317
- case obj["command"]
318
-
319
- # Consumer is ready for a message
320
- when "next"
321
- stream = consumer_hash[r][:stream]
322
- consumer = consumer_hash[r][:consumer]
323
-
324
- @__emit_queues[stream][consumer][:ready] = true
325
- tuple_json = get_consumer_tuple(stream, consumer)
326
-
327
- # End cycle for consumer if it has processed all tuples
328
- if tuple_json.nil? && end_cycle_received
329
- send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
330
- consumers_running -= 1
331
- if consumers_running == 0
332
- return
333
- end
334
-
335
- # TODO break if last consumer
336
- elsif !tuple_json.nil?
337
- # Emit tuple to consumer
338
- emit_consumer_tuple(stream, consumer, tuple_json)
339
- emitted = true
340
- end
341
-
342
- when "end_cycle"
343
- end_cycle_received = true
344
- when "kill_cycle"
345
- cycle_killed = true
346
- return
347
- end
348
- end
349
- end
350
-
351
-
352
- # Exit after ending consumer cycles
353
- if consumers_running == 0
354
- return
355
- end
356
- end
357
-
358
- rescue => e
359
- cdisplay e.message
360
- cdisplay e.backtrace
361
- ensure
362
- # cleanup
363
- if cycle_killed
364
- send_to_consumers(KILL_CYCLE_MESSAGE, false)
365
- send_to_consumees(KILL_CYCLE_MESSAGE, false)
366
- end
367
- end
368
-
369
-
370
- end
371
-
372
-
373
-
374
- private
375
-
376
- BUFSIZE = 8192
377
-
378
- # Each reading pipe has a read buffer and message queue
379
- @__read_buffers = {}
380
- @__read_buffered_messages = {}
381
-
382
-
383
- # Return availible reading streams
384
- def self.select_read_streams(read_streams)
385
-
386
- rs = []
387
- read_streams.each do |read_stream|
388
- @__read_buffered_messages[read_stream] ||= []
389
- if !@__read_buffered_messages[read_stream].empty?
390
- rs << read_stream
391
- end
392
- end
393
- return rs unless rs.empty?
394
-
395
- rs, ws, es = IO.select(read_streams, [], [])
396
- return rs
397
- end
398
-
399
-
400
- # Read a JSON message
401
- def self.read_message(read_stream)
402
-
403
- @__read_buffers[read_stream] ||= ""
404
- @__read_buffered_messages[read_stream] ||= []
405
- if !@__read_buffered_messages[read_stream].empty?
406
- obj = @__read_buffered_messages[read_stream].shift
407
- return obj
408
- end
409
-
410
- # read message from stream
411
- loop do
412
-
413
- while !@__read_buffers[read_stream].include? ENDMARKER
414
- segment = read_stream.sysread(BUFSIZE)
415
- @__read_buffers[read_stream] << segment
416
- end
417
-
418
- read_buffer = @__read_buffers[read_stream]
419
- if read_buffer.include? ENDMARKER
420
- objs = read_buffer.split(ENDMARKER)
421
- ends = read_buffer.scan(ENDMARKER)
422
- if objs.count == ends.count # We have a full number of messages
423
- objs.each do |obj|
424
- begin
425
- @__read_buffered_messages[read_stream] << JSON.parse(obj)
426
- rescue JSON::ParserError
427
- cdisplay "READMESSAGE: invalid JSON #{obj}"
428
- end
429
- end
430
- @__read_buffers[read_stream] = ""
431
- return @__read_buffered_messages[read_stream].shift
432
- else
433
-
434
- (0..ends.count-1).each do |i|
435
- obj = objs[i]
436
- begin
437
- @__read_buffered_messages[read_stream] << JSON.parse(obj)
438
- rescue JSON::ParserError
439
- cdisplay "READMESSAGE: invalid JSON #{obj}"
440
- end
441
- end
442
-
443
- @__read_buffers[read_stream] = objs[ends.count..-1].join(ENDMARKER)
444
- return @__read_buffered_messages[read_stream].shift
445
- end
446
- end
447
- end
448
- end
449
-
450
-
451
- # Write JSON message
452
- def self.write_message(write_stream, msg)
453
- write_msg = msg.strip + ENDMARKER
454
- write_stream.write write_msg
455
- write_stream.flush
456
- end
457
-
458
- # Handshake connection to multilang
459
- def self.handshake(write_stream, read_stream)
460
- begin
461
- write_message write_stream, HANDSHAKE_MESSAGE
462
- msg = read_message(read_stream)
463
- rescue Exception => e
464
- cdisplay(e)
465
- cdisplay("Error handshaking node")
466
- raise e
467
- end
468
- end
469
-
470
-
471
- # Build the hash of consumer streams for lookup when receiving responses
472
- def self.build_consumer_hash()
473
- consumer_hash = {}
474
- @__emit_queues.each_pair do |stream, consumers|
475
- consumers.each_key do |consumer|
476
-
477
- pipes = @__consumer_pipes[stream][consumer]
478
- if pipes.has_key? "rd_parent_1"
479
- read_stream = pipes["rd_parent_1"]
480
- consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
481
-
482
- elsif pipes.has_key? "rd_parent_2"
483
- read_stream = pipes["rd_parent_2"]
484
- consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
485
- end
486
- end
487
- end
488
-
489
- return consumer_hash
490
- end
491
-
492
-
493
- # Send object to every consumer of the operation, regardless of stream
494
- def self.send_to_consumers(json_obj)
495
- @__consumer_pipes.each_pair do |stream, consumers|
496
- consumers.each_pair do |consumer, pipe|
497
-
498
- # Single or Left hand pipe
499
- if (pipe.has_key? "wr_parent_1")
500
- write_stream = get_write_stream(stream, consumer, 1)
501
- write_message(write_stream, json_obj)
502
- elsif (pipe.has_key? "wr_parent_2")
503
- write_stream = get_write_stream(stream, consumer, 2)
504
- write_message(write_stream, json_obj)
505
- end
506
-
507
- cdisplay "emitted #{json_obj.chomp} to #{consumer}"
508
- end
509
- end
510
- end
511
-
512
-
513
- # Send object to every consumer of the operation, regardless of stream
514
- def self.send_to_consumees(json_obj)
515
- pipes = @__consumee_pipes
516
- # Left hand(or singular) input
517
- if (pipes.has_key? "wr_parent_1")
518
- write_stream = get_write_stream(stream, consumer, 1)
519
- write_message(write_stream, json_obj)
520
- end
521
-
522
- # Right hand input
523
- if (pipes.has_key? "wr_parent_2")
524
- write_stream = get_write_stream(stream, consumer, 2)
525
- write_message(write_stream, json_obj)
526
- end
527
- end
528
-
529
-
530
- # Get the write pipe of the stream consumer
531
- def self.get_write_stream(stream, consumer, number=1)
532
- wr_pipe = "wr_parent_" + number.to_s
533
- @__consumer_pipes[stream][consumer][wr_pipe]
534
- end
535
-
536
-
537
- # Get tuple for sending to consumer of stream
538
- def self.get_consumer_tuple(stream, consumer)
539
- @__emit_queues[stream][consumer][:write_queue].shift
540
- end
541
-
542
-
543
- def self.send_command_tuple(stream, consumer, json_obj)
544
- pipe = @__consumer_pipes[stream][consumer]
545
- # Single or Left hand pipe
546
- if (pipe.has_key? "wr_parent_1")
547
- write_stream = get_write_stream(stream, consumer, 1)
548
- write_message(write_stream, json_obj)
549
-
550
- # Right hand pipe
551
- elsif (pipe.has_key? "wr_parent_2")
552
- write_stream = get_write_stream(stream, consumer, 2)
553
- write_message(write_stream, json_obj)
554
- end
555
- @__emit_queues[stream][consumer][:ready] = false
556
- end
557
-
558
-
559
- # Emit tuple_json to the consumer of a stream
560
- def self.emit_consumer_tuple(stream, consumer, tuple_json)
561
- begin
562
- display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
563
- rescue JSON::ParserError
564
- cdisplay "Error: invalid JSON"
565
- end
566
-
567
- pipe = @__consumer_pipes[stream][consumer]
568
- # Single or Left hand pipe
569
- if (pipe.has_key? "wr_parent_1")
570
- write_stream = get_write_stream(stream, consumer, 1)
571
- write_message(write_stream, tuple_json)
572
-
573
- # Right hand pipe
574
- elsif (pipe.has_key? "wr_parent_2")
575
- write_stream = get_write_stream(stream, consumer, 2)
576
- write_message(write_stream, tuple_json)
577
- end
578
-
579
- @__emit_queues[stream][consumer][:ready] = false
580
- cdisplay "emitted tuple #{display_json} to #{consumer} "
581
- end
582
-
583
-
584
- # Build a tuple and format into JSON
585
- def self.build_tuple_json(tuple, meta = nil, column_aliases = nil)
586
- meta ||= {}
587
- column_aliases ||= {}
588
- values = {}
589
- tuple.each do |k, v|
590
- if(k == "id")
591
- nextx
592
- elsif(k == "confidence" or k == "since" or k == "source")
593
- meta[k] = v
594
- else
595
- values[k] = v
596
- end
597
- end
598
- tuple_json = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
599
- return tuple_json
600
- end
601
-
602
-
603
- # Construct a multilang command
604
- def self.command(arg, ignore_stderr=false)
605
- cdisplay("could not extract meta information. missing zillabyte.conf.yml?") if @__meta.nil?
606
-
607
- full_script = File.join(@__dir, @__meta["script"])
608
- stderr_opt = "2> /dev/null" if ignore_stderr
609
-
610
- case @__meta["language"]
611
- when "ruby"
612
- # Execute in the bundler context
613
- cmd = "cd \"#{@__dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr_opt}"
614
- when "python"#{
615
- if(File.directory?("#{@__dir}/vEnv"))
616
- cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{@__dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr_opt}"
617
- else
618
- cmd = "cd \"#{@__dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} #{stderr_opt}"
619
- end
620
- when "js"
621
- require("zillabyte/api/settings")
622
- cmd = "cd \"#{@__dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
623
- else
624
- cdisplay("no language specified")
625
- end
626
- return cmd
627
- end
628
-
629
-
630
- # Display a colored, formatted message
631
- def self.cdisplay(msg, useName=true)
632
- @__tester.cdisplay(@__name, msg, useName)
633
- end
634
-
635
-
636
- end