zillabyte-cli 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,7 @@ require 'zillabyte/api/components'
11
11
  class Zillabyte::Runner::ComponentOperation
12
12
 
13
13
  NEXT_MESSAGE = "{\"command\": \"next\"}\n"
14
+ DONE_MESSAGE = "{\"command\": \"done\"}\n"
14
15
  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
15
16
  ENDMARKER = "\nend\n"
16
17
 
@@ -38,15 +39,14 @@ class Zillabyte::Runner::ComponentOperation
38
39
  @__emit_queues[stream][consumer] = {:write_queue => [], :ready => true}
39
40
  end
40
41
  end
41
-
42
42
  begin
43
43
  case @__type
44
44
  when "source"
45
45
  self.run_input()
46
- when "sink"
47
- self.run_output()
48
46
  when "component"
49
47
  self.run_rpc_component()
48
+
49
+ # Component outputs act in same manner as sinks
50
50
  else
51
51
  Zillabyte::Runner::MultilangOperation.run(node, dir, consumee, consumer_pipes, tester, meta, options)
52
52
  end
@@ -64,24 +64,111 @@ class Zillabyte::Runner::ComponentOperation
64
64
 
65
65
  # Read input from file
66
66
  if input
67
+ messages = []
67
68
  cdisplay "reading from file...."
68
69
  csv_rows = CSV.read("#{input}")
70
+ fields = @__node["fields"].map {|f| f.keys[0]}
71
+ csv_rows.each do |row|
72
+ tuple = {}
73
+ fields.each {|f| tuple[f] = row.shift}
69
74
 
70
- else
75
+ tuple_json = build_tuple_json(tuple)
76
+
77
+ @__emit_queues.each_pair do |stream, consumers|
78
+ consumers.each_pair do |consumer, emitter|
79
+ emitter[:write_queue] << tuple_json
80
+ end
81
+ end
82
+
83
+ # Index streams and consumers by their pipes for lookup
84
+ consumer_hash = {}
85
+ @__emit_queues.each_pair do |stream, consumers|
86
+ consumers.each_key do |consumer|
87
+ read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
88
+ consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
89
+ end
90
+ end
91
+
92
+
93
+ # Send first tuple
94
+ @__emit_queues.each_pair do |stream, consumers|
95
+ consumers.each_key do |consumer|
96
+ tuple_json = get_consumer_tuple(stream, consumer)
97
+ emit_consumer_tuple(stream, consumer, tuple_json)
98
+ end
99
+ end
100
+
101
+ # Sent tuples to consumers as appropriate
102
+ loop do
103
+
104
+ # Retrieve messages from consumers
105
+ rs, ws, es = IO.select(consumer_hash.keys, [], [])
71
106
 
107
+ # Emit tuples to consumers
108
+ emitted = false
109
+ rs.each do |r|
110
+
111
+ # Read from consumer
112
+ msg = read_message(r)
113
+
114
+ stream = consumer_hash[r][:stream]
115
+ consumer = consumer_hash[r][:consumer]
116
+
117
+ # Consumer is ready for next message
118
+ if msg["command"] && msg["command"] == "next"
119
+
120
+ @__emit_queues[stream][consumer][:ready] = true
121
+ tuple_json = get_consumer_tuple(stream, consumer)
122
+
123
+ # If all messages have been sent to consumer, end their cycle
124
+ if tuple_json.nil?
125
+ write_stream = get_write_stream(stream, consumer)
126
+ cdisplay "ending cycle for #{consumer}"
127
+ write_message(write_stream, END_CYCLE_MESSAGE)
128
+ write_message(write_stream, DONE_MESSAGE)
129
+
130
+ else
131
+ # Emit tuple to consumer
132
+ emit_consumer_tuple(stream, consumer, tuple_json)
133
+ emitted = true
134
+ end
135
+ end
136
+ end
137
+
138
+ # Exit when done emitting
139
+ if !emitted
140
+ return
141
+ end
142
+ end
143
+ end
144
+
145
+ else
72
146
  stdin = @__consumee[:rd_child]
73
147
  loop do
74
148
 
75
149
  msg = stdin.gets
150
+ if msg == "end\n"
151
+ @__consumer_pipes.each_pair do |stream, consumers|
152
+ consumers.each_pair do |consumer, pipe|
153
+ write_stream = pipe[:wr_parent]
154
+ write_message(write_stream, END_CYCLE_MESSAGE)
155
+ write_message(write_stream, DONE_MESSAGE)
156
+ end
157
+ end
158
+
159
+ break
160
+ end
76
161
 
77
162
  # Build tuple
78
163
  begin
164
+
79
165
  tuple = JSON.parse(msg)
80
166
  rescue JSON::ParserError
81
167
  cdisplay "Error: invalid JSON"
82
168
  next
83
169
  end
84
170
 
171
+
85
172
  # Check input for correct fields
86
173
  has_fields = true
87
174
  fields = @__node['fields'].flat_map {|h| h.keys}
@@ -124,6 +211,28 @@ class Zillabyte::Runner::ComponentOperation
124
211
  # The input component(singular at the moment)
125
212
  read_streams = consumer_hash.keys.concat [@__consumee[:rd_child]]
126
213
 
214
+ # Start communication with API
215
+ api = @__tester.session.api
216
+ caller = Zillabyte::API::Components.new(api)
217
+ component_id = @__node["id"]
218
+ output_format = @__node["output_format"]
219
+ component_info = api.request(
220
+ :expects => 200,
221
+ :method => :get,
222
+ :path => "/flows/#{component_id}"
223
+ )
224
+ component_schema = component_info.body["schema"]
225
+ cdisplay("error: The requested component is not properly registered.") if component_info.nil?
226
+
227
+ component_nodes = component_schema["nodes"]
228
+ source_nodes = []
229
+ component_nodes.each do |node|
230
+ source_nodes << node if node["type"] == "source"
231
+ end
232
+ cdisplay("error: This component has multiple input streams. Currently we only support single input streams, sorry!") if source_nodes.size > 1
233
+
234
+ fields = source_nodes[0]["fields"]
235
+
127
236
  # Receive and handle messages
128
237
  loop do
129
238
 
@@ -135,21 +244,19 @@ class Zillabyte::Runner::ComponentOperation
135
244
 
136
245
  # Handle tuple through RPC
137
246
  if obj['tuple']
247
+ cdisplay("error: The number of inputs to the component does not match the declared number for stream #{source_nodes[0]["name"]}.") if obj['tuple'].size != fields.size
248
+ rpc_inputs = []
249
+ fields.each do |field|
250
+ rpc_inputs << obj['tuple'][field.keys[0]]
251
+ end
138
252
 
139
253
  display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
140
254
 
141
255
 
142
- # Start communication with API
143
- api = "API #{@__tester.session.api}"
144
- caller = Zillabyte::API::Components.new(api)
145
- component_id = @__node["id"]
146
- output_format = @__node["output_format"]
147
-
148
256
  # Send RPC call
149
- tuple = obj['tuple']
150
257
  cdisplay "sending RPC call..."
151
258
  call_options = {
152
- :rpc_inputs => [tuple],
259
+ :rpc_inputs => [rpc_inputs],
153
260
  :output_format => output_format
154
261
  }
155
262
 
@@ -161,7 +268,10 @@ class Zillabyte::Runner::ComponentOperation
161
268
  cdisplay("error: #{res['error']}")
162
269
  next
163
270
  end
164
- run_ids = res['run_ids']
271
+ run_ids = []
272
+ res['execute_ids'].each do |q, qid|
273
+ run_ids << qid
274
+ end
165
275
  status = res['status']
166
276
 
167
277
 
@@ -275,60 +385,6 @@ class Zillabyte::Runner::ComponentOperation
275
385
 
276
386
  end
277
387
 
278
- # Run a component output
279
- def self.run_output()
280
-
281
-
282
- output = @__options[:output]
283
- header_written = false
284
- read_stream = @__consumee[:rd_child]
285
-
286
- loop do
287
- # Read a tuple
288
- obj = read_message(read_stream)
289
-
290
-
291
- # Add row
292
- if obj['tuple']
293
-
294
- if output
295
- filename = "#{output}.csv"
296
- t = obj['tuple']
297
- m = obj['meta'] || {}
298
-
299
- # Write header
300
- if header_written == false
301
- keys = [t.keys, m.keys].flatten
302
- csv_str = keys
303
- f = File.open(filename, "w")
304
- f.write(csv_str)
305
- f.close()
306
- header_written = true
307
- end
308
-
309
- # Write CSV row
310
- vals = [t.values, m.values].flatten
311
- csv_str = vals.to_csv
312
-
313
- f = File.open(filename, "w")
314
- f.write(csv_str)
315
- f.close()
316
- else
317
-
318
- display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
319
- cdisplay "received #{display_json}"
320
- end
321
-
322
- # Ready for next message
323
- write_message(@__consumee[:wr_child], NEXT_MESSAGE)
324
-
325
- # End cycle
326
- elsif obj['command'] && obj['command'] == "end_cycle"
327
- break
328
- end
329
- end
330
-
331
- end
332
388
 
333
389
 
334
390
  private
@@ -526,4 +582,4 @@ class Zillabyte::Runner::ComponentOperation
526
582
  end
527
583
 
528
584
 
529
- end
585
+ end
@@ -8,9 +8,9 @@ require 'thread'
8
8
  class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
9
9
  include Zillabyte::Helpers
10
10
 
11
- def run (dir = Dir.pwd, session = nil, options = {})
11
+ def run (meta, dir = Dir.pwd, session = nil, options = {})
12
12
 
13
- if session.nil?
13
+ if meta.nil? or session.nil?
14
14
  return
15
15
  end
16
16
 
@@ -22,23 +22,6 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
22
22
  output = options[:output]
23
23
  otype = options[:output_type]
24
24
 
25
- # Get component metadata
26
- meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, @session, {:test => true})
27
- if meta.nil? || meta["nodes"].nil?
28
- error "this is not a valid zillabyte app directory"
29
- exit
30
- end
31
-
32
-
33
- # Check that multilang version is atleast 0.1.0
34
- version = meta["multilang_version"] || "0.0.0"
35
- version_arr = version.split('.').map {|v| v.to_i}
36
- if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
37
- display "The version of zillabyte used in your application is outdated."
38
- display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
39
- return
40
- end
41
-
42
25
  # Show the user what we know about their app...
43
26
  display "inferring your app details..."
44
27
  describe_component(meta)
@@ -156,8 +139,26 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
156
139
  # TODO handle inputs
157
140
  if input.nil?
158
141
 
142
+ source = ""
143
+ @nodes.each do |n|
144
+ name = n["name"]
145
+ type = n["type"]
146
+
147
+ if type == "source"
148
+ if source == ""
149
+ source = name
150
+ else
151
+ display "Cannot run component with multiple input sources without input files"
152
+ return
153
+ end
154
+ end
155
+ end
156
+
157
+ display ""
158
+ display "To view results: Enter 'end' "
159
+ display "To exit the test: Enter 'Ctrl-C' "
160
+ display ""
159
161
 
160
- display "Use Ctrl-C to exit"
161
162
  while true
162
163
 
163
164
  # TODO this doesnt handle multiple sources
@@ -165,15 +166,8 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
165
166
 
166
167
  display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
167
168
  msg = ask
168
-
169
- begin
170
- JSON.parse(msg)
171
- rescue JSON::ParserError
172
- display "Received invalid JSON object"
173
- next
174
- end
175
169
  # Send tuple to source
176
- @operation_pipes["stream_1"][:wr_parent].puts msg
170
+ @operation_pipes[source][:wr_parent].puts msg
177
171
  end
178
172
  end
179
173
 
@@ -212,8 +206,8 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
212
206
  colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
213
207
  rjust = 20
214
208
 
215
- display "#{'app name'.rjust(rjust)}: #{meta['name']}"
216
- display "#{'app language'.rjust(rjust)}: #{meta['language']}"
209
+ display "#{'component name'.rjust(rjust)}: #{meta['name']}"
210
+ display "#{'component language'.rjust(rjust)}: #{meta['language']}"
217
211
  meta['nodes'].each_with_index do |node, index|
218
212
  color = @colors[node['name']] ||= colors[index % colors.length]
219
213
  display (("="*rjust + " operation ##{index}").colorize(color))
@@ -241,4 +235,4 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
241
235
 
242
236
 
243
237
 
244
- end
238
+ end
@@ -361,7 +361,7 @@ class Zillabyte::Runner::MultilangOperation
361
361
 
362
362
  # Multilang sent a ping
363
363
  elsif obj['ping']
364
- write_message(to_ml, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
364
+ write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
365
365
  end
366
366
  end
367
367
 
@@ -433,6 +433,7 @@ class Zillabyte::Runner::MultilangOperation
433
433
  mutlilang_count = 0
434
434
  end_cycle_received = false
435
435
 
436
+
436
437
  # Receive and handle messages
437
438
  loop do
438
439
 
@@ -444,12 +445,13 @@ class Zillabyte::Runner::MultilangOperation
444
445
  if r == stdout
445
446
  msg = r.gets
446
447
  msg = msg.sub(/\n/, "")
447
- cdisplay("log: #{msg}")
448
+ cdisplay("LOG: #{msg}")
448
449
  next
449
450
  end
450
451
 
451
452
  # Receive an object
452
453
  obj = read_message(r)
454
+
453
455
  if obj["command"]
454
456
  case obj["command"]
455
457
 
@@ -540,7 +542,6 @@ class Zillabyte::Runner::MultilangOperation
540
542
  end_cycle_received = true
541
543
 
542
544
  if mutlilang_count == 0
543
-
544
545
  @__emit_queues.each_pair do |stream, consumers|
545
546
  consumers.each_pair do |consumer, emitter|
546
547
  if emitter[:ready]
@@ -571,7 +572,7 @@ class Zillabyte::Runner::MultilangOperation
571
572
 
572
573
  # Multilang sent a ping
573
574
  elsif obj['ping']
574
- write_message(to_ml, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
575
+ write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
575
576
  end
576
577
  end
577
578
 
@@ -751,6 +752,9 @@ class Zillabyte::Runner::MultilangOperation
751
752
  # Ask operation for next tuple
752
753
  write_message(@__consumee[:wr_child], NEXT_MESSAGE)
753
754
 
755
+ # Multilang sent a ping
756
+ elsif obj['ping']
757
+ write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
754
758
  end
755
759
  end
756
760
 
@@ -866,6 +870,7 @@ class Zillabyte::Runner::MultilangOperation
866
870
 
867
871
  if messages.empty?
868
872
  cdisplay "empty relation"
873
+ cdisplay "use Ctrl-C to exit"
869
874
  return
870
875
  end
871
876
 
@@ -913,6 +918,7 @@ class Zillabyte::Runner::MultilangOperation
913
918
  f.close()
914
919
  cdisplay("output written to #{filename}")
915
920
  end
921
+ cdisplay "\n \nPress Ctrl-C to exit"
916
922
  end
917
923
 
918
924
 
@@ -956,6 +962,7 @@ class Zillabyte::Runner::MultilangOperation
956
962
  loop do
957
963
 
958
964
  while !@__read_buffers[read_stream].include? ENDMARKER
965
+
959
966
  segment = read_stream.sysread(BUFSIZE)
960
967
  @__read_buffers[read_stream] << segment
961
968
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zillabyte-cli
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - zillabyte
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-06-26 00:00:00.000000000 Z
11
+ date: 2014-06-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -274,7 +274,6 @@ files:
274
274
  - lib/zillabyte/cli/components.rb
275
275
  - lib/zillabyte/cli/config.rb
276
276
  - lib/zillabyte/cli/counters.rb
277
- - lib/zillabyte/cli/executes.rb
278
277
  - lib/zillabyte/cli/flows.rb
279
278
  - lib/zillabyte/cli/git.rb
280
279
  - lib/zillabyte/cli/help.rb
@@ -294,8 +293,16 @@ files:
294
293
  - lib/zillabyte/cli/templates/apps/python/requirements.txt
295
294
  - lib/zillabyte/cli/templates/apps/python/zillabyte.conf.yaml
296
295
  - lib/zillabyte/cli/templates/apps/ruby/Gemfile
297
- - lib/zillabyte/cli/templates/apps/ruby/app.rb
296
+ - lib/zillabyte/cli/templates/apps/ruby/app.rb.erb
298
297
  - lib/zillabyte/cli/templates/apps/ruby/zillabyte.conf.yaml
298
+ - lib/zillabyte/cli/templates/components/js/simple_function.js
299
+ - lib/zillabyte/cli/templates/components/js/zillabyte.conf.yaml
300
+ - lib/zillabyte/cli/templates/components/python/component.py
301
+ - lib/zillabyte/cli/templates/components/python/requirements.txt
302
+ - lib/zillabyte/cli/templates/components/python/zillabyte.conf.yaml
303
+ - lib/zillabyte/cli/templates/components/ruby/Gemfile
304
+ - lib/zillabyte/cli/templates/components/ruby/component.rb.erb
305
+ - lib/zillabyte/cli/templates/components/ruby/zillabyte.conf.yaml
299
306
  - lib/zillabyte/cli/templates/python/#simple_function.py#
300
307
  - lib/zillabyte/cli/version.rb
301
308
  - lib/zillabyte/cli/zillalogs.rb