zillabyte-cli 0.0.16 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +9 -9
  2. data/lib/#zillabyte-cli.rb# +5 -0
  3. data/lib/zillabyte/api/{flows.rb → apps.rb} +35 -23
  4. data/lib/zillabyte/api/data.rb +12 -1
  5. data/lib/zillabyte/api/logs.rb +4 -4
  6. data/lib/zillabyte/api/queries.rb +15 -6
  7. data/lib/zillabyte/api/zillalogs.rb +1 -1
  8. data/lib/zillabyte/api.rb +40 -38
  9. data/lib/zillabyte/auth.rb +19 -11
  10. data/lib/zillabyte/cli/#logs.rb# +12 -0
  11. data/lib/zillabyte/cli/{flows.rb → apps.rb} +407 -177
  12. data/lib/zillabyte/cli/auth.rb +1 -1
  13. data/lib/zillabyte/cli/base.rb +5 -4
  14. data/lib/zillabyte/cli/config.rb +3 -2
  15. data/lib/zillabyte/cli/counters.rb +1 -1
  16. data/lib/zillabyte/cli/help.rb +1 -1
  17. data/lib/zillabyte/cli/helpers/data_schema_builder.rb +1 -1
  18. data/lib/zillabyte/cli/helpers/table_output_builder.rb +25 -0
  19. data/lib/zillabyte/cli/log_formatter.rb +4 -3
  20. data/lib/zillabyte/cli/query.rb +107 -26
  21. data/lib/zillabyte/cli/relations.rb +226 -78
  22. data/lib/zillabyte/cli/sources.rb +1 -1
  23. data/lib/zillabyte/cli/templates/js/simple_function.js +5 -0
  24. data/lib/zillabyte/cli/templates/python/#simple_function.py# +27 -0
  25. data/lib/zillabyte/cli/templates/python/simple_function.py +3 -0
  26. data/lib/zillabyte/cli/templates/ruby/{simple_function.rb → simple_app.rb} +6 -6
  27. data/lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml +1 -1
  28. data/lib/zillabyte/cli/version.rb +1 -1
  29. data/lib/zillabyte/cli/zillalogs.rb +1 -1
  30. data/lib/zillabyte/command.rb +10 -2
  31. data/lib/zillabyte/common/{progress.rb → session.rb} +1 -1
  32. data/lib/zillabyte/helpers.rb +9 -4
  33. data/lib/zillabyte-cli/version.rb +1 -1
  34. data/zillabyte-cli.gemspec +2 -0
  35. metadata +25 -7
@@ -8,26 +8,31 @@ require 'securerandom'
8
8
  require 'colorize'
9
9
  require 'time_difference'
10
10
 
11
- # manage custom flows
11
+ # manage custom apps
12
12
  #
13
- class Zillabyte::Command::Flows < Zillabyte::Command::Base
13
+ class Zillabyte::Command::Apps < Zillabyte::Command::Base
14
14
 
15
- # flows
15
+ MAX_POLL_SECONDS = 60 * 5
16
+ POLL_SLEEP = 0.5
17
+
18
+ # apps
16
19
  #
17
- # list custom flows
20
+ # list custom apps
21
+ # --type TYPE # specify an output type i.e. json
18
22
  #
19
23
  def index
20
24
  self.list
21
25
  end
22
26
 
23
- # flows
24
- #
25
- # list custom flows
27
+ # apps
26
28
  #
29
+ # list custom apps
30
+ # --type TYPE # specify an output type i.e. json
27
31
  def list
32
+ type = options[:type]
28
33
 
29
34
  headings = ["id", "name", "state", "cycles"]
30
- rows = api.flow.list.map do |row|
35
+ rows = api.app.list.map do |row|
31
36
  if headings.size == 0
32
37
  headings = row.keys
33
38
  headings.delete("rel_dir")
@@ -35,137 +40,164 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
35
40
  v = row.values_at *headings
36
41
  v
37
42
  end
38
- display "flows:\n" + Terminal::Table.new(:headings => headings, :rows => rows).to_s
39
- display "Total number of flows: "+rows.length.to_s
43
+
44
+ display "apps:\n" if type.nil?
45
+ display TableOutputBuilder.build_table(headings, rows, type)
46
+ display "Total number of apps: "+rows.length.to_s if type.nil?
40
47
  end
41
48
 
42
49
 
43
- # flows:push [DIR]
50
+ # apps:push [DIR]
44
51
  #
45
- # uploads a flow
52
+ # uploads an app
46
53
  #
47
54
  # --config CONFIG_FILE # use the given config file
55
+ # --type TYPE # specify an output type i.e. json
48
56
  #
49
57
  #Examples:
50
58
  #
51
- # $ zillabyte flows:push .
59
+ # $ zillabyte apps:push .
52
60
  #
53
61
  def push
54
62
 
55
63
  since = Time.now.utc.to_s
56
64
  dir = options[:directory] || shift_argument || Dir.pwd
57
- res = api.flows.push_directory dir, progress, options
65
+ type = options[:type]
66
+
67
+ res = api.apps.push_directory dir, session, options
58
68
 
59
69
  if res['error']
60
- display "error: #{res['error_message']}"
70
+ error("error: #{res['error_message']}", type)
61
71
  else
62
- display "flow ##{res['id']} #{res['action']}"
72
+ display "app ##{res['id']} #{res['action']}" if type.nil?
63
73
  end
64
74
 
65
- display "Starting up your flow...please wait..."
75
+ display "Starting up your app...please wait..." if type.nil?
66
76
  sleep(2) # wait for kill command
67
77
 
68
78
  lf = LogFormatter::Startup.new
69
79
  api.logs.get_startup(res['id'], "_ALL_", {:push => true}) do |hash|
70
80
 
71
81
  # Error?
72
- error hash['error_message'] if hash['error']
82
+ error(hash['error_message'], type) if hash['error']
73
83
 
74
84
  # Print it
75
- lf.print_log_line(hash)
85
+ lf.print_log_line(hash) if type.nil?
76
86
 
77
87
  # Exit when we get the 'done' message
78
- # exit(0) if (hash['line'] || '').downcase.include?("flow deployed")
88
+ # exit(0) if (hash['line'] || '').downcase.include?("app deployed")
79
89
 
80
90
  end
91
+
92
+ display {}.to_json if type == "json"
81
93
 
82
94
  end
83
- alias_command "push", "flows:push"
95
+ alias_command "push", "apps:push"
84
96
 
85
97
 
86
98
 
87
- # flows:pull ID DIR
99
+ # apps:pull ID DIR
88
100
  #
89
- # pulls a flow source to a directory.
101
+ # pulls an app source to a directory.
90
102
  #
91
103
  # --force # pulls even if the directory exists
104
+ # --type TYPE # specify an output type i.e. json
92
105
  #
93
106
  #Examples:
94
107
  #
95
- # $ zillabyte flows:pull .
108
+ # $ zillabyte apps:pull .
96
109
  #
97
110
  def pull
98
111
 
99
112
  id = options[:id] || shift_argument
100
113
  dir = options[:directory] || shift_argument
114
+ type = options[:type]
115
+
101
116
 
102
- error "no id given" if id.nil?
103
- error "no directory given" if dir.nil?
117
+ error("no id given", type) if id.nil?
118
+ error("no directory given", type) if dir.nil?
104
119
 
105
120
  # Create if not exists..
106
121
  if File.exists?(dir)
107
122
  if Dir.entries(dir).size != 2 and options[:force].nil?
108
- error "target directory not empty. use --force to override"
123
+ error("target directory not empty. use --force to override", type)
109
124
  end
110
125
  else
111
126
  FileUtils.mkdir_p(dir)
112
127
  end
113
128
 
114
- res = api.flows.pull_to_directory id, dir, progress
129
+ res = api.apps.pull_to_directory id, dir, session
115
130
 
116
131
  if res['error']
117
- display "error: #{res['error_message']}"
132
+ error("error: #{res['error_message']}", type)
118
133
  else
119
- display "flow ##{res['id']} pulled to #{dir}"
134
+ if type == "json"
135
+ display {}.to_json
136
+ else
137
+ display "app ##{res['id']} pulled to #{dir}"
138
+ end
120
139
  end
121
140
 
122
141
  end
123
- alias_command "pull", "flows:pull"
142
+ alias_command "pull", "apps:pull"
124
143
 
125
144
 
126
- # flows:delete ID
145
+ # apps:delete ID
127
146
  #
128
- # deletes a flow. if the flow is running, this command will kill it.
147
+ # deletes an app. if the app is running, this command will kill it.
129
148
  #
130
149
  # -f, --force # don't ask for confirmation
150
+ # --type TYPE # specify an output type i.e. json
151
+ #
131
152
  def delete
132
153
  id = options[:id] || shift_argument
133
154
  if id.nil?
134
155
  id = read_name_from_conf(options)
135
156
  options[:flow_name] = true
136
157
  end
137
-
138
158
  forced = options[:force]
159
+ type = options[:type]
160
+
139
161
  if not forced
162
+
163
+ if !type.nil?
164
+ error("specify -f, --force to confirm deletion", type)
165
+ end
166
+
140
167
  while true
141
168
 
142
- display "This operation cannot be undone. Are you sure you want to delete this flow? (yes/no):", false
169
+ display "This operation cannot be undone. Are you sure you want to delete this app? (yes/no):", false
143
170
  confirm = ask
144
171
  break if confirm == "yes" || confirm == "no"
145
- display "Please enter 'yes' to delete the flow or 'no' to exit"
172
+ display "Please enter 'yes' to delete the app or 'no' to exit"
146
173
  end
147
174
  end
148
175
 
149
176
  confirmed = forced || confirm == "yes"
150
177
 
151
178
  if confirmed
152
- response = api.flows.delete(id, options)
153
- display response["body"]
154
-
179
+ response = api.apps.delete(id, options)
180
+ if type == "json"
181
+ display {}.to_json
182
+ else
183
+ display response["body"]
184
+ end
155
185
  end
156
186
 
157
187
  end
158
188
 
159
189
 
160
- # flows:prep [DIR]
190
+ # apps:prep [DIR]
191
+ #
192
+ # prepares an app for execution
161
193
  #
162
- # prepares a flow for execution
163
194
  #
164
195
  def prep
165
196
 
166
197
  dir = options[:directory] || shift_argument || Dir.pwd
198
+
167
199
  meta = Zillabyte::CLI::Config.get_config_info(dir)
168
-
200
+
169
201
  case meta["language"]
170
202
  when "ruby"
171
203
 
@@ -192,29 +224,33 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
192
224
  end
193
225
 
194
226
  end
195
- alias_command "prep", "flows:prep"
227
+ alias_command "prep", "apps:prep"
196
228
 
197
229
 
198
230
 
199
231
 
200
- # flows:init [LANG] [DIR]
232
+ # apps:init [LANG] [DIR]
201
233
  #
202
234
  # initializes a new executable in DIR
203
235
  # [LANG] defaults to ruby, and [DIR] to the current directory
204
236
  #
237
+ # --type TYPE # specify an output type i.e. json
238
+ #
205
239
  #Examples:
206
240
  #
207
- # $ zillabyte flows:init python contact_extractor
241
+ # $ zillabyte apps:init python contact_extractor
208
242
  #
209
243
  def init
210
244
 
211
245
  lang = options[:lang] || shift_argument || "ruby"
212
246
  dir = options[:dir] || shift_argument || Dir.pwd
247
+ type = options[:type]
248
+
213
249
  languages = ["ruby","python", "js"]
214
250
 
215
- error "Unsupported language #{lang}. We only support #{languages.join(', ')}." if not languages.include? lang
251
+ error("Unsupported language #{lang}. We only support #{languages.join(', ')}.", type) if not languages.include? lang
216
252
 
217
- display "initializing empty #{lang} flow in #{dir}"
253
+ display "initializing empty #{lang} app in #{dir}" if type.nil?
218
254
  FileUtils.cp_r( File.expand_path("../templates/#{lang}", __FILE__) + "/." , dir )
219
255
 
220
256
 
@@ -222,66 +258,133 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
222
258
 
223
259
 
224
260
 
225
- # flows:logs FLOW_ID [OPERATION_NAME]
261
+ # apps:logs APP_ID [OPERATION_NAME]
226
262
  #
227
263
  # streams logs from the distributed workers
228
264
  #
265
+ # --type TYPE # specify an output type i.e. json
229
266
  # -v, --verbose LEVEL # sets the verbosity (error, info, debug) (default: info)
230
267
  #
231
268
  def logs
232
269
 
233
- flow_id = options[:flow] || shift_argument
270
+ app_id = options[:app] || shift_argument
234
271
  operation_id = options[:operation] || shift_argument || '_ALL_'
235
272
  category = options[:verbose] || '_ALL_'
273
+ type = options[:type]
274
+
236
275
  carry_settings = {
237
276
  :category => category
238
277
  }
239
278
  api_options = {}
240
279
 
241
- if flow_id.nil?
242
- flow_id = read_name_from_conf(options)
243
- api_options["flow_name"] = true
280
+ if app_id.nil?
281
+ app_id = read_name_from_conf(options)
282
+ api_options[:flow_name] = true
244
283
  end
245
284
 
246
- display "Retrieving logs for flow ##{flow_id}...please wait..."
285
+ display "Retrieving logs for app ##{app_id}...please wait..." if type.nil?
247
286
  lf = LogFormatter::Operation.new
248
- self.api.logs.get(flow_id, operation_id, api_options) do |line|
287
+ self.api.logs.get(app_id, operation_id, api_options) do |line|
249
288
 
250
- error line['error_message'] if line['error']
251
- lf.print_log_line(line)
289
+ error(line['error_message'], type) if line['error']
290
+ lf.print_log_line(line) if type.nil?
252
291
 
253
292
  end
254
293
 
255
294
  end
256
- alias_command "logs", "flows:logs"
295
+ alias_command "logs", "apps:logs"
257
296
 
258
297
 
259
- # flows:cycles ID [OPTIONS]
298
+
299
+ # apps:errors ID
300
+ #
301
+ # Show recent errors generated by the app
302
+ # --type TYPE # specify an output type i.e. json
303
+ #
304
+ def errors
305
+
306
+ # Init
307
+ app_id = options[:id] || shift_argument
308
+
309
+ # No name?
310
+ if app_id.nil?
311
+ app_id = read_name_from_conf(options)
312
+ options[:flow_name] = true
313
+ end
314
+
315
+ type = options[:type]
316
+
317
+ # Make the request
318
+ res = api.request(
319
+ :expects => 200,
320
+ :method => :get,
321
+ :body => options.to_json,
322
+ :path => "/flows/#{CGI.escape(app_id)}/errors"
323
+ )
324
+
325
+ # Render
326
+ display "Recent errors:" if type.nil?
327
+ headings = ["operation", "date", "error"]
328
+ rows = (res.body["recent_errors"] || []).map do |row|
329
+ if row['date']
330
+ d = Time.at(row['date']/1000)
331
+ else
332
+ d = nil
333
+ end
334
+ [row['name'], d, row['message']]
335
+ end
336
+ rows.sort! do |a,b|
337
+ a[1] <=> b[1]
338
+ end
339
+ color_map = {}
340
+ colors = LogFormatter::COLORS.clone
341
+ rows.each do |row|
342
+ name = row[0]
343
+ time = row[1]
344
+ message = row[2].strip
345
+ color_map[name] ||= colors.shift
346
+ if time
347
+ display "#{"* #{name} - #{time_ago_in_words(time)} ago".colorize(color_map[name])}:" if type.nil?
348
+ else
349
+ display "#{"* #{name}".colorize(color_map[name])}:" if type.nil?
350
+ end
351
+ message.split('\n').each do |sub_line|
352
+ display " #{sub_line}" if type.nil?
353
+ end
354
+ end
355
+
356
+ end
357
+
358
+ # apps:cycles ID [OPTIONS]
260
359
  #
261
- # operations on the flow's cycles (batches).
262
- # with no options, the command lists the flows cycles
263
- # -n, --next # request the flow to move to the next cycle
264
- # -f, --forever # don't wait on cycles any more
360
+ # operations on the app's cycles (batches).
361
+ # with no options, the command lists the apps cycles
362
+ # -n, --next # request the app to move to the next cycle
363
+ # -f, --forever # don't wait on cycles any more
364
+ # --type TYPE # specify an output type i.e. json
365
+ #
265
366
  def cycles
266
- flow_id = options[:id] || shift_argument
367
+ app_id = options[:id] || shift_argument
368
+ type = options[:type]
369
+
267
370
  trigger_next = options[:next] || false
268
371
  trigger_forever = options[:forever] || false
269
372
 
270
- if flow_id.nil?
271
- flow_id = read_name_from_conf(options)
373
+ if app_id.nil?
374
+ app_id = read_name_from_conf(options)
272
375
  options[:flow_name] = true
273
376
  end
274
377
 
275
378
  if trigger_next
276
- # Trigger the next flow
277
- response = api.flows.create_cycle(flow_id, options)
379
+ # Trigger the next app
380
+ response = api.apps.create_cycle(app_id, options)
278
381
  elsif trigger_forever
279
- response = api.flows.run_forever(flow_id, options)
382
+ response = api.apps.run_forever(app_id, options)
280
383
  else
281
- # List the flows
282
- response = api.flows.list_cycles(flow_id, options)
283
- # TODO List the sequence number for this flow.
284
- display "Most recent cyles of the flow:"
384
+ # List the apps
385
+ response = api.apps.list_cycles(app_id, options)
386
+ # TODO List the sequence number for this app.
387
+ display "Most recent cyles of the app:" if type.nil?
285
388
  headings = ["Cycle_id", "State", "Start", "End"]
286
389
  rows = response["cycles"]
287
390
  rows = rows.map do |row|
@@ -289,20 +392,57 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
289
392
  end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
290
393
  [row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
291
394
  end
292
- display Terminal::Table.new(:headings => headings, :rows => rows).to_s
293
- display "Total number of cycles executed: #{response['total']}"
395
+
396
+ display TableOutputBuilder.build_table(headings, rows, type)
397
+ display "Total number of cycles executed: #{response['total']}" if type.nil?
398
+ return
294
399
  end
295
- if response["body"]
296
- display response["body"]
297
- elsif response["error"]
298
- error response["error"]
400
+
401
+ if response["job_id"]
402
+ options[:job_id] = response["job_id"]
403
+ app_id = response["flow_id"]
404
+ options.delete :flow_name
405
+
406
+ start = Time.now.utc
407
+ display "Next cycle request sent. If your app was RETIRED this may take slightly longer." if type.nil?
408
+
409
+ while(Time.now.utc < start + MAX_POLL_SECONDS) do
410
+
411
+ # Poll
412
+ res = self.api.apps.cycles_poll(app_id, options)
413
+
414
+ case res['status']
415
+ when 'completed'
416
+ if res['return']
417
+ if type == "json"
418
+ return {}.to_json
419
+ else
420
+ display res['return']
421
+ end
422
+ else
423
+ throw "something is wrong: #{res}"
424
+ end
425
+ # success! continue below
426
+ break
427
+ when 'running'
428
+ sleep(POLL_SLEEP)
429
+ # display ".", false
430
+ else
431
+ throw "unknown status: #{res}"
432
+ end
433
+
434
+ end
435
+ elsif response["error"]
436
+ error(response["error"], type)
437
+ else
438
+ error("remote server error (a380)", type)
299
439
  end
300
440
  end
301
441
 
302
442
 
303
- # flows:test
443
+ # apps:test
304
444
  #
305
- # tests a local flow with sample data
445
+ # tests a local app with sample data
306
446
  #
307
447
  # --config CONFIG_FILE # use the given config file
308
448
  # --output OUTPUT_FILE # writes sink output to a file
@@ -312,6 +452,8 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
312
452
  def test
313
453
 
314
454
  output = options[:output]
455
+ type = options[:type]
456
+
315
457
  max_seconds = (options[:wait] || "30").to_i
316
458
  batches = (options[:batches] || "1").to_i
317
459
 
@@ -364,33 +506,34 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
364
506
  # INIT
365
507
  dir = options[:dir] || Dir.pwd
366
508
 
367
- meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, self, {:test => true})
509
+ meta = Zillabyte::API::Apps.get_rich_meta_info_from_script(dir, self, {:test => true})
368
510
  if meta.nil?
369
- error "this is not a valid zillabyte flow directory"
511
+ error "this is not a valid zillabyte app directory"
370
512
  exit
371
513
  end
372
514
 
373
- # Show the user what we know about their flow...
374
- display "inferring your flow details..."
515
+ # Show the user what we know about their app...
516
+ display "inferring your app details..."
375
517
  colors = {}
376
- describe_flow(meta, colors)
518
+ describe_app(meta, colors)
377
519
 
378
520
 
379
- # Extract the flow's information..
521
+ # Extract the app's information..
380
522
  nodes = meta["nodes"]
381
523
  write_to_next_each = []
382
524
  write_queue = []
383
525
  stream_messages = {}
384
526
  default_stream = "_default"
385
527
 
386
- split_branches = false
387
-
388
528
  # Iterate all nodes sequentially and invoke them in separate processes...
389
529
  nodes.each do |node|
390
530
 
391
531
  # Init
392
532
  type = node["type"]
393
533
  name = node["name"]
534
+ consumes = node["consumes"]
535
+ emits = node["emits"]
536
+
394
537
  color = colors[name] || :default
395
538
 
396
539
  op_display = lambda do |msg, override_color = nil|
@@ -403,6 +546,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
403
546
  # A source from relation?
404
547
  if node['matches'] or node["relation"]
405
548
  matches = node['matches'] || (node["relation"]["query"])
549
+ emits = emits.first #For spouting from a relation, there should only be one emits
406
550
  op_display.call "Grabbing remote data"
407
551
 
408
552
  res = api.query.agnostic(matches)
@@ -427,9 +571,9 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
427
571
  end
428
572
  read_msg = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
429
573
  values = Hash[values.map{|k, v| [truncate_message(k), truncate_message(v)]}]
430
- op_display.call "emit tuple: #{values} #{meta}"
431
- stream_messages[default_stream] ||= []
432
- stream_messages[default_stream] << read_msg
574
+ op_display.call "emitted: #{values} #{meta} to #{emits}"
575
+ stream_messages[emits] ||= []
576
+ stream_messages[emits] << read_msg
433
577
  end
434
578
 
435
579
  # Done processing...
@@ -439,10 +583,14 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
439
583
 
440
584
  # A regular source..
441
585
  stream_messages[default_stream] ||= []
442
- batches.times do |i|
443
- stream_messages[default_stream] << "{\"command\": \"next\"}\n"
444
- end
445
-
586
+ stream_messages[default_stream] << "{\"command\": \"begin_cycle\"}\n"
587
+ emits.each {|ss| stream_messages[ss] = []} #initialize streams
588
+ stream_size_at_last_call_to_next_tuple = Hash[emits.map {|ss| [ss, 0]}] #initialize initial size of streams (all 0)
589
+ # the above initializations are used to deal with the case where end_cycle_policy == "null_emit"
590
+ n_batches_emitted = 1
591
+ end_cycle_received = false
592
+ last_call_next_tuple = false
593
+
446
594
  end
447
595
 
448
596
  # An Aggregate?
@@ -488,16 +636,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
488
636
  # A Sink?
489
637
  elsif type == "sink"
490
638
 
491
- if split_branches || stream_messages.size > 1
492
- split_branches = true
493
- if node['consumes'].nil?
494
- error "The node #{name} must declare which stream it 'consumes'"
495
- end
496
- sink_stream = node["consumes"]
497
- messages = stream_messages[sink_stream] || []
498
- else
499
- messages = stream_messages.values.first || []
639
+ if consumes.nil?
640
+ error "The node #{name} must declare which stream it 'consumes'"
500
641
  end
642
+ messages = stream_messages[consumes] || []
501
643
 
502
644
  table = Terminal::Table.new :title => name
503
645
  csv_str = CSV.generate do |csv|
@@ -529,8 +671,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
529
671
  op_display.call "output written to #{filename}"
530
672
  end
531
673
 
532
-
533
-
534
674
  next
535
675
  end
536
676
 
@@ -548,28 +688,32 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
548
688
  write_queue = []
549
689
  read_queue = []
550
690
 
551
- # Get the incoming stream
552
- if !split_branches && stream_messages.size == 1 # i.e. the number of streams we're dealing with right now
553
- # Assume default stream
554
- stream_name = stream_messages.keys.first
555
- write_queue = stream_messages.values.first.clone
556
- stream_messages.delete(stream_name)
691
+ if consumes.nil?
692
+ # Assume default stream (this should only happen for the source)
693
+ stream_name = default_stream
557
694
  else
558
- # Multiple streams...
559
- split_branches = true;
560
- if node['consumes'].nil?
561
- error "The node #{name} must declare which stream it 'consumes'"
562
- end
563
- stream_name = node["consumes"]
564
- write_queue = stream_messages[stream_name].clone()
565
- stream_messages.delete(stream_name)
695
+ stream_name = consumes
566
696
  end
697
+ write_queue = stream_messages[stream_name].clone
698
+ stream_messages.delete(stream_name)
567
699
 
568
700
  # Start writing the messages...
569
701
  stuff_to_read = false
570
702
  writing_thread = Thread.start do
571
- until(write_queue.empty?)
572
-
703
+
704
+ while(true)
705
+
706
+ case type
707
+ when 'source'
708
+ break if n_batches_emitted > batches
709
+ if write_queue.empty?
710
+ sleep 0.5
711
+ next
712
+ end
713
+ else
714
+ break if write_queue.empty?
715
+ end
716
+
573
717
  # Make sure we're not reading anything...
574
718
  while(stuff_to_read) # TODO: semaphores
575
719
  sleep 0.5 # spin wait
@@ -584,7 +728,8 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
584
728
  display_hash = Hash[write_json['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
585
729
  op_display.call "receiving: #{display_hash}"
586
730
  elsif write_json['command'] == 'next'
587
- op_display.call "starting next source batch"
731
+ last_call_next_tuple = true
732
+ op_display.call "getting next set of tuples in the batch"
588
733
  else
589
734
  puts write_json
590
735
  end
@@ -605,10 +750,43 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
605
750
  reading_thread = Thread.start do
606
751
  while(true)
607
752
 
753
+ # If the end cycle command is received, we either trigger the next cycle if the number of emitted
754
+ # cycles is less than what the user requested, or we break
755
+ if type == "source" and end_cycle_received
756
+ write_queue << "{\"command\": \"begin_cycle\"}\n"
757
+ n_batches_emitted += 1
758
+ end_cycle_received = false
759
+ last_call_next_tuple = false
760
+ stuff_to_read = false
761
+ break if n_batches_emitted > batches
762
+ sleep 0.5
763
+ next
764
+ end
765
+
608
766
  # Get next message
609
767
  read_msg = read_message(stdout, color)
610
768
  if read_msg == "done" || read_msg.nil?
611
769
  stuff_to_read = false
770
+
771
+ # For sources, if we receive a "done", check to see if any of the streams emitted by the source has
772
+ # increased in size since the last call to next_tuple. If so, the cycle isn't over, otherwise, the
773
+ # current call to next_tuple emitted nothing and if the end_cycle_policy is set to null_emit, this
774
+ # should end the current cycle.
775
+ if type == "source"
776
+ if last_call_next_tuple and node["end_cycle_policy"] == "null_emit"
777
+ end_cycle_received = true
778
+ emits.each do |ss|
779
+ end_cycle_received = false if stream_messages[ss].size > stream_size_at_last_call_to_next_tuple[ss]
780
+ break
781
+ end
782
+ next if end_cycle_received
783
+ end
784
+
785
+ # If the policy isn't "null_emit", then just request next_tuple again
786
+ write_queue << "{\"command\": \"next\"}\n"
787
+ end
788
+
789
+ # For other operations, if the queue is empty then we're done
612
790
  if write_queue.empty?
613
791
  break # exit while loop
614
792
  else
@@ -620,20 +798,49 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
620
798
 
621
799
  # Process message
622
800
  obj = JSON.parse(read_msg)
801
+
802
+ # process the received tuple or other commands
623
803
  if obj['tuple']
624
-
625
- # Conver to a incoming tuple for the next operation
804
+
805
+ # if
806
+ tt = obj['tuple']
807
+ tt.each do |kk, vv|
808
+ if tt[kk].nil? and type == "source" and node["end_cycle_policy"] == "null_emit"
809
+ end_cycle_received = true
810
+ # read rest of stuff in stdout buffer until "done"
811
+ mm = nil
812
+ while(mm != "done")
813
+ mm = read_message(stdout, color)
814
+ end
815
+ break
816
+ end
817
+ end
818
+ next if end_cycle_received
819
+
820
+ # Convert to a incoming tuple for the next operation
626
821
  next_msg = {
627
822
  :tuple => obj['tuple'],
628
823
  :meta => obj['meta']
629
824
  }
630
- emit_stream = obj['stream'] || default_stream
825
+ emit_stream = obj['stream']
631
826
  stream_messages[emit_stream] ||= []
632
827
  stream_messages[emit_stream] << next_msg.to_json
828
+
633
829
  display_hash = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
634
830
  op_display.call "emitted: #{display_hash} to #{emit_stream}"
831
+
832
+ # track stream message size to end cycles when necessary
833
+ stream_size_at_last_call_to_next_tuple[emit_stream] = stream_messages[emit_stream].size if type == "source"
834
+ elsif obj['command'] == 'end_cycle'
835
+ end_cycle_received = true
836
+ # command:end_cycle should always be followed by done, read it (below) so
837
+ # that it doesn't interfere with next
838
+ read_message(stdout, color)
635
839
  elsif obj['command'] == 'log'
636
840
  op_display.call "log: #{obj['msg']}"
841
+ elsif obj['command'] == 'fail'
842
+ op_display.call "error: #{obj['msg']}", :red
843
+ exit(1)
637
844
  else
638
845
  error "unknown message: #{read_msg}"
639
846
  end
@@ -641,7 +848,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
641
848
  end
642
849
  end
643
850
 
644
-
645
851
  # stderr thread
646
852
  stderr_thread = Thread.start do
647
853
  stderr.each do |line|
@@ -670,60 +876,61 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
670
876
  end
671
877
  rescue PTY::ChildExited
672
878
  puts "The child process exited!"
673
- ensure
674
- # We need to check here to see if we need to turn on multistream mode.
675
- # Note that we still need the redundant checks above, in case the user
676
- # didn't honor the emits contract.
677
- if node["emits"] && node["emits"].size() > 1
678
- split_branches = true
679
- end
680
879
  end
681
880
  end
682
881
 
683
882
  end
684
- alias_command "test", "flows:test"
883
+ alias_command "test", "apps:test"
685
884
 
686
885
 
687
- # flows:kill ID
886
+ # apps:kill ID
688
887
  #
689
- # kills the given flow
888
+ # kills the given app
690
889
  #
691
890
  # --config CONFIG_FILE # use the given config file
891
+ # --type TYPE # specify an output type i.e. json
692
892
  #
693
893
  def kill
694
894
 
695
895
  id = options[:id] || shift_argument
896
+ type = options[:type]
897
+
696
898
  if id.nil?
697
899
  id = read_name_from_conf(options)
698
900
  options[:flow_name] = true
699
901
  end
700
902
 
701
- display "Killing flow ##{id}...please wait..."
702
- api.flows.kill(id, options)
703
- display "Flow ##{id} killed"
704
-
705
- end
706
-
707
-
708
-
903
+ display "Killing app ##{id}...please wait..." if type.nil?
904
+ api.apps.kill(id, options)
709
905
 
906
+ if type == "json"
907
+ display {}.to_json
908
+ else
909
+ display "App ##{id} killed"
910
+ end
710
911
 
912
+ end
711
913
 
712
- # flows:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]
914
+
915
+ # apps:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]
713
916
  #
714
- # runs a local flow with live data
917
+ # runs a local app with live data
715
918
  #
716
919
  # --config CONFIG_FILE # use the given config file
920
+ # --type TYPE # specify an output type i.e. json
921
+ #
717
922
  # HIDDEN:
718
923
  def live_run
719
924
 
720
925
  name = options[:name] || shift_argument
926
+ type = options[:type]
927
+
721
928
  thread_id = options[:thread] || shift_argument || ""
722
929
  dir = options[:directory] || shift_argument || Dir.pwd
723
930
  meta = Zillabyte::CLI::Config.get_config_info(dir)
724
931
 
725
932
  if meta.nil?
726
- throw "could not find meta information for: #{dir}"
933
+ error("could not find meta information for: #{dir}", type)
727
934
  end
728
935
 
729
936
  if(thread_id == "")
@@ -732,73 +939,91 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
732
939
  exec(command("--execute_live --name #{name.to_s} --pipe #{thread_id}"))
733
940
  end
734
941
  end
735
- alias_command "live_run", "flows:live_run"
942
+ alias_command "live_run", "apps:live_run"
736
943
 
737
944
 
738
- # flows:info [DIR]
945
+ # apps:info [DIR]
739
946
  #
740
- # outputs the info for the flow in the dir.
947
+ # outputs the info for the app in the dir.
741
948
  #
742
949
  # --pretty # Pretty prints the info output
950
+ # --type TYPE # specify an output type i.e. json
743
951
  #
744
952
  def info
745
953
  info_file = SecureRandom.uuid
954
+ type = options[:type]
955
+
746
956
  cmd = command("--info --file #{info_file}")
747
- flow_info = Zillabyte::Command::Flows.get_info(cmd, info_file)
957
+ app_info = Zillabyte::Command::Apps.get_info(cmd, info_file)
748
958
 
749
- if options[:pretty]
750
- puts JSON.pretty_generate(JSON.parse(flow_info))
959
+ if type == "json"
960
+ puts app.info
751
961
  else
752
- puts flow_info
962
+ if options[:pretty]
963
+ puts JSON.pretty_generate(JSON.parse(app_info))
964
+ else
965
+ puts app_info
966
+ end
753
967
  end
968
+
754
969
  exit
755
970
  end
756
- alias_command "info", "flows:info"
971
+ alias_command "info", "apps:info"
757
972
 
758
973
 
974
+ #
975
+ # --type TYPE # specify an output type i.e. json
976
+ #
759
977
  def self.get_info(cmd, info_file, options = {})
978
+ type = options[:type]
760
979
  response = `#{cmd}`
761
980
  if($?.exitstatus == 1)
762
981
  File.delete("#{info_file}") if File.exists?(info_file)
763
- puts "error: #{response}" if options[:test]
764
- Process.exit 1
982
+ if options[:type].nil?
983
+ exit(1)
984
+ else
985
+ Zillabyte::Helpers.error("error: #{response}", type)
986
+ end
765
987
  end
766
988
 
767
- flow_info = {}
989
+ app_info = {}
768
990
  File.open("#{info_file}", "r+").each do |line|
769
991
  line = JSON.parse(line)
770
992
  if(line["type"])
771
- flow_info["nodes"] << line
993
+ app_info["nodes"] << line
772
994
  else
773
- flow_info = line
774
- flow_info["nodes"] = []
995
+ app_info = line
996
+ app_info["nodes"] = []
775
997
  end
776
998
  end
777
999
  File.delete("#{info_file}")
778
1000
 
779
- flow_info = flow_info.to_json
1001
+ app_info = app_info.to_json
780
1002
  if(File.exists?("info_to_java.in"))
781
1003
  java_pipe = open("info_to_java.in","w+")
782
- java_pipe.puts(flow_info+"\n")
1004
+ java_pipe.puts(app_info+"\n")
783
1005
  java_pipe.flush
784
1006
  java_pipe.close()
785
1007
  end
786
1008
 
787
- flow_info
1009
+ app_info
788
1010
  end
789
1011
 
790
1012
 
791
1013
  private
792
1014
 
793
-
794
-
1015
+ #
1016
+ #
1017
+ # --type TYPE # specify an output type i.e. json
1018
+ #
795
1019
  def command(arg="--execute_live", ignore_stderr = false)
796
1020
 
797
1021
  dir = options[:directory] || shift_argument || Dir.pwd
798
- meta = Zillabyte::CLI::Config.get_config_info(dir, progress=nil, options)
1022
+ type = options[:type]
1023
+ meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
799
1024
  #meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
800
- error "could not extract meta information. missing zillabyte.conf.yml?" if meta.nil?
801
- error meta["error_message"] if meta['status'] == "error"
1025
+ error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
1026
+ error(meta["error_message"], type) if meta['status'] == "error"
802
1027
  full_script = File.join(dir, meta["script"])
803
1028
  stderr_opt = "2> /dev/null" if ignore_stderr
804
1029
 
@@ -819,19 +1044,21 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
819
1044
  cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
820
1045
 
821
1046
  else
822
- error "no language specified"
1047
+ error("no language specified", type)
823
1048
  end
824
1049
 
825
1050
  return cmd
826
1051
 
827
1052
  end
828
1053
 
829
-
830
- def describe_flow(meta, colors = {})
1054
+ #
1055
+ #
1056
+ #
1057
+ def describe_app(meta, colors = {})
831
1058
  @colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
832
1059
  rjust = 20
833
- display "#{'flow name'.rjust(rjust)}: #{meta['name']}"
834
- display "#{'flow language'.rjust(rjust)}: #{meta['language']}"
1060
+ display "#{'app name'.rjust(rjust)}: #{meta['name']}"
1061
+ display "#{'app language'.rjust(rjust)}: #{meta['language']}"
835
1062
 
836
1063
  meta['nodes'].each_with_index do |node, index|
837
1064
  colors[node['name']] ||= @colors.shift
@@ -845,9 +1072,12 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
845
1072
 
846
1073
  end
847
1074
 
1075
+ #
1076
+ #
848
1077
  def read_name_from_conf(options = {})
849
- hash = Zillabyte::API::Flows.get_rich_meta_info_from_script Dir.pwd, options
850
- error "No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow." if hash["error"]
1078
+ type = options[:type]
1079
+ hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
1080
+ error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify an app id or run command from the directory containing the app.",type) if hash["error"]
851
1081
  hash["name"]
852
1082
  end
853
1083