zillabyte-cli 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +9 -9
- data/lib/#zillabyte-cli.rb# +5 -0
- data/lib/zillabyte/api/{flows.rb → apps.rb} +35 -23
- data/lib/zillabyte/api/data.rb +12 -1
- data/lib/zillabyte/api/logs.rb +4 -4
- data/lib/zillabyte/api/queries.rb +15 -6
- data/lib/zillabyte/api/zillalogs.rb +1 -1
- data/lib/zillabyte/api.rb +40 -38
- data/lib/zillabyte/auth.rb +19 -11
- data/lib/zillabyte/cli/#logs.rb# +12 -0
- data/lib/zillabyte/cli/{flows.rb → apps.rb} +407 -177
- data/lib/zillabyte/cli/auth.rb +1 -1
- data/lib/zillabyte/cli/base.rb +5 -4
- data/lib/zillabyte/cli/config.rb +3 -2
- data/lib/zillabyte/cli/counters.rb +1 -1
- data/lib/zillabyte/cli/help.rb +1 -1
- data/lib/zillabyte/cli/helpers/data_schema_builder.rb +1 -1
- data/lib/zillabyte/cli/helpers/table_output_builder.rb +25 -0
- data/lib/zillabyte/cli/log_formatter.rb +4 -3
- data/lib/zillabyte/cli/query.rb +107 -26
- data/lib/zillabyte/cli/relations.rb +226 -78
- data/lib/zillabyte/cli/sources.rb +1 -1
- data/lib/zillabyte/cli/templates/js/simple_function.js +5 -0
- data/lib/zillabyte/cli/templates/python/#simple_function.py# +27 -0
- data/lib/zillabyte/cli/templates/python/simple_function.py +3 -0
- data/lib/zillabyte/cli/templates/ruby/{simple_function.rb → simple_app.rb} +6 -6
- data/lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml +1 -1
- data/lib/zillabyte/cli/version.rb +1 -1
- data/lib/zillabyte/cli/zillalogs.rb +1 -1
- data/lib/zillabyte/command.rb +10 -2
- data/lib/zillabyte/common/{progress.rb → session.rb} +1 -1
- data/lib/zillabyte/helpers.rb +9 -4
- data/lib/zillabyte-cli/version.rb +1 -1
- data/zillabyte-cli.gemspec +2 -0
- metadata +25 -7
@@ -8,26 +8,31 @@ require 'securerandom'
|
|
8
8
|
require 'colorize'
|
9
9
|
require 'time_difference'
|
10
10
|
|
11
|
-
# manage custom
|
11
|
+
# manage custom apps
|
12
12
|
#
|
13
|
-
class Zillabyte::Command::
|
13
|
+
class Zillabyte::Command::Apps < Zillabyte::Command::Base
|
14
14
|
|
15
|
-
|
15
|
+
MAX_POLL_SECONDS = 60 * 5
|
16
|
+
POLL_SLEEP = 0.5
|
17
|
+
|
18
|
+
# apps
|
16
19
|
#
|
17
|
-
# list custom
|
20
|
+
# list custom apps
|
21
|
+
# --type TYPE # specify an output type i.e. json
|
18
22
|
#
|
19
23
|
def index
|
20
24
|
self.list
|
21
25
|
end
|
22
26
|
|
23
|
-
#
|
24
|
-
#
|
25
|
-
# list custom flows
|
27
|
+
# apps
|
26
28
|
#
|
29
|
+
# list custom apps
|
30
|
+
# --type TYPE # specify an output type i.e. json
|
27
31
|
def list
|
32
|
+
type = options[:type]
|
28
33
|
|
29
34
|
headings = ["id", "name", "state", "cycles"]
|
30
|
-
rows = api.
|
35
|
+
rows = api.app.list.map do |row|
|
31
36
|
if headings.size == 0
|
32
37
|
headings = row.keys
|
33
38
|
headings.delete("rel_dir")
|
@@ -35,137 +40,164 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
35
40
|
v = row.values_at *headings
|
36
41
|
v
|
37
42
|
end
|
38
|
-
|
39
|
-
display "
|
43
|
+
|
44
|
+
display "apps:\n" if type.nil?
|
45
|
+
display TableOutputBuilder.build_table(headings, rows, type)
|
46
|
+
display "Total number of apps: "+rows.length.to_s if type.nil?
|
40
47
|
end
|
41
48
|
|
42
49
|
|
43
|
-
#
|
50
|
+
# apps:push [DIR]
|
44
51
|
#
|
45
|
-
# uploads
|
52
|
+
# uploads an app
|
46
53
|
#
|
47
54
|
# --config CONFIG_FILE # use the given config file
|
55
|
+
# --type TYPE # specify an output type i.e. json
|
48
56
|
#
|
49
57
|
#Examples:
|
50
58
|
#
|
51
|
-
# $ zillabyte
|
59
|
+
# $ zillabyte apps:push .
|
52
60
|
#
|
53
61
|
def push
|
54
62
|
|
55
63
|
since = Time.now.utc.to_s
|
56
64
|
dir = options[:directory] || shift_argument || Dir.pwd
|
57
|
-
|
65
|
+
type = options[:type]
|
66
|
+
|
67
|
+
res = api.apps.push_directory dir, session, options
|
58
68
|
|
59
69
|
if res['error']
|
60
|
-
|
70
|
+
error("error: #{res['error_message']}", type)
|
61
71
|
else
|
62
|
-
display "
|
72
|
+
display "app ##{res['id']} #{res['action']}" if type.nil?
|
63
73
|
end
|
64
74
|
|
65
|
-
display "Starting up your
|
75
|
+
display "Starting up your app...please wait..." if type.nil?
|
66
76
|
sleep(2) # wait for kill command
|
67
77
|
|
68
78
|
lf = LogFormatter::Startup.new
|
69
79
|
api.logs.get_startup(res['id'], "_ALL_", {:push => true}) do |hash|
|
70
80
|
|
71
81
|
# Error?
|
72
|
-
error
|
82
|
+
error(hash['error_message'], type) if hash['error']
|
73
83
|
|
74
84
|
# Print it
|
75
|
-
lf.print_log_line(hash)
|
85
|
+
lf.print_log_line(hash) if type.nil?
|
76
86
|
|
77
87
|
# Exit when we get the 'done' message
|
78
|
-
# exit(0) if (hash['line'] || '').downcase.include?("
|
88
|
+
# exit(0) if (hash['line'] || '').downcase.include?("app deployed")
|
79
89
|
|
80
90
|
end
|
91
|
+
|
92
|
+
display {}.to_json if type == "json"
|
81
93
|
|
82
94
|
end
|
83
|
-
alias_command "push", "
|
95
|
+
alias_command "push", "apps:push"
|
84
96
|
|
85
97
|
|
86
98
|
|
87
|
-
#
|
99
|
+
# apps:pull ID DIR
|
88
100
|
#
|
89
|
-
# pulls
|
101
|
+
# pulls an app source to a directory.
|
90
102
|
#
|
91
103
|
# --force # pulls even if the directory exists
|
104
|
+
# --type TYPE # specify an output type i.e. json
|
92
105
|
#
|
93
106
|
#Examples:
|
94
107
|
#
|
95
|
-
# $ zillabyte
|
108
|
+
# $ zillabyte apps:pull .
|
96
109
|
#
|
97
110
|
def pull
|
98
111
|
|
99
112
|
id = options[:id] || shift_argument
|
100
113
|
dir = options[:directory] || shift_argument
|
114
|
+
type = options[:type]
|
115
|
+
|
101
116
|
|
102
|
-
error
|
103
|
-
error
|
117
|
+
error("no id given", type) if id.nil?
|
118
|
+
error("no directory given", type) if dir.nil?
|
104
119
|
|
105
120
|
# Create if not exists..
|
106
121
|
if File.exists?(dir)
|
107
122
|
if Dir.entries(dir).size != 2 and options[:force].nil?
|
108
|
-
error
|
123
|
+
error("target directory not empty. use --force to override", type)
|
109
124
|
end
|
110
125
|
else
|
111
126
|
FileUtils.mkdir_p(dir)
|
112
127
|
end
|
113
128
|
|
114
|
-
res = api.
|
129
|
+
res = api.apps.pull_to_directory id, dir, session
|
115
130
|
|
116
131
|
if res['error']
|
117
|
-
|
132
|
+
error("error: #{res['error_message']}", type)
|
118
133
|
else
|
119
|
-
|
134
|
+
if type == "json"
|
135
|
+
display {}.to_json
|
136
|
+
else
|
137
|
+
display "app ##{res['id']} pulled to #{dir}"
|
138
|
+
end
|
120
139
|
end
|
121
140
|
|
122
141
|
end
|
123
|
-
alias_command "pull", "
|
142
|
+
alias_command "pull", "apps:pull"
|
124
143
|
|
125
144
|
|
126
|
-
#
|
145
|
+
# apps:delete ID
|
127
146
|
#
|
128
|
-
# deletes
|
147
|
+
# deletes an app. if the app is running, this command will kill it.
|
129
148
|
#
|
130
149
|
# -f, --force # don't ask for confirmation
|
150
|
+
# --type TYPE # specify an output type i.e. json
|
151
|
+
#
|
131
152
|
def delete
|
132
153
|
id = options[:id] || shift_argument
|
133
154
|
if id.nil?
|
134
155
|
id = read_name_from_conf(options)
|
135
156
|
options[:flow_name] = true
|
136
157
|
end
|
137
|
-
|
138
158
|
forced = options[:force]
|
159
|
+
type = options[:type]
|
160
|
+
|
139
161
|
if not forced
|
162
|
+
|
163
|
+
if !type.nil?
|
164
|
+
error("specify -f, --force to confirm deletion", type)
|
165
|
+
end
|
166
|
+
|
140
167
|
while true
|
141
168
|
|
142
|
-
display "This operation cannot be undone. Are you sure you want to delete this
|
169
|
+
display "This operation cannot be undone. Are you sure you want to delete this app? (yes/no):", false
|
143
170
|
confirm = ask
|
144
171
|
break if confirm == "yes" || confirm == "no"
|
145
|
-
display "Please enter 'yes' to delete the
|
172
|
+
display "Please enter 'yes' to delete the app or 'no' to exit"
|
146
173
|
end
|
147
174
|
end
|
148
175
|
|
149
176
|
confirmed = forced || confirm == "yes"
|
150
177
|
|
151
178
|
if confirmed
|
152
|
-
response = api.
|
153
|
-
|
154
|
-
|
179
|
+
response = api.apps.delete(id, options)
|
180
|
+
if type == "json"
|
181
|
+
display {}.to_json
|
182
|
+
else
|
183
|
+
display response["body"]
|
184
|
+
end
|
155
185
|
end
|
156
186
|
|
157
187
|
end
|
158
188
|
|
159
189
|
|
160
|
-
#
|
190
|
+
# apps:prep [DIR]
|
191
|
+
#
|
192
|
+
# prepares an app for execution
|
161
193
|
#
|
162
|
-
# prepares a flow for execution
|
163
194
|
#
|
164
195
|
def prep
|
165
196
|
|
166
197
|
dir = options[:directory] || shift_argument || Dir.pwd
|
198
|
+
|
167
199
|
meta = Zillabyte::CLI::Config.get_config_info(dir)
|
168
|
-
|
200
|
+
|
169
201
|
case meta["language"]
|
170
202
|
when "ruby"
|
171
203
|
|
@@ -192,29 +224,33 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
192
224
|
end
|
193
225
|
|
194
226
|
end
|
195
|
-
alias_command "prep", "
|
227
|
+
alias_command "prep", "apps:prep"
|
196
228
|
|
197
229
|
|
198
230
|
|
199
231
|
|
200
|
-
#
|
232
|
+
# apps:init [LANG] [DIR]
|
201
233
|
#
|
202
234
|
# initializes a new executable in DIR
|
203
235
|
# [LANG] defaults to ruby, and [DIR] to the current directory
|
204
236
|
#
|
237
|
+
# --type TYPE # specify an output type i.e. json
|
238
|
+
#
|
205
239
|
#Examples:
|
206
240
|
#
|
207
|
-
# $ zillabyte
|
241
|
+
# $ zillabyte apps:init python contact_extractor
|
208
242
|
#
|
209
243
|
def init
|
210
244
|
|
211
245
|
lang = options[:lang] || shift_argument || "ruby"
|
212
246
|
dir = options[:dir] || shift_argument || Dir.pwd
|
247
|
+
type = options[:type]
|
248
|
+
|
213
249
|
languages = ["ruby","python", "js"]
|
214
250
|
|
215
|
-
error
|
251
|
+
error("Unsupported language #{lang}. We only support #{languages.join(', ')}.", type) if not languages.include? lang
|
216
252
|
|
217
|
-
display "initializing empty #{lang}
|
253
|
+
display "initializing empty #{lang} app in #{dir}" if type.nil?
|
218
254
|
FileUtils.cp_r( File.expand_path("../templates/#{lang}", __FILE__) + "/." , dir )
|
219
255
|
|
220
256
|
|
@@ -222,66 +258,133 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
222
258
|
|
223
259
|
|
224
260
|
|
225
|
-
#
|
261
|
+
# apps:logs APP_ID [OPERATION_NAME]
|
226
262
|
#
|
227
263
|
# streams logs from the distributed workers
|
228
264
|
#
|
265
|
+
# --type TYPE # specify an output type i.e. json
|
229
266
|
# -v, --verbose LEVEL # sets the verbosity (error, info, debug) (default: info)
|
230
267
|
#
|
231
268
|
def logs
|
232
269
|
|
233
|
-
|
270
|
+
app_id = options[:app] || shift_argument
|
234
271
|
operation_id = options[:operation] || shift_argument || '_ALL_'
|
235
272
|
category = options[:verbose] || '_ALL_'
|
273
|
+
type = options[:type]
|
274
|
+
|
236
275
|
carry_settings = {
|
237
276
|
:category => category
|
238
277
|
}
|
239
278
|
api_options = {}
|
240
279
|
|
241
|
-
if
|
242
|
-
|
243
|
-
api_options[
|
280
|
+
if app_id.nil?
|
281
|
+
app_id = read_name_from_conf(options)
|
282
|
+
api_options[:flow_name] = true
|
244
283
|
end
|
245
284
|
|
246
|
-
display "Retrieving logs for
|
285
|
+
display "Retrieving logs for app ##{app_id}...please wait..." if type.nil?
|
247
286
|
lf = LogFormatter::Operation.new
|
248
|
-
self.api.logs.get(
|
287
|
+
self.api.logs.get(app_id, operation_id, api_options) do |line|
|
249
288
|
|
250
|
-
error
|
251
|
-
lf.print_log_line(line)
|
289
|
+
error(line['error_message'], type) if line['error']
|
290
|
+
lf.print_log_line(line) if type.nil?
|
252
291
|
|
253
292
|
end
|
254
293
|
|
255
294
|
end
|
256
|
-
alias_command "logs", "
|
295
|
+
alias_command "logs", "apps:logs"
|
257
296
|
|
258
297
|
|
259
|
-
|
298
|
+
|
299
|
+
# apps:errors ID
|
300
|
+
#
|
301
|
+
# Show recent errors generated by the app
|
302
|
+
# --type TYPE # specify an output type i.e. json
|
303
|
+
#
|
304
|
+
def errors
|
305
|
+
|
306
|
+
# Init
|
307
|
+
app_id = options[:id] || shift_argument
|
308
|
+
|
309
|
+
# No name?
|
310
|
+
if app_id.nil?
|
311
|
+
app_id = read_name_from_conf(options)
|
312
|
+
options[:flow_name] = true
|
313
|
+
end
|
314
|
+
|
315
|
+
type = options[:type]
|
316
|
+
|
317
|
+
# Make the request
|
318
|
+
res = api.request(
|
319
|
+
:expects => 200,
|
320
|
+
:method => :get,
|
321
|
+
:body => options.to_json,
|
322
|
+
:path => "/flows/#{CGI.escape(app_id)}/errors"
|
323
|
+
)
|
324
|
+
|
325
|
+
# Render
|
326
|
+
display "Recent errors:" if type.nil?
|
327
|
+
headings = ["operation", "date", "error"]
|
328
|
+
rows = (res.body["recent_errors"] || []).map do |row|
|
329
|
+
if row['date']
|
330
|
+
d = Time.at(row['date']/1000)
|
331
|
+
else
|
332
|
+
d = nil
|
333
|
+
end
|
334
|
+
[row['name'], d, row['message']]
|
335
|
+
end
|
336
|
+
rows.sort! do |a,b|
|
337
|
+
a[1] <=> b[1]
|
338
|
+
end
|
339
|
+
color_map = {}
|
340
|
+
colors = LogFormatter::COLORS.clone
|
341
|
+
rows.each do |row|
|
342
|
+
name = row[0]
|
343
|
+
time = row[1]
|
344
|
+
message = row[2].strip
|
345
|
+
color_map[name] ||= colors.shift
|
346
|
+
if time
|
347
|
+
display "#{"* #{name} - #{time_ago_in_words(time)} ago".colorize(color_map[name])}:" if type.nil?
|
348
|
+
else
|
349
|
+
display "#{"* #{name}".colorize(color_map[name])}:" if type.nil?
|
350
|
+
end
|
351
|
+
message.split('\n').each do |sub_line|
|
352
|
+
display " #{sub_line}" if type.nil?
|
353
|
+
end
|
354
|
+
end
|
355
|
+
|
356
|
+
end
|
357
|
+
|
358
|
+
# apps:cycles ID [OPTIONS]
|
260
359
|
#
|
261
|
-
# operations on the
|
262
|
-
# with no options, the command lists the
|
263
|
-
# -n, --next
|
264
|
-
# -f, --forever
|
360
|
+
# operations on the app's cycles (batches).
|
361
|
+
# with no options, the command lists the apps cycles
|
362
|
+
# -n, --next # request the app to move to the next cycle
|
363
|
+
# -f, --forever # don't wait on cycles any more
|
364
|
+
# --type TYPE # specify an output type i.e. json
|
365
|
+
#
|
265
366
|
def cycles
|
266
|
-
|
367
|
+
app_id = options[:id] || shift_argument
|
368
|
+
type = options[:type]
|
369
|
+
|
267
370
|
trigger_next = options[:next] || false
|
268
371
|
trigger_forever = options[:forever] || false
|
269
372
|
|
270
|
-
if
|
271
|
-
|
373
|
+
if app_id.nil?
|
374
|
+
app_id = read_name_from_conf(options)
|
272
375
|
options[:flow_name] = true
|
273
376
|
end
|
274
377
|
|
275
378
|
if trigger_next
|
276
|
-
# Trigger the next
|
277
|
-
response = api.
|
379
|
+
# Trigger the next app
|
380
|
+
response = api.apps.create_cycle(app_id, options)
|
278
381
|
elsif trigger_forever
|
279
|
-
response = api.
|
382
|
+
response = api.apps.run_forever(app_id, options)
|
280
383
|
else
|
281
|
-
# List the
|
282
|
-
response = api.
|
283
|
-
# TODO List the sequence number for this
|
284
|
-
display "Most recent cyles of the
|
384
|
+
# List the apps
|
385
|
+
response = api.apps.list_cycles(app_id, options)
|
386
|
+
# TODO List the sequence number for this app.
|
387
|
+
display "Most recent cyles of the app:" if type.nil?
|
285
388
|
headings = ["Cycle_id", "State", "Start", "End"]
|
286
389
|
rows = response["cycles"]
|
287
390
|
rows = rows.map do |row|
|
@@ -289,20 +392,57 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
289
392
|
end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
|
290
393
|
[row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
|
291
394
|
end
|
292
|
-
|
293
|
-
display
|
395
|
+
|
396
|
+
display TableOutputBuilder.build_table(headings, rows, type)
|
397
|
+
display "Total number of cycles executed: #{response['total']}" if type.nil?
|
398
|
+
return
|
294
399
|
end
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
400
|
+
|
401
|
+
if response["job_id"]
|
402
|
+
options[:job_id] = response["job_id"]
|
403
|
+
app_id = response["flow_id"]
|
404
|
+
options.delete :flow_name
|
405
|
+
|
406
|
+
start = Time.now.utc
|
407
|
+
display "Next cycle request sent. If your app was RETIRED this may take slightly longer." if type.nil?
|
408
|
+
|
409
|
+
while(Time.now.utc < start + MAX_POLL_SECONDS) do
|
410
|
+
|
411
|
+
# Poll
|
412
|
+
res = self.api.apps.cycles_poll(app_id, options)
|
413
|
+
|
414
|
+
case res['status']
|
415
|
+
when 'completed'
|
416
|
+
if res['return']
|
417
|
+
if type == "json"
|
418
|
+
return {}.to_json
|
419
|
+
else
|
420
|
+
display res['return']
|
421
|
+
end
|
422
|
+
else
|
423
|
+
throw "something is wrong: #{res}"
|
424
|
+
end
|
425
|
+
# success! continue below
|
426
|
+
break
|
427
|
+
when 'running'
|
428
|
+
sleep(POLL_SLEEP)
|
429
|
+
# display ".", false
|
430
|
+
else
|
431
|
+
throw "unknown status: #{res}"
|
432
|
+
end
|
433
|
+
|
434
|
+
end
|
435
|
+
elsif response["error"]
|
436
|
+
error(response["error"], type)
|
437
|
+
else
|
438
|
+
error("remote server error (a380)", type)
|
299
439
|
end
|
300
440
|
end
|
301
441
|
|
302
442
|
|
303
|
-
#
|
443
|
+
# apps:test
|
304
444
|
#
|
305
|
-
# tests a local
|
445
|
+
# tests a local app with sample data
|
306
446
|
#
|
307
447
|
# --config CONFIG_FILE # use the given config file
|
308
448
|
# --output OUTPUT_FILE # writes sink output to a file
|
@@ -312,6 +452,8 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
312
452
|
def test
|
313
453
|
|
314
454
|
output = options[:output]
|
455
|
+
type = options[:type]
|
456
|
+
|
315
457
|
max_seconds = (options[:wait] || "30").to_i
|
316
458
|
batches = (options[:batches] || "1").to_i
|
317
459
|
|
@@ -364,33 +506,34 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
364
506
|
# INIT
|
365
507
|
dir = options[:dir] || Dir.pwd
|
366
508
|
|
367
|
-
meta = Zillabyte::API::
|
509
|
+
meta = Zillabyte::API::Apps.get_rich_meta_info_from_script(dir, self, {:test => true})
|
368
510
|
if meta.nil?
|
369
|
-
error "this is not a valid zillabyte
|
511
|
+
error "this is not a valid zillabyte app directory"
|
370
512
|
exit
|
371
513
|
end
|
372
514
|
|
373
|
-
# Show the user what we know about their
|
374
|
-
display "inferring your
|
515
|
+
# Show the user what we know about their app...
|
516
|
+
display "inferring your app details..."
|
375
517
|
colors = {}
|
376
|
-
|
518
|
+
describe_app(meta, colors)
|
377
519
|
|
378
520
|
|
379
|
-
# Extract the
|
521
|
+
# Extract the app's information..
|
380
522
|
nodes = meta["nodes"]
|
381
523
|
write_to_next_each = []
|
382
524
|
write_queue = []
|
383
525
|
stream_messages = {}
|
384
526
|
default_stream = "_default"
|
385
527
|
|
386
|
-
split_branches = false
|
387
|
-
|
388
528
|
# Iterate all nodes sequentially and invoke them in separate processes...
|
389
529
|
nodes.each do |node|
|
390
530
|
|
391
531
|
# Init
|
392
532
|
type = node["type"]
|
393
533
|
name = node["name"]
|
534
|
+
consumes = node["consumes"]
|
535
|
+
emits = node["emits"]
|
536
|
+
|
394
537
|
color = colors[name] || :default
|
395
538
|
|
396
539
|
op_display = lambda do |msg, override_color = nil|
|
@@ -403,6 +546,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
403
546
|
# A source from relation?
|
404
547
|
if node['matches'] or node["relation"]
|
405
548
|
matches = node['matches'] || (node["relation"]["query"])
|
549
|
+
emits = emits.first #For spouting from a relation, there should only be one emits
|
406
550
|
op_display.call "Grabbing remote data"
|
407
551
|
|
408
552
|
res = api.query.agnostic(matches)
|
@@ -427,9 +571,9 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
427
571
|
end
|
428
572
|
read_msg = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
|
429
573
|
values = Hash[values.map{|k, v| [truncate_message(k), truncate_message(v)]}]
|
430
|
-
op_display.call "
|
431
|
-
stream_messages[
|
432
|
-
stream_messages[
|
574
|
+
op_display.call "emitted: #{values} #{meta} to #{emits}"
|
575
|
+
stream_messages[emits] ||= []
|
576
|
+
stream_messages[emits] << read_msg
|
433
577
|
end
|
434
578
|
|
435
579
|
# Done processing...
|
@@ -439,10 +583,14 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
439
583
|
|
440
584
|
# A regular source..
|
441
585
|
stream_messages[default_stream] ||= []
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
586
|
+
stream_messages[default_stream] << "{\"command\": \"begin_cycle\"}\n"
|
587
|
+
emits.each {|ss| stream_messages[ss] = []} #initialize streams
|
588
|
+
stream_size_at_last_call_to_next_tuple = Hash[emits.map {|ss| [ss, 0]}] #initialize initial size of streams (all 0)
|
589
|
+
# the above initializations are used to deal with the case where end_cycle_policy == "null_emit"
|
590
|
+
n_batches_emitted = 1
|
591
|
+
end_cycle_received = false
|
592
|
+
last_call_next_tuple = false
|
593
|
+
|
446
594
|
end
|
447
595
|
|
448
596
|
# An Aggregate?
|
@@ -488,16 +636,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
488
636
|
# A Sink?
|
489
637
|
elsif type == "sink"
|
490
638
|
|
491
|
-
if
|
492
|
-
|
493
|
-
if node['consumes'].nil?
|
494
|
-
error "The node #{name} must declare which stream it 'consumes'"
|
495
|
-
end
|
496
|
-
sink_stream = node["consumes"]
|
497
|
-
messages = stream_messages[sink_stream] || []
|
498
|
-
else
|
499
|
-
messages = stream_messages.values.first || []
|
639
|
+
if consumes.nil?
|
640
|
+
error "The node #{name} must declare which stream it 'consumes'"
|
500
641
|
end
|
642
|
+
messages = stream_messages[consumes] || []
|
501
643
|
|
502
644
|
table = Terminal::Table.new :title => name
|
503
645
|
csv_str = CSV.generate do |csv|
|
@@ -529,8 +671,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
529
671
|
op_display.call "output written to #{filename}"
|
530
672
|
end
|
531
673
|
|
532
|
-
|
533
|
-
|
534
674
|
next
|
535
675
|
end
|
536
676
|
|
@@ -548,28 +688,32 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
548
688
|
write_queue = []
|
549
689
|
read_queue = []
|
550
690
|
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
stream_name = stream_messages.keys.first
|
555
|
-
write_queue = stream_messages.values.first.clone
|
556
|
-
stream_messages.delete(stream_name)
|
691
|
+
if consumes.nil?
|
692
|
+
# Assume default stream (this should only happen for the source)
|
693
|
+
stream_name = default_stream
|
557
694
|
else
|
558
|
-
|
559
|
-
split_branches = true;
|
560
|
-
if node['consumes'].nil?
|
561
|
-
error "The node #{name} must declare which stream it 'consumes'"
|
562
|
-
end
|
563
|
-
stream_name = node["consumes"]
|
564
|
-
write_queue = stream_messages[stream_name].clone()
|
565
|
-
stream_messages.delete(stream_name)
|
695
|
+
stream_name = consumes
|
566
696
|
end
|
697
|
+
write_queue = stream_messages[stream_name].clone
|
698
|
+
stream_messages.delete(stream_name)
|
567
699
|
|
568
700
|
# Start writing the messages...
|
569
701
|
stuff_to_read = false
|
570
702
|
writing_thread = Thread.start do
|
571
|
-
|
572
|
-
|
703
|
+
|
704
|
+
while(true)
|
705
|
+
|
706
|
+
case type
|
707
|
+
when 'source'
|
708
|
+
break if n_batches_emitted > batches
|
709
|
+
if write_queue.empty?
|
710
|
+
sleep 0.5
|
711
|
+
next
|
712
|
+
end
|
713
|
+
else
|
714
|
+
break if write_queue.empty?
|
715
|
+
end
|
716
|
+
|
573
717
|
# Make sure we're not reading anything...
|
574
718
|
while(stuff_to_read) # TODO: semaphores
|
575
719
|
sleep 0.5 # spin wait
|
@@ -584,7 +728,8 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
584
728
|
display_hash = Hash[write_json['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
|
585
729
|
op_display.call "receiving: #{display_hash}"
|
586
730
|
elsif write_json['command'] == 'next'
|
587
|
-
|
731
|
+
last_call_next_tuple = true
|
732
|
+
op_display.call "getting next set of tuples in the batch"
|
588
733
|
else
|
589
734
|
puts write_json
|
590
735
|
end
|
@@ -605,10 +750,43 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
605
750
|
reading_thread = Thread.start do
|
606
751
|
while(true)
|
607
752
|
|
753
|
+
# If the end cycle command is received, we either trigger the next cycle if the number of emitted
|
754
|
+
# cycles is less than what the user requested, or we break
|
755
|
+
if type == "source" and end_cycle_received
|
756
|
+
write_queue << "{\"command\": \"begin_cycle\"}\n"
|
757
|
+
n_batches_emitted += 1
|
758
|
+
end_cycle_received = false
|
759
|
+
last_call_next_tuple = false
|
760
|
+
stuff_to_read = false
|
761
|
+
break if n_batches_emitted > batches
|
762
|
+
sleep 0.5
|
763
|
+
next
|
764
|
+
end
|
765
|
+
|
608
766
|
# Get next message
|
609
767
|
read_msg = read_message(stdout, color)
|
610
768
|
if read_msg == "done" || read_msg.nil?
|
611
769
|
stuff_to_read = false
|
770
|
+
|
771
|
+
# For sources, if we receive a "done", check to see if any of the streams emitted by the source has
|
772
|
+
# increased in size since the last call to next_tuple. If so, the cycle isn't over, otherwise, the
|
773
|
+
# current call to next_tuple emitted nothing and if the end_cycle_policy is set to null_emit, this
|
774
|
+
# should end the current cycle.
|
775
|
+
if type == "source"
|
776
|
+
if last_call_next_tuple and node["end_cycle_policy"] == "null_emit"
|
777
|
+
end_cycle_received = true
|
778
|
+
emits.each do |ss|
|
779
|
+
end_cycle_received = false if stream_messages[ss].size > stream_size_at_last_call_to_next_tuple[ss]
|
780
|
+
break
|
781
|
+
end
|
782
|
+
next if end_cycle_received
|
783
|
+
end
|
784
|
+
|
785
|
+
# If the policy isn't "null_emit", then just request next_tuple again
|
786
|
+
write_queue << "{\"command\": \"next\"}\n"
|
787
|
+
end
|
788
|
+
|
789
|
+
# For other operations, if the queue is empty then we're done
|
612
790
|
if write_queue.empty?
|
613
791
|
break # exit while loop
|
614
792
|
else
|
@@ -620,20 +798,49 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
620
798
|
|
621
799
|
# Process message
|
622
800
|
obj = JSON.parse(read_msg)
|
801
|
+
|
802
|
+
# process the received tuple or other commands
|
623
803
|
if obj['tuple']
|
624
|
-
|
625
|
-
#
|
804
|
+
|
805
|
+
# if
|
806
|
+
tt = obj['tuple']
|
807
|
+
tt.each do |kk, vv|
|
808
|
+
if tt[kk].nil? and type == "source" and node["end_cycle_policy"] == "null_emit"
|
809
|
+
end_cycle_received = true
|
810
|
+
# read rest of stuff in stdout buffer until "done"
|
811
|
+
mm = nil
|
812
|
+
while(mm != "done")
|
813
|
+
mm = read_message(stdout, color)
|
814
|
+
end
|
815
|
+
break
|
816
|
+
end
|
817
|
+
end
|
818
|
+
next if end_cycle_received
|
819
|
+
|
820
|
+
# Convert to a incoming tuple for the next operation
|
626
821
|
next_msg = {
|
627
822
|
:tuple => obj['tuple'],
|
628
823
|
:meta => obj['meta']
|
629
824
|
}
|
630
|
-
emit_stream = obj['stream']
|
825
|
+
emit_stream = obj['stream']
|
631
826
|
stream_messages[emit_stream] ||= []
|
632
827
|
stream_messages[emit_stream] << next_msg.to_json
|
828
|
+
|
633
829
|
display_hash = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
|
634
830
|
op_display.call "emitted: #{display_hash} to #{emit_stream}"
|
831
|
+
|
832
|
+
# track stream message size to end cycles when necessary
|
833
|
+
stream_size_at_last_call_to_next_tuple[emit_stream] = stream_messages[emit_stream].size if type == "source"
|
834
|
+
elsif obj['command'] == 'end_cycle'
|
835
|
+
end_cycle_received = true
|
836
|
+
# command:end_cycle should always be followed by done, read it (below) so
|
837
|
+
# that it doesn't interfere with next
|
838
|
+
read_message(stdout, color)
|
635
839
|
elsif obj['command'] == 'log'
|
636
840
|
op_display.call "log: #{obj['msg']}"
|
841
|
+
elsif obj['command'] == 'fail'
|
842
|
+
op_display.call "error: #{obj['msg']}", :red
|
843
|
+
exit(1)
|
637
844
|
else
|
638
845
|
error "unknown message: #{read_msg}"
|
639
846
|
end
|
@@ -641,7 +848,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
641
848
|
end
|
642
849
|
end
|
643
850
|
|
644
|
-
|
645
851
|
# stderr thread
|
646
852
|
stderr_thread = Thread.start do
|
647
853
|
stderr.each do |line|
|
@@ -670,60 +876,61 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
670
876
|
end
|
671
877
|
rescue PTY::ChildExited
|
672
878
|
puts "The child process exited!"
|
673
|
-
ensure
|
674
|
-
# We need to check here to see if we need to turn on multistream mode.
|
675
|
-
# Note that we still need the redundant checks above, in case the user
|
676
|
-
# didn't honor the emits contract.
|
677
|
-
if node["emits"] && node["emits"].size() > 1
|
678
|
-
split_branches = true
|
679
|
-
end
|
680
879
|
end
|
681
880
|
end
|
682
881
|
|
683
882
|
end
|
684
|
-
alias_command "test", "
|
883
|
+
alias_command "test", "apps:test"
|
685
884
|
|
686
885
|
|
687
|
-
#
|
886
|
+
# apps:kill ID
|
688
887
|
#
|
689
|
-
# kills the given
|
888
|
+
# kills the given app
|
690
889
|
#
|
691
890
|
# --config CONFIG_FILE # use the given config file
|
891
|
+
# --type TYPE # specify an output type i.e. json
|
692
892
|
#
|
693
893
|
def kill
|
694
894
|
|
695
895
|
id = options[:id] || shift_argument
|
896
|
+
type = options[:type]
|
897
|
+
|
696
898
|
if id.nil?
|
697
899
|
id = read_name_from_conf(options)
|
698
900
|
options[:flow_name] = true
|
699
901
|
end
|
700
902
|
|
701
|
-
display "Killing
|
702
|
-
api.
|
703
|
-
display "Flow ##{id} killed"
|
704
|
-
|
705
|
-
end
|
706
|
-
|
707
|
-
|
708
|
-
|
903
|
+
display "Killing app ##{id}...please wait..." if type.nil?
|
904
|
+
api.apps.kill(id, options)
|
709
905
|
|
906
|
+
if type == "json"
|
907
|
+
display {}.to_json
|
908
|
+
else
|
909
|
+
display "App ##{id} killed"
|
910
|
+
end
|
710
911
|
|
912
|
+
end
|
711
913
|
|
712
|
-
|
914
|
+
|
915
|
+
# apps:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]
|
713
916
|
#
|
714
|
-
# runs a local
|
917
|
+
# runs a local app with live data
|
715
918
|
#
|
716
919
|
# --config CONFIG_FILE # use the given config file
|
920
|
+
# --type TYPE # specify an output type i.e. json
|
921
|
+
#
|
717
922
|
# HIDDEN:
|
718
923
|
def live_run
|
719
924
|
|
720
925
|
name = options[:name] || shift_argument
|
926
|
+
type = options[:type]
|
927
|
+
|
721
928
|
thread_id = options[:thread] || shift_argument || ""
|
722
929
|
dir = options[:directory] || shift_argument || Dir.pwd
|
723
930
|
meta = Zillabyte::CLI::Config.get_config_info(dir)
|
724
931
|
|
725
932
|
if meta.nil?
|
726
|
-
|
933
|
+
error("could not find meta information for: #{dir}", type)
|
727
934
|
end
|
728
935
|
|
729
936
|
if(thread_id == "")
|
@@ -732,73 +939,91 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
732
939
|
exec(command("--execute_live --name #{name.to_s} --pipe #{thread_id}"))
|
733
940
|
end
|
734
941
|
end
|
735
|
-
alias_command "live_run", "
|
942
|
+
alias_command "live_run", "apps:live_run"
|
736
943
|
|
737
944
|
|
738
|
-
#
|
945
|
+
# apps:info [DIR]
|
739
946
|
#
|
740
|
-
# outputs the info for the
|
947
|
+
# outputs the info for the app in the dir.
|
741
948
|
#
|
742
949
|
# --pretty # Pretty prints the info output
|
950
|
+
# --type TYPE # specify an output type i.e. json
|
743
951
|
#
|
744
952
|
def info
|
745
953
|
info_file = SecureRandom.uuid
|
954
|
+
type = options[:type]
|
955
|
+
|
746
956
|
cmd = command("--info --file #{info_file}")
|
747
|
-
|
957
|
+
app_info = Zillabyte::Command::Apps.get_info(cmd, info_file)
|
748
958
|
|
749
|
-
if
|
750
|
-
puts
|
959
|
+
if type == "json"
|
960
|
+
puts app.info
|
751
961
|
else
|
752
|
-
|
962
|
+
if options[:pretty]
|
963
|
+
puts JSON.pretty_generate(JSON.parse(app_info))
|
964
|
+
else
|
965
|
+
puts app_info
|
966
|
+
end
|
753
967
|
end
|
968
|
+
|
754
969
|
exit
|
755
970
|
end
|
756
|
-
alias_command "info", "
|
971
|
+
alias_command "info", "apps:info"
|
757
972
|
|
758
973
|
|
974
|
+
#
|
975
|
+
# --type TYPE # specify an output type i.e. json
|
976
|
+
#
|
759
977
|
def self.get_info(cmd, info_file, options = {})
|
978
|
+
type = options[:type]
|
760
979
|
response = `#{cmd}`
|
761
980
|
if($?.exitstatus == 1)
|
762
981
|
File.delete("#{info_file}") if File.exists?(info_file)
|
763
|
-
|
764
|
-
|
982
|
+
if options[:type].nil?
|
983
|
+
exit(1)
|
984
|
+
else
|
985
|
+
Zillabyte::Helpers.error("error: #{response}", type)
|
986
|
+
end
|
765
987
|
end
|
766
988
|
|
767
|
-
|
989
|
+
app_info = {}
|
768
990
|
File.open("#{info_file}", "r+").each do |line|
|
769
991
|
line = JSON.parse(line)
|
770
992
|
if(line["type"])
|
771
|
-
|
993
|
+
app_info["nodes"] << line
|
772
994
|
else
|
773
|
-
|
774
|
-
|
995
|
+
app_info = line
|
996
|
+
app_info["nodes"] = []
|
775
997
|
end
|
776
998
|
end
|
777
999
|
File.delete("#{info_file}")
|
778
1000
|
|
779
|
-
|
1001
|
+
app_info = app_info.to_json
|
780
1002
|
if(File.exists?("info_to_java.in"))
|
781
1003
|
java_pipe = open("info_to_java.in","w+")
|
782
|
-
java_pipe.puts(
|
1004
|
+
java_pipe.puts(app_info+"\n")
|
783
1005
|
java_pipe.flush
|
784
1006
|
java_pipe.close()
|
785
1007
|
end
|
786
1008
|
|
787
|
-
|
1009
|
+
app_info
|
788
1010
|
end
|
789
1011
|
|
790
1012
|
|
791
1013
|
private
|
792
1014
|
|
793
|
-
|
794
|
-
|
1015
|
+
#
|
1016
|
+
#
|
1017
|
+
# --type TYPE # specify an output type i.e. json
|
1018
|
+
#
|
795
1019
|
def command(arg="--execute_live", ignore_stderr = false)
|
796
1020
|
|
797
1021
|
dir = options[:directory] || shift_argument || Dir.pwd
|
798
|
-
|
1022
|
+
type = options[:type]
|
1023
|
+
meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
|
799
1024
|
#meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
|
800
|
-
error
|
801
|
-
error
|
1025
|
+
error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
|
1026
|
+
error(meta["error_message"], type) if meta['status'] == "error"
|
802
1027
|
full_script = File.join(dir, meta["script"])
|
803
1028
|
stderr_opt = "2> /dev/null" if ignore_stderr
|
804
1029
|
|
@@ -819,19 +1044,21 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
819
1044
|
cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
|
820
1045
|
|
821
1046
|
else
|
822
|
-
error
|
1047
|
+
error("no language specified", type)
|
823
1048
|
end
|
824
1049
|
|
825
1050
|
return cmd
|
826
1051
|
|
827
1052
|
end
|
828
1053
|
|
829
|
-
|
830
|
-
|
1054
|
+
#
|
1055
|
+
#
|
1056
|
+
#
|
1057
|
+
def describe_app(meta, colors = {})
|
831
1058
|
@colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
|
832
1059
|
rjust = 20
|
833
|
-
display "#{'
|
834
|
-
display "#{'
|
1060
|
+
display "#{'app name'.rjust(rjust)}: #{meta['name']}"
|
1061
|
+
display "#{'app language'.rjust(rjust)}: #{meta['language']}"
|
835
1062
|
|
836
1063
|
meta['nodes'].each_with_index do |node, index|
|
837
1064
|
colors[node['name']] ||= @colors.shift
|
@@ -845,9 +1072,12 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
845
1072
|
|
846
1073
|
end
|
847
1074
|
|
1075
|
+
#
|
1076
|
+
#
|
848
1077
|
def read_name_from_conf(options = {})
|
849
|
-
|
850
|
-
|
1078
|
+
type = options[:type]
|
1079
|
+
hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
|
1080
|
+
error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify an app id or run command from the directory containing the app.",type) if hash["error"]
|
851
1081
|
hash["name"]
|
852
1082
|
end
|
853
1083
|
|