zillabyte-cli 0.0.20 → 0.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1158 @@
1
+ require "zillabyte/cli/base"
2
+ require "zillabyte/cli/config"
3
+ require "zillabyte/common"
4
+ require "pty"
5
+ require 'indentation'
6
+ require 'open3'
7
+ require 'securerandom'
8
+ require 'colorize'
9
+ require 'time_difference'
10
+
11
+ # manage custom apps
12
+ #
13
+ class Zillabyte::Command::Apps < Zillabyte::Command::Base
14
+
15
+ MAX_POLL_SECONDS = 60 * 5
16
+ POLL_SLEEP = 0.5
17
+
18
+ # apps
19
+ #
20
+ # list custom apps
21
+ # --type TYPE # specify an output type i.e. json
22
+ #
23
+ def index
24
+ self.list
25
+ end
26
+
27
+ # apps
28
+ #
29
+ # list custom apps
30
+ # --type TYPE # specify an output type i.e. json
31
+ def list
32
+ type = options[:type]
33
+
34
+ headings = ["id", "name", "state", "cycles"]
35
+ rows = api.app.list.map do |row|
36
+ if headings.size == 0
37
+ headings = row.keys
38
+ headings.delete("rel_dir")
39
+ end
40
+ v = row.values_at *headings
41
+ v
42
+ end
43
+
44
+ display "apps:\n" if type.nil?
45
+ display TableOutputBuilder.build_table(headings, rows, type)
46
+ display "Total number of apps: "+rows.length.to_s if type.nil?
47
+ end
48
+
49
+
50
+ # apps:push [DIR]
51
+ #
52
+ # uploads an app
53
+ #
54
+ # --config CONFIG_FILE # use the given config file
55
+ # --type TYPE # specify an output type i.e. json
56
+ # --directory DIR # app directory
57
+ #
58
+ #Examples:
59
+ #
60
+ # $ zillabyte apps:push .
61
+ #
62
+ def push
63
+
64
+ since = Time.now.utc.to_s
65
+ dir = options[:directory] || shift_argument
66
+ if dir.nil?
67
+ dir = Dir.pwd
68
+ else
69
+ dir = File.expand_path(dir)
70
+ end
71
+ options[:directory] = dir
72
+ type = options[:type]
73
+
74
+ res = api.apps.push_directory dir, session, options
75
+
76
+ if res['error']
77
+ error("error: #{res['error_message']}", type)
78
+ else
79
+ display "app ##{res['id']} #{res['action']}" if type.nil?
80
+ end
81
+
82
+ display "Starting up your app...please wait..." if type.nil?
83
+ sleep(2) # wait for kill command
84
+
85
+ lf = LogFormatter::Startup.new
86
+ api.logs.get_startup(res['id'], "_ALL_", {:push => true}) do |hash|
87
+
88
+ # Error?
89
+ error(hash['error_message'], type) if hash['error']
90
+
91
+ # Print it
92
+ lf.print_log_line(hash) if type.nil?
93
+
94
+ # Exit when we get the 'done' message
95
+ # exit(0) if (hash['line'] || '').downcase.include?("app deployed")
96
+
97
+ end
98
+ display "{}" if type == "json"
99
+
100
+ end
101
+ alias_command "push", "apps:push"
102
+
103
+
104
+
105
+ # apps:pull ID DIR
106
+ #
107
+ # pulls an app source to a directory.
108
+ #
109
+ # --force # pulls even if the directory exists
110
+ # --type TYPE # specify an output type i.e. json
111
+ # --directory DIR # Directory of the app
112
+ #
113
+ #Examples:
114
+ #
115
+ # $ zillabyte apps:pull .
116
+ #
117
+ def pull
118
+
119
+ app_id = options[:id] || shift_argument
120
+
121
+ if !(app_id =~ /^\d*$/)
122
+ options[:is_name] = true
123
+ end
124
+
125
+ dir = options[:directory] || shift_argument
126
+ error("no directory given", type) if dir.nil?
127
+ dir = File.expand_path(dir)
128
+
129
+ type = options[:type]
130
+
131
+ error("no id given", type) if app_id.nil?
132
+
133
+ # Create if not exists..
134
+ if File.exists?(dir)
135
+ if Dir.entries(dir).size != 2 and options[:force].nil?
136
+ error("target directory not empty. use --force to override", type)
137
+ end
138
+ else
139
+ FileUtils.mkdir_p(dir)
140
+ end
141
+
142
+ res = api.apps.pull_to_directory app_id, dir, session, options
143
+
144
+ if res['error']
145
+ error("error: #{res['error_message']}", type)
146
+ else
147
+ if type == "json"
148
+ display "{}"
149
+ else
150
+ display "app ##{res['id']} pulled to #{dir}"
151
+ end
152
+ end
153
+
154
+ end
155
+ alias_command "pull", "apps:pull"
156
+
157
+
158
+ # apps:delete ID
159
+ #
160
+ # deletes an app. if the app is running, this command will kill it.
161
+ #
162
+ # -f, --force # don't ask for confirmation
163
+ # --type TYPE # specify an output type i.e. json
164
+ #
165
+ def delete
166
+ app_id = options[:id] || shift_argument
167
+
168
+ if app_id.nil?
169
+ app_id = read_name_from_conf(options)
170
+ options[:is_name] = true
171
+ elsif !(app_id =~ /^\d*$/)
172
+ options[:is_name] = true
173
+ end
174
+ forced = options[:force]
175
+ type = options[:type]
176
+
177
+ if not forced
178
+
179
+ if !type.nil?
180
+ error("specify -f, --force to confirm deletion", type)
181
+ end
182
+
183
+ while true
184
+
185
+ display "This operation cannot be undone. Are you sure you want to delete this app? (yes/no):", false
186
+ confirm = ask
187
+ break if confirm == "yes" || confirm == "no"
188
+ display "Please enter 'yes' to delete the app or 'no' to exit"
189
+ end
190
+ end
191
+
192
+ confirmed = forced || confirm == "yes"
193
+
194
+ if confirmed
195
+ response = api.apps.delete(app_id, options)
196
+ if type == "json"
197
+ display "{}"
198
+ else
199
+ display response["body"]
200
+ end
201
+ end
202
+
203
+ end
204
+
205
+
206
+ # apps:prep [DIR]
207
+ #
208
+ # --directory DIR # app directory
209
+ # --type TYPE # specify an output type i.e. json
210
+ #
211
+ def prep
212
+
213
+ type = options[:type]
214
+ dir = options[:directory] || shift_argument
215
+ if dir.nil?
216
+ dir = Dir.pwd
217
+ else
218
+ dir = File.expand_path(dir)
219
+ end
220
+ options[:directory] = dir
221
+ meta = Zillabyte::CLI::Config.get_config_info(dir)
222
+
223
+ if meta.nil?
224
+ error("The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file.", type)
225
+ end
226
+
227
+ case meta["language"]
228
+ when "ruby"
229
+
230
+ # Execute in the bundler context
231
+ full_script = File.join(dir, meta["script"])
232
+ cmd = "cd \"#{meta['home_dir']}\"; unset BUNDLE_GEMFILE; unset RUBYOPT; bundle install"
233
+ exec(cmd)
234
+
235
+ when "python"
236
+ vDir = "#{meta['home_dir']}/vEnv"
237
+ lock_file = meta['home_dir']+"/zillabyte_thread_lock_file"
238
+ if File.exists?(lock_file)
239
+ sleep(1) while File.exists?(lock_file)
240
+ else
241
+ begin
242
+ cmd = "touch #{lock_file}; virtualenv --clear --system-site-packages #{vDir}; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{vDir}/bin/pip install -r #{meta['home_dir']}/requirements.txt"
243
+ system cmd, :out => :out
244
+ ensure
245
+ File.delete(lock_file)
246
+ end
247
+ end
248
+
249
+ when "js"
250
+ end
251
+
252
+ end
253
+ alias_command "prep", "apps:prep"
254
+
255
+
256
+
257
+
258
+ # apps:init [LANG] [DIR]
259
+ #
260
+ # initializes a new executable in DIR
261
+ # [LANG] defaults to ruby, and [DIR] to the current directory
262
+ #
263
+ # --type TYPE # specify an output type i.e. json
264
+ # --directory DIR # Directory of the app
265
+ #
266
+ #Examples:
267
+ #
268
+ # $ zillabyte apps:init python contact_extractor
269
+ #
270
+ def init
271
+
272
+ lang = options[:lang] || shift_argument || "ruby"
273
+ dir = options[:directory] || shift_argument
274
+ if dir.nil?
275
+ dir = Dir.pwd
276
+ else
277
+ dir = File.expand_path(dir)
278
+ end
279
+ type = options[:type]
280
+
281
+ languages = ["ruby","python", "js"]
282
+
283
+ error("Unsupported language #{lang}. We only support #{languages.join(', ')}.", type) if not languages.include? lang
284
+
285
+ display "initializing empty #{lang} app in #{dir}" if type.nil?
286
+ FileUtils.cp_r( File.expand_path("../templates/#{lang}", __FILE__) + "/." , dir )
287
+
288
+
289
+ end
290
+
291
+
292
+
293
+ # apps:logs ID [OPERATION_NAME]
294
+ #
295
+ # streams logs from the distributed workers
296
+ #
297
+ # --type TYPE # specify an output type i.e. json
298
+ # -v, --verbose LEVEL # sets the verbosity (error, info, debug) (default: info)
299
+ #
300
+ def logs
301
+
302
+ app_id = options[:id] || shift_argument
303
+ if app_id.nil?
304
+ app_id = read_name_from_conf(options)
305
+ options[:is_name] = true
306
+ elsif !(app_id =~ /^\d*$/)
307
+ options[:is_name] = true
308
+ end
309
+
310
+ operation_id = options[:operation] || shift_argument || '_ALL_'
311
+ category = options[:verbose] || '_ALL_'
312
+ type = options[:type]
313
+
314
+ carry_settings = {
315
+ :category => category
316
+ }
317
+
318
+ display "Retrieving logs for app ##{app_id}...please wait..." if type.nil?
319
+ lf = LogFormatter::Operation.new
320
+ self.api.logs.get(app_id, operation_id, options) do |line|
321
+
322
+ error(line['error_message'], type) if line['error']
323
+ lf.print_log_line(line) if type.nil?
324
+
325
+ end
326
+
327
+ end
328
+ alias_command "logs", "apps:logs"
329
+
330
+
331
+
332
+ # apps:errors ID
333
+ #
334
+ # Show recent errors generated by the app
335
+ # --type TYPE # specify an output type i.e. json
336
+ #
337
+ def errors
338
+
339
+ # Init
340
+ app_id = options[:id] || shift_argument
341
+
342
+ # No name?
343
+ if app_id.nil?
344
+ app_id = read_name_from_conf(options)
345
+ options[:is_name] = true
346
+ elsif !(app_id =~ /^\d*$/)
347
+ options[:is_name] = true
348
+ end
349
+
350
+ type = options[:type]
351
+
352
+ # Make the request
353
+ res = api.request(
354
+ :expects => 200,
355
+ :method => :get,
356
+ :body => options.to_json,
357
+ :path => "/flows/#{CGI.escape(app_id)}/errors"
358
+ )
359
+
360
+ # Render
361
+ display "Recent errors:" if type.nil?
362
+ headings = ["operation", "date", "error"]
363
+ rows = (res.body["recent_errors"] || []).map do |row|
364
+ if row['date']
365
+ d = Time.at(row['date']/1000)
366
+ else
367
+ d = nil
368
+ end
369
+ [row['name'], d, row['message']]
370
+ end
371
+ rows.sort! do |a,b|
372
+ a[1] <=> b[1]
373
+ end
374
+ color_map = {}
375
+ colors = LogFormatter::COLORS.clone
376
+ rows.each do |row|
377
+ name = row[0]
378
+ time = row[1]
379
+ message = row[2].strip
380
+ color_map[name] ||= colors.shift
381
+ if time
382
+ display "#{"* #{name} - #{time_ago_in_words(time)} ago".colorize(color_map[name])}:" if type.nil?
383
+ else
384
+ display "#{"* #{name}".colorize(color_map[name])}:" if type.nil?
385
+ end
386
+ message.split('\n').each do |sub_line|
387
+ display " #{sub_line}" if type.nil?
388
+ end
389
+ end
390
+
391
+ end
392
+
393
+ # apps:cycles ID [OPTIONS]
394
+ #
395
+ # operations on the app's cycles (batches).
396
+ # with no options, the command lists the apps cycles
397
+ # -n, --next # request the app to move to the next cycle
398
+ # -f, --forever # don't wait on cycles any more
399
+ # --type TYPE # specify an output type i.e. json
400
+ #
401
+ def cycles
402
+ app_id = options[:id] || shift_argument
403
+ type = options[:type]
404
+
405
+ trigger_next = options[:next] || false
406
+ trigger_forever = options[:forever] || false
407
+
408
+ if app_id.nil?
409
+ app_id = read_name_from_conf(options)
410
+ options[:is_name] = true
411
+ elsif !(app_id =~ /^\d*$/)
412
+ options[:is_name] = true
413
+ end
414
+
415
+ if trigger_next
416
+ # Trigger the next app
417
+ response = api.apps.create_cycle(app_id, options)
418
+ elsif trigger_forever
419
+ response = api.apps.run_forever(app_id, options)
420
+ else
421
+ # List the apps
422
+ response = api.apps.list_cycles(app_id, options)
423
+ # TODO List the sequence number for this app.
424
+ display "Most recent cyles of the app:" if type.nil?
425
+ headings = ["Cycle_id", "State", "Start", "End"]
426
+ rows = response["cycles"]
427
+ rows = rows.map do |row|
428
+ start_time = DateTime.parse(row["start"]).strftime("%m/%d/%Y %I:%M%p")
429
+ end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
430
+ [row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
431
+ end
432
+
433
+ display TableOutputBuilder.build_table(headings, rows, type)
434
+ display "Total number of cycles executed: #{response['total']}" if type.nil?
435
+ return
436
+ end
437
+
438
+ if response["job_id"]
439
+ options[:job_id] = response["job_id"]
440
+ app_id = response["flow_id"]
441
+ options.delete :is_name
442
+
443
+ start = Time.now.utc
444
+ display "Next cycle request sent. If your app was RETIRED this may take slightly longer." if type.nil?
445
+
446
+ while(Time.now.utc < start + MAX_POLL_SECONDS) do
447
+
448
+ # Poll
449
+ res = self.api.apps.cycles_poll(app_id, options)
450
+
451
+ case res['status']
452
+ when 'completed'
453
+ if res['return']
454
+ if type == "json"
455
+ return "{}"
456
+ else
457
+ display res['return']
458
+ end
459
+ else
460
+ throw "something is wrong: #{res}"
461
+ end
462
+ # success! continue below
463
+ break
464
+ when 'running'
465
+ sleep(POLL_SLEEP)
466
+ # display ".", false
467
+ else
468
+ throw "unknown status: #{res}"
469
+ end
470
+
471
+ end
472
+ elsif response["error"]
473
+ error(response["error"], type)
474
+ else
475
+ error("remote server error (a380)", type)
476
+ end
477
+ end
478
+
479
+
480
+ # apps:test [DIR]
481
+ #
482
+ # tests a local app with sample data
483
+ #
484
+ # --config CONFIG_FILE # use the given config file
485
+ # --output OUTPUT_FILE # writes sink output to a file
486
+ # --wait MAX # max time to spend on each operation (default 10 seconds)
487
+ # --batches BATCHES # number of batches to emit (default 1)
488
+ # --directory DIR # app directory
489
+ #
490
+ def test
491
+
492
+ output = options[:output]
493
+ otype = options[:type] #type is used below for something else
494
+
495
+ max_seconds = (options[:wait] || "30").to_i
496
+ batches = (options[:batches] || "1").to_i
497
+
498
+ def read_message(read_stream, color)
499
+ msg = nil
500
+ read_stream.each do |line|
501
+ line.strip!
502
+ if(line == "end")
503
+ return msg
504
+ end
505
+ begin
506
+ hash = JSON.parse(line)
507
+ if(hash["command"] == "done")
508
+ msg = "done"
509
+ else
510
+ msg = hash.to_json
511
+ end
512
+ rescue
513
+ next
514
+ end
515
+ end
516
+ msg
517
+ end
518
+
519
+ def write_message(write_stream, msg)
520
+ write_stream.write msg.strip + "\n"
521
+ write_stream.write "end\n"
522
+ write_stream.flush
523
+ end
524
+
525
+ def truncate_message(msg)
526
+ return msg if(!msg.instance_of?(String))
527
+ t_length = 50 # truncates entries to this length
528
+ m_length = msg.length
529
+ msg_out = m_length > t_length ? msg[0..t_length-3]+"..." : msg
530
+ msg_out
531
+ end
532
+
533
+ def handshake(write_stream, read_stream, node, color)
534
+ begin
535
+ write_message write_stream, "{\"pidDir\": \"/tmp\"}\n"
536
+ read_message read_stream, color # Read to "end\n"
537
+ rescue Exception => e
538
+ puts "Error handshaking node: #{node}"
539
+ raise e
540
+ end
541
+ end
542
+
543
+
544
+
545
+ # INIT
546
+ dir = options[:directory] || shift_argument
547
+ if dir.nil?
548
+ dir = Dir.pwd
549
+ else
550
+ dir = File.expand_path(dir)
551
+ end
552
+ options[:directory] = dir
553
+
554
+ meta = Zillabyte::API::Apps.get_rich_meta_info_from_script(dir, self, {:test => true})
555
+ if meta.nil?
556
+ error "this is not a valid zillabyte app directory"
557
+ exit
558
+ end
559
+
560
+ # Show the user what we know about their app...
561
+ display "inferring your app details..."
562
+ colors = {}
563
+ describe_app(meta, colors)
564
+
565
+
566
+ # Extract the app's information..
567
+ nodes = meta["nodes"]
568
+ write_to_next_each = []
569
+ write_queue = []
570
+ stream_messages = {}
571
+ default_stream = "_default"
572
+
573
+ # Iterate all nodes sequentially and invoke them in separate processes...
574
+ nodes.each do |node|
575
+
576
+ # Init
577
+ type = node["type"]
578
+ name = node["name"]
579
+ consumes = node["consumes"]
580
+ emits = node["emits"]
581
+
582
+ color = colors[name] || :default
583
+
584
+ op_display = lambda do |msg, override_color = nil|
585
+ display "#{name} - #{msg}".colorize(override_color || color)
586
+ end
587
+
588
+ # A Source?
589
+ if type == "source"
590
+
591
+ # A source from relation?
592
+ if node['matches'] or node["relation"]
593
+ matches = node['matches'] || (node["relation"]["query"])
594
+ emits = emits.first #For spouting from a relation, there should only be one emits
595
+ op_display.call "Grabbing remote data"
596
+
597
+ res = api.query.agnostic(matches)
598
+ rows = res["rows"]
599
+ column_aliases = res["column_aliases"]
600
+
601
+
602
+ if(rows.nil? or rows.length == 0)
603
+ raise NameError, "Could not find data that matches your 'matches' clause"
604
+ end
605
+ rows.each do |tuple|
606
+ values = {}
607
+ meta = {}
608
+ tuple.each do |k, v|
609
+ if(k == "id")
610
+ next
611
+ elsif(k == "confidence" or k == "since" or k == "source")
612
+ meta[k] = v
613
+ else
614
+ values[k] = v
615
+ end
616
+ end
617
+ read_msg = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
618
+ values = Hash[values.map{|k, v| [truncate_message(k), truncate_message(v)]}]
619
+ op_display.call "emitted: #{values} #{meta} to #{emits}"
620
+ stream_messages[emits] ||= []
621
+ stream_messages[emits] << read_msg
622
+ end
623
+
624
+ # Done processing...
625
+ next
626
+
627
+ else
628
+
629
+ # A regular source..
630
+ stream_messages[default_stream] ||= []
631
+ stream_messages[default_stream] << "{\"command\": \"begin_cycle\"}\n"
632
+ emits.each {|ss| stream_messages[ss] = []} #initialize streams
633
+ stream_size_at_last_call_to_next_tuple = Hash[emits.map {|ss| [ss, 0]}] #initialize initial size of streams (all 0)
634
+ # the above initializations are used to deal with the case where end_cycle_policy == "null_emit"
635
+ n_batches_emitted = 1
636
+ end_cycle_received = false
637
+ last_call_next_tuple = false
638
+
639
+ end
640
+
641
+ # An Aggregate?
642
+ elsif type == "aggregate"
643
+ if node['consumes']
644
+ input_stream = node['consumes']
645
+ else
646
+ input_stream = stream_messages.keys.first
647
+ end
648
+ messages = stream_messages[input_stream] || []
649
+ stream_messages[input_stream] = []
650
+
651
+ group_by = node['group_by']
652
+ group_tuples = {}
653
+ messages.each do |msg|
654
+ msg = JSON.parse(msg)
655
+ tuple = msg["tuple"].to_json
656
+ meta = msg["meta"].to_json
657
+ column_aliases = msg["column_aliases"] || {}
658
+ aliases = Hash[column_aliases.map{|h| [h["alias"],h["concrete_name"]]}]
659
+ gt = {}
660
+ group_by.each do |field|
661
+ field_name = aliases[field] || field
662
+ gt[field] = msg["tuple"][field_name]
663
+ end
664
+
665
+ msg_no_brackets = "\"tuple\": #{tuple}, \"meta\": #{meta}, \"column_aliases\": #{column_aliases.to_json}"
666
+ if group_tuples[gt]
667
+ group_tuples[gt] << msg_no_brackets
668
+ else
669
+ group_tuples[gt] = [msg_no_brackets]
670
+ end
671
+ end
672
+
673
+ group_tuples.each do |group_tuple, tuples|
674
+ stream_messages[input_stream] << "{\"command\": \"begin_group\", \"tuple\": #{group_tuple.to_json}, \"meta\":{}}\n"
675
+ tuples.each do |t|
676
+ stream_messages[input_stream] << "{\"command\": \"aggregate\", #{t}}\n"
677
+ end
678
+ stream_messages[input_stream] << "{\"command\": \"end_group\"}\n"
679
+ end
680
+
681
+ # A Sink?
682
+ elsif type == "sink"
683
+
684
+ if consumes.nil?
685
+ error "The node #{name} must declare which stream it 'consumes'"
686
+ end
687
+ messages = stream_messages[consumes] || []
688
+
689
+ table = Terminal::Table.new :title => name
690
+ csv_str = CSV.generate do |csv|
691
+ header_written = false;
692
+ messages.each do |msg|
693
+ obj = JSON.parse(msg)
694
+ if obj['tuple']
695
+ if header_written == false
696
+ keys = [obj['tuple'].keys, obj['meta'].keys].flatten
697
+ csv << keys
698
+ table << keys
699
+ table << :separator
700
+ header_written = true
701
+ end
702
+ vals = [obj['tuple'].values, obj['meta'].values].flatten
703
+ csv << vals
704
+ table << vals
705
+ end
706
+ end
707
+ end
708
+
709
+ display table.to_s.colorize(color)
710
+
711
+ if output
712
+ filename = "#{output}.csv"
713
+ f = File.open(filename, "w")
714
+ f.write(csv_str)
715
+ f.close()
716
+ op_display.call "output written to #{filename}"
717
+ end
718
+
719
+ next
720
+ end
721
+
722
+
723
+ cmd = command("--execute_live --name #{name}", otype, dir)
724
+ begin
725
+
726
+ # Start the operation...
727
+ op_display.call "beginning #{type} #{name}"
728
+ Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thread|
729
+ begin
730
+
731
+ # Init
732
+ handshake stdin, stdout, node, color
733
+ write_queue = []
734
+ read_queue = []
735
+
736
+ if consumes.nil?
737
+ # Assume default stream (this should only happen for the source)
738
+ stream_name = default_stream
739
+ else
740
+ stream_name = consumes
741
+ end
742
+ write_queue = stream_messages[stream_name].clone
743
+ stream_messages.delete(stream_name)
744
+
745
+ # Start writing the messages...
746
+ stuff_to_read = false
747
+ writing_thread = Thread.start do
748
+
749
+ while(true)
750
+
751
+ case type
752
+ when 'source'
753
+ break if n_batches_emitted > batches
754
+ if write_queue.empty?
755
+ sleep 0.5
756
+ next
757
+ end
758
+ else
759
+ break if write_queue.empty?
760
+ end
761
+
762
+ # Make sure we're not reading anything...
763
+ while(stuff_to_read) # TODO: semaphores
764
+ sleep 0.5 # spin wait
765
+ end
766
+
767
+ # Get next mesage
768
+ write_msg = write_queue.shift
769
+
770
+ # Make it human-understable
771
+ write_json = JSON.parse(write_msg)
772
+ if write_json['tuple']
773
+ display_hash = Hash[write_json['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
774
+ op_display.call "receiving: #{display_hash}"
775
+ elsif write_json['command'] == 'next'
776
+ last_call_next_tuple = true
777
+ op_display.call "getting next set of tuples in the batch"
778
+ else
779
+ puts write_json
780
+ end
781
+
782
+ # Actually send it to the process
783
+ begin
784
+ write_message stdin, write_msg
785
+ stuff_to_read = true
786
+ sleep 0.1
787
+ rescue Exception => e
788
+ puts "Error running #{cmd}: #{e}"
789
+ raise e
790
+ end
791
+ end
792
+ end
793
+
794
+ # Start reading messages...
795
+ reading_thread = Thread.start do
796
+ while(true)
797
+
798
+ # If the end cycle command is received, we either trigger the next cycle if the number of emitted
799
+ # cycles is less than what the user requested, or we break
800
+ if type == "source" and end_cycle_received
801
+ write_queue << "{\"command\": \"begin_cycle\"}\n"
802
+ n_batches_emitted += 1
803
+ end_cycle_received = false
804
+ last_call_next_tuple = false
805
+ stuff_to_read = false
806
+ break if n_batches_emitted > batches
807
+ sleep 0.5
808
+ next
809
+ end
810
+
811
+ # Get next message
812
+ read_msg = read_message(stdout, color)
813
+ if read_msg == "done" || read_msg.nil?
814
+ stuff_to_read = false
815
+
816
+ # For sources, if we receive a "done", check to see if any of the streams emitted by the source has
817
+ # increased in size since the last call to next_tuple. If so, the cycle isn't over, otherwise, the
818
+ # current call to next_tuple emitted nothing and if the end_cycle_policy is set to null_emit, this
819
+ # should end the current cycle.
820
+ if type == "source"
821
+ if last_call_next_tuple and node["end_cycle_policy"] == "null_emit"
822
+ end_cycle_received = true
823
+ emits.each do |ss|
824
+ end_cycle_received = false if stream_messages[ss].size > stream_size_at_last_call_to_next_tuple[ss]
825
+ break
826
+ end
827
+ next if end_cycle_received
828
+ end
829
+
830
+ # If the policy isn't "null_emit", then just request next_tuple again
831
+ write_queue << "{\"command\": \"next\"}\n"
832
+ end
833
+
834
+ # For other operations, if the queue is empty then we're done
835
+ if write_queue.empty?
836
+ break # exit while loop
837
+ else
838
+ sleep 0.5 # spin wait
839
+ next
840
+ end
841
+ end
842
+ stuff_to_read = true
843
+
844
+ # Process message
845
+ obj = JSON.parse(read_msg)
846
+
847
+ # process the received tuple or other commands
848
+ if obj['tuple']
849
+
850
+ # if
851
+ tt = obj['tuple']
852
+ tt.each do |kk, vv|
853
+ if tt[kk].nil? and type == "source" and node["end_cycle_policy"] == "null_emit"
854
+ end_cycle_received = true
855
+ # read rest of stuff in stdout buffer until "done"
856
+ mm = nil
857
+ while(mm != "done")
858
+ mm = read_message(stdout, color)
859
+ end
860
+ break
861
+ end
862
+ end
863
+ next if end_cycle_received
864
+
865
+ # Convert to a incoming tuple for the next operation
866
+ next_msg = {
867
+ :tuple => obj['tuple'],
868
+ :meta => obj['meta']
869
+ }
870
+ emit_stream = obj['stream']
871
+ stream_messages[emit_stream] ||= []
872
+ stream_messages[emit_stream] << next_msg.to_json
873
+
874
+ display_hash = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
875
+ op_display.call "emitted: #{display_hash} to #{emit_stream}"
876
+
877
+ # track stream message size to end cycles when necessary
878
+ stream_size_at_last_call_to_next_tuple[emit_stream] = stream_messages[emit_stream].size if type == "source"
879
+ elsif obj['command'] == 'end_cycle'
880
+ end_cycle_received = true
881
+ # command:end_cycle should always be followed by done, read it (below) so
882
+ # that it doesn't interfere with next
883
+ read_message(stdout, color)
884
+ elsif obj['command'] == 'log'
885
+ op_display.call "log: #{obj['msg']}"
886
+ elsif obj['command'] == 'fail'
887
+ op_display.call "error: #{obj['msg']}", :red
888
+ exit(1)
889
+ else
890
+ error "unknown message: #{read_msg}"
891
+ end
892
+
893
+ end
894
+ end
895
+
896
+ # stderr thread
897
+ stderr_thread = Thread.start do
898
+ stderr.each do |line|
899
+ op_display.call("stderr: #{line}", :red)
900
+ end
901
+ end
902
+
903
+ begin
904
+ killed = Timeout.timeout(max_seconds) do
905
+ reading_thread.join()
906
+ writing_thread.join()
907
+ stderr_thread.kill()
908
+ op_display.call "completed #{type} #{name}"
909
+ end
910
+ rescue Timeout::Error
911
+ op_display.call "max time reached. preempting #{type} #{name}. set --wait to increase", :red
912
+ reading_thread.kill() if reading_thread.alive?
913
+ writing_thread.kill() if writing_thread.alive?
914
+ stderr_thread.kill() if stderr_thread.alive?
915
+ end
916
+
917
+ rescue Errno::EIO
918
+ puts "Errno:EIO error, but this probably just means " +
919
+ "that the process has finished giving output"
920
+ end
921
+ end
922
+ rescue PTY::ChildExited
923
+ puts "The child process exited!"
924
+ end
925
+ end
926
+
927
+ end
928
+ alias_command "test", "apps:test"
929
+
930
+
931
+ # apps:kill ID
932
+ #
933
+ # kills the given app
934
+ #
935
+ # --config CONFIG_FILE # use the given config file
936
+ # --type TYPE # specify an output type i.e. json
937
+ #
938
+ def kill
939
+
940
+ id = options[:id] || shift_argument
941
+ type = options[:type]
942
+
943
+ if id.nil?
944
+ id = read_name_from_conf(options)
945
+ options[:is_name] = true
946
+ elsif !(id =~ /^\d*$/)
947
+ options[:is_name] = true
948
+ end
949
+
950
+ display "Killing app ##{id}...please wait..." if type.nil?
951
+ api.apps.kill(id, options)
952
+
953
+ if type == "json"
954
+ display "{}"
955
+ else
956
+ display "App ##{id} killed"
957
+ end
958
+
959
+ end
960
+
961
+
962
+ # apps:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]
963
+ #
964
+ # runs a local app with live data
965
+ #
966
+ # --config CONFIG_FILE # use the given config file
967
+ # --type TYPE # specify an output type i.e. json
968
+ # --directory DIR # Directory of the app
969
+ #
970
+ # HIDDEN:
971
+ def live_run
972
+
973
+ name = options[:name] || shift_argument
974
+ type = options[:type]
975
+
976
+ thread_id = options[:thread] || shift_argument || ""
977
+ dir = options[:directory] || shift_argument
978
+ if dir.nil?
979
+ dir = Dir.pwd
980
+ else
981
+ dir = File.expand_path(dir)
982
+ end
983
+ options[:directory] = dir
984
+
985
+ meta = Zillabyte::CLI::Config.get_config_info(dir, options)
986
+
987
+ if meta.nil?
988
+ error("could not find meta information for: #{dir}", type)
989
+ end
990
+
991
+ if(thread_id == "")
992
+ exec(command("--execute_live --name #{name.to_s}",type, dir))
993
+ else
994
+ exec(command("--execute_live --name #{name.to_s} --pipe #{thread_id}",type, dir))
995
+ end
996
+ end
997
+ alias_command "live_run", "apps:live_run"
998
+
999
+
1000
+ # apps:info [DIR]
1001
+ #
1002
+ # outputs the info for the app in the dir.
1003
+ #
1004
+ # --pretty # Pretty prints the info output
1005
+ # --type TYPE # specify an output type i.e. json
1006
+ # --directory DIR # Directory of the app
1007
+ #
1008
+ def info
1009
+
1010
+ dir = options[:directory] || shift_argument
1011
+ if dir.nil?
1012
+ dir = Dir.pwd
1013
+ else
1014
+ dir = File.expand_path(dir)
1015
+ end
1016
+ options[:directory] = dir
1017
+
1018
+ info_file = "#{dir}/#{SecureRandom.uuid}"
1019
+ type = options[:type]
1020
+
1021
+ cmd = command("--info --file #{info_file}", type, dir)
1022
+ <<<<<<< HEAD
1023
+ app_info = Zillabyte::Command::Apps.get_info(cmd, info_file, dir)
1024
+ =======
1025
+ app_info = Zillabyte::Command::Apps.get_info(cmd, info_file, options)
1026
+ >>>>>>> b7501a641428a0952068d1837b20293619aba0cd
1027
+
1028
+ if type == "json"
1029
+ puts app_info
1030
+ else
1031
+ if options[:pretty]
1032
+ puts JSON.pretty_generate(JSON.parse(app_info))
1033
+ else
1034
+ puts app_info
1035
+ end
1036
+ end
1037
+
1038
+ exit
1039
+ end
1040
+ alias_command "info", "apps:info"
1041
+
1042
+
1043
+ #
1044
+ # --type TYPE # specify an output type i.e. json
1045
+ #
1046
+ <<<<<<< HEAD
1047
+ def self.get_info(cmd, info_file, dir, options = {})
1048
+ =======
1049
+ def self.get_info(cmd, info_file, options = {})
1050
+
1051
+ >>>>>>> b7501a641428a0952068d1837b20293619aba0cd
1052
+ type = options[:type]
1053
+
1054
+ response = `#{cmd}`
1055
+ if($?.exitstatus == 1)
1056
+
1057
+ File.delete("#{info_file}") if File.exists?(info_file)
1058
+
1059
+ if options[:type].nil?
1060
+ exit(1)
1061
+ else
1062
+ Zillabyte::Helpers.error("error: #{response}", type)
1063
+ end
1064
+ end
1065
+
1066
+ app_info = {}
1067
+ File.open("#{info_file}", "r+").each do |line|
1068
+ line = JSON.parse(line)
1069
+ if(line["type"])
1070
+ app_info["nodes"] << line
1071
+ else
1072
+ app_info = line
1073
+ app_info["nodes"] = []
1074
+ end
1075
+ end
1076
+ File.delete("#{info_file}")
1077
+
1078
+ app_info = app_info.to_json
1079
+ if(File.exists?("#{dir}/info_to_java.in"))
1080
+ java_pipe = open("#{dir}/info_to_java.in","w+")
1081
+ java_pipe.puts(app_info+"\n")
1082
+ java_pipe.flush
1083
+ java_pipe.close()
1084
+ end
1085
+
1086
+ app_info
1087
+ end
1088
+
1089
+
1090
+ private
1091
+
1092
+ #
1093
+ # --type TYPE # specify an output type i.e. json
1094
+ #
1095
+ def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false)
1096
+ meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
1097
+
1098
+ #meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
1099
+ error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
1100
+ error(meta["error_message"], type) if meta['status'] == "error"
1101
+ full_script = File.join(dir, meta["script"])
1102
+ stderr_opt = "2> /dev/null" if ignore_stderr
1103
+
1104
+ case meta["language"]
1105
+ when "ruby"
1106
+ # Execute in the bundler context
1107
+ cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr_opt}"
1108
+
1109
+ when "python"#{
1110
+ if(File.directory?("#{dir}/vEnv"))
1111
+ cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr_opt}"
1112
+ else
1113
+ cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} #{stderr_opt}"
1114
+ end
1115
+
1116
+ when "js"
1117
+ # cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" #{arg}"
1118
+ cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
1119
+
1120
+ else
1121
+ error("no language specified", type)
1122
+ end
1123
+
1124
+ return cmd
1125
+
1126
+ end
1127
+
1128
+ #
1129
+ #
1130
+ #
1131
+ def describe_app(meta, colors = {})
1132
+ @colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
1133
+ rjust = 20
1134
+ display "#{'app name'.rjust(rjust)}: #{meta['name']}"
1135
+ display "#{'app language'.rjust(rjust)}: #{meta['language']}"
1136
+
1137
+ meta['nodes'].each_with_index do |node, index|
1138
+ colors[node['name']] ||= @colors.shift
1139
+ color = colors[node['name']]
1140
+ display (("="*rjust + " operation ##{index}").colorize(color))
1141
+ display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
1142
+ display "#{"type".rjust(rjust)}: #{node['type'].to_s.colorize(color)}"
1143
+ display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
1144
+ display "#{"emits".rjust(rjust)}: #{JSON.pretty_generate(node['emits']).indent(rjust+2).lstrip.colorize(color)}" if node['emits']
1145
+ end
1146
+
1147
+ end
1148
+
1149
+ #
1150
+ #
1151
+ def read_name_from_conf(options = {})
1152
+ type = options[:type]
1153
+ hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
1154
+ error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify an app id or run command from the directory containing the app.",type) if hash["error"]
1155
+ hash["name"]
1156
+ end
1157
+
1158
+ end