zillabyte-cli 0.1.44 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/#zillabyte-cli.rb# +5 -0
- data/lib/zillabyte-cli/version.rb +1 -1
- data/lib/zillabyte/api/components.rb +2 -2
- data/lib/zillabyte/api/flows.rb +0 -58
- data/lib/zillabyte/cli/#logs.rb# +12 -0
- data/lib/zillabyte/cli/#repl.rb# +43 -0
- data/lib/zillabyte/cli/data.rb +36 -6
- data/lib/zillabyte/cli/flows.rb +47 -103
- data/lib/zillabyte/cli/templates/python/#simple_function.py# +27 -0
- data/lib/zillabyte/common/log_fetcher.rb +7 -1
- data/lib/zillabyte/helpers.rb +118 -6
- data/zillabyte-cli.gemspec +1 -1
- metadata +7 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eea29081785924f4123050342b0106d98355f0c8
|
4
|
+
data.tar.gz: 9e5e777aa135171b8230ced0011622519e450539
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9b6441517bb14bd08ce83ddc6502e99287434f80403e6a1ee634c9880546ac772c32ea49bffe320737d72b7c70f1135e65862026df7ae89df3542f61447d5788
|
7
|
+
data.tar.gz: ad27686e26e9ebe20d2a4e38879449108dc2f8df4be5c70fecd97617cd409ec2f457dc337a09f3b4772144d5cf586d0e71533150a3551d7d5b5e4537e73e7924
|
@@ -26,12 +26,12 @@ class Zillabyte::API::Components < Zillabyte::API::Flows
|
|
26
26
|
|
27
27
|
nodes = component["schema"]["nodes"]
|
28
28
|
nodes.each do |node|
|
29
|
-
if node["type"] == "source"
|
29
|
+
if node["type"] == "input" or node["type"] == "source"
|
30
30
|
inputs += "#{node["name"]}:\n"
|
31
31
|
node["fields"].each do |field|
|
32
32
|
inputs += " (#{field.keys[0]}, #{field.values[0].upcase})\n"
|
33
33
|
end
|
34
|
-
elsif node["type"] == "sink"
|
34
|
+
elsif node["type"] == "output" or node["type"] == "sink"
|
35
35
|
outputs += "#{node["name"]}:\n"
|
36
36
|
node["columns"].each do |field|
|
37
37
|
outputs += " (#{field.keys[0]}, #{field.values[0].upcase})\n"
|
data/lib/zillabyte/api/flows.rb
CHANGED
@@ -197,66 +197,8 @@ class Zillabyte::API::Flows < Zillabyte::API::Base
|
|
197
197
|
res.body
|
198
198
|
|
199
199
|
end
|
200
|
-
|
201
|
-
####################################################IMPORTANT####################################################
|
202
|
-
# #
|
203
|
-
# get_rich_meta_info_from_script only works with zillabyte apps:push now! The pipe "info_to_ruby.in" is #
|
204
|
-
# hardcoded as the meta info passing pipe, so if 2 processes are trying to access it simultaneously there might #
|
205
|
-
# be problems since it gets created/destroyed within this script (attempting to create a fifo that already #
|
206
|
-
# exists or deleting a fifo that still has information in it may occur). #
|
207
|
-
# #
|
208
|
-
####################################################IMPORTANT####################################################
|
209
|
-
def self.get_rich_meta_info_from_script(dir, session = nil, options = {})
|
210
|
-
require("zillabyte/cli/config")
|
211
|
-
hash = Zillabyte::CLI::Config.get_config_info(dir, session, options)
|
212
|
-
if hash.nil?
|
213
|
-
return {"status" => "error", "error" => "invalid_directory", "error_message" => "The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file."}
|
214
|
-
end
|
215
|
-
hash["flow_type"] = "app" if hash["flow_type"].nil? # default to app
|
216
200
|
|
217
|
-
type = options[:output_type]
|
218
|
-
|
219
|
-
full_script = File.join(dir, hash['script'])
|
220
|
-
command = nil
|
221
|
-
|
222
|
-
require("securerandom")
|
223
|
-
info_file = "#{dir}/#{SecureRandom.uuid}"
|
224
|
-
arg = "--info --file \"#{info_file}\""
|
225
|
-
|
226
|
-
|
227
|
-
case hash["language"]
|
228
|
-
when "ruby"
|
229
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
230
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
231
|
-
when "python"
|
232
|
-
if(File.directory?("#{dir}/vEnv"))
|
233
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}"
|
234
|
-
else
|
235
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}"
|
236
|
-
end
|
237
|
-
when "js"
|
238
|
-
require("zillabyte/api/settings")
|
239
|
-
#command = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" --info"
|
240
|
-
command = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg}"
|
241
|
-
else
|
242
|
-
session.error "unsupported language: #{hash["language"]}" if session
|
243
|
-
return nil
|
244
|
-
end
|
245
201
|
|
246
|
-
require("zillabyte/cli/flows")
|
247
|
-
results = Zillabyte::Command::Flows.get_info(command, info_file, dir, options)
|
248
202
|
|
249
|
-
begin
|
250
|
-
meta = JSON.parse(results.split("\n").first.strip) # Throws error if invalid json
|
251
|
-
rescue Exception => e
|
252
|
-
session.error("the underlying app did not return a valid result. command: #{command} error: #{e.message}", type) if session
|
253
|
-
exit
|
254
|
-
end
|
255
|
-
|
256
|
-
# Merge with the new info
|
257
|
-
return hash.merge(meta)
|
258
|
-
|
259
|
-
end
|
260
|
-
|
261
203
|
|
262
204
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require "zillabyte/cli/base"
|
2
|
+
require 'readline'
|
3
|
+
# REPL console for zillabyte commands
|
4
|
+
#
|
5
|
+
class Zillabyte::Command::Repl < Zillabyte::Command::Base
|
6
|
+
|
7
|
+
# repl
|
8
|
+
#
|
9
|
+
# start a console session for zillabyte
|
10
|
+
# --quiet # HIDDEN
|
11
|
+
# --history HISTORY# HIDDEN hack to allow history for readline
|
12
|
+
def index
|
13
|
+
if !options[:quiet]
|
14
|
+
v = `zillabyte version`
|
15
|
+
display "\n#{v}Type q,exit or Ctrl+D to quit\n\n"
|
16
|
+
end
|
17
|
+
server = `echo $ZILLABYTE_API_HOST` || ""
|
18
|
+
prompt = ""
|
19
|
+
if server && server.chomp.length > 0
|
20
|
+
prompt = "#{server.chomp} "
|
21
|
+
end
|
22
|
+
prompt += "zillabyte $ "
|
23
|
+
if options[:history]
|
24
|
+
#p options[:history]
|
25
|
+
history = JSON.parse(options[:history])
|
26
|
+
history.last(50).each do |his|
|
27
|
+
# TODO: Handle single quotes ??
|
28
|
+
Readline::HISTORY << his
|
29
|
+
end
|
30
|
+
end
|
31
|
+
# TODO: Add tab completion for basic commands, app/relation names etc.
|
32
|
+
while cmd = Readline.readline(prompt, true)
|
33
|
+
if cmd && cmd.length > 0
|
34
|
+
if cmd.downcase == "exit" || cmd.downcase == "q"
|
35
|
+
display "" # TODO Make Ctrl+D print a newline too
|
36
|
+
return
|
37
|
+
else
|
38
|
+
exec "zillabyte #{cmd}; zillabyte repl --quiet --history '#{Readline::HISTORY.to_a.to_json}'"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
data/lib/zillabyte/cli/data.rb
CHANGED
@@ -449,7 +449,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
449
449
|
aliases = []
|
450
450
|
end
|
451
451
|
|
452
|
-
valid_types=["string", "integer", "float"]
|
452
|
+
valid_types=["string", "integer", "float", "double", "boolean", "array", "map"]
|
453
453
|
meta_names=["id", "confidence", "since", "source"]
|
454
454
|
|
455
455
|
if description.nil?
|
@@ -484,7 +484,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
484
484
|
def get_dataset_properties(schema, is_public, description = "", aliases = nil)
|
485
485
|
|
486
486
|
description ||= ""
|
487
|
-
valid_types=["string", "integer", "float"]
|
487
|
+
valid_types=["string", "integer", "float", "double", "boolean", "array", "map"]
|
488
488
|
meta_names=["id", "confidence", "since", "source"]
|
489
489
|
if schema.nil?
|
490
490
|
schema = ""
|
@@ -545,13 +545,19 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
545
545
|
n_columns = dataset["columns"].size
|
546
546
|
float_cols = []
|
547
547
|
int_cols = []
|
548
|
+
array_cols = []
|
549
|
+
map_cols = []
|
548
550
|
(0...n_columns).each do |i|
|
549
551
|
value = dataset["columns"][i].values[0].downcase
|
550
552
|
# use integer instead of numeric to be consistent with multilang type declarations
|
551
553
|
if(value == 'integer')
|
552
554
|
int_cols << i
|
553
|
-
elsif(value == "float")
|
555
|
+
elsif(value == "float" or value == "double")
|
554
556
|
float_cols << i
|
557
|
+
elsif(value == "array")
|
558
|
+
array_cols << i
|
559
|
+
elsif(value == "map")
|
560
|
+
map_cols << i
|
555
561
|
end
|
556
562
|
end
|
557
563
|
|
@@ -561,18 +567,42 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
561
567
|
require("csv")
|
562
568
|
CSV.foreach(file) do |row|
|
563
569
|
if row.size != n_columns
|
564
|
-
error("
|
570
|
+
error("Dataset expects #{n_columns} column(s). Found a row with #{row.size}::\n #{row}", type)
|
565
571
|
end
|
566
572
|
|
567
573
|
float_cols.each do |i|
|
568
574
|
if (!(row[i] =~ /^[+-]?(\d+(.\d+)?)$/))
|
569
|
-
|
575
|
+
error("Column #{i+1} should be a FLOAT or DOUBLE in row ::\n #{row}", type)
|
570
576
|
end
|
571
577
|
end
|
572
578
|
|
573
579
|
int_cols.each do |i|
|
574
580
|
if (!(row[i] =~ /^[+-]?(\d+)$/))
|
575
|
-
|
581
|
+
error("Column #{i+1} should be an INTEGER in row ::\n #{row}", type)
|
582
|
+
end
|
583
|
+
end
|
584
|
+
|
585
|
+
array_cols.each do |i|
|
586
|
+
err = "Column #{i+1} should be an ARRAY (i.e. [v0,v1,...]) in row ::\n #{row}\n Please remember to escape strings in the array!"
|
587
|
+
begin
|
588
|
+
array = JSON.parse(row[i])
|
589
|
+
rescue
|
590
|
+
error(err, type)
|
591
|
+
end
|
592
|
+
if !array.instance_of?(Array)
|
593
|
+
error(err, type)
|
594
|
+
end
|
595
|
+
end
|
596
|
+
|
597
|
+
map_cols.each do |i|
|
598
|
+
err = "Column #{i+1} should be a MAP (i.e. {\"k1\":v1,\"k2\":v2,...}) in row ::\n #{row}\n Please remember to escape strings in the map!"
|
599
|
+
begin
|
600
|
+
map = JSON.parse(row[i])
|
601
|
+
rescue
|
602
|
+
error(err, type)
|
603
|
+
end
|
604
|
+
if !map.instance_of?(Hash)
|
605
|
+
error(err, type)
|
576
606
|
end
|
577
607
|
end
|
578
608
|
rows << row
|
data/lib/zillabyte/cli/flows.rb
CHANGED
@@ -78,7 +78,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
78
78
|
# Get the meta info
|
79
79
|
require('zillabyte/api/flows')
|
80
80
|
options[:local_flag] = 1
|
81
|
-
hash =
|
81
|
+
hash = get_rich_info(dir, session, options)
|
82
82
|
if hash.nil?
|
83
83
|
session.error("unable to extract #{flow_type} meta information", type) if session
|
84
84
|
end
|
@@ -332,7 +332,12 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
332
332
|
instances = res.body["instances"].select do |instance|
|
333
333
|
!instance.nil?
|
334
334
|
end
|
335
|
-
|
335
|
+
if instances.empty?
|
336
|
+
if type
|
337
|
+
display res.body.to_json
|
338
|
+
end
|
339
|
+
return
|
340
|
+
end
|
336
341
|
|
337
342
|
# a nested hash so can't map directly
|
338
343
|
# instead i'll map separately and then merge the arrays
|
@@ -445,9 +450,11 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
445
450
|
#
|
446
451
|
# Runs a local flow with live data.
|
447
452
|
#
|
448
|
-
# --config CONFIG_FILE
|
449
|
-
# --dir DIR
|
450
|
-
# --output_type OUTPUT_TYPE
|
453
|
+
# --config CONFIG_FILE # Use the given config file
|
454
|
+
# --dir DIR # Flow directory
|
455
|
+
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
456
|
+
# --host HOST # #HIDDEN
|
457
|
+
# --port PORT # #HIDDEN
|
451
458
|
#
|
452
459
|
# HIDDEN:
|
453
460
|
def live_run
|
@@ -455,7 +462,9 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
455
462
|
name = options[:name] || shift_argument
|
456
463
|
type = options[:output_type]
|
457
464
|
|
458
|
-
|
465
|
+
host = options[:host]
|
466
|
+
port = options[:port]
|
467
|
+
|
459
468
|
dir = options[:dir] || shift_argument
|
460
469
|
if dir.nil?
|
461
470
|
dir = Dir.pwd
|
@@ -469,10 +478,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
469
478
|
error("could not find meta information for: #{dir}", type)
|
470
479
|
end
|
471
480
|
|
472
|
-
if(
|
473
|
-
exec(command("--execute_live --name #{name.to_s}",type, dir))
|
481
|
+
if(host && port)
|
482
|
+
exec(Zillabyte::Helpers.command("--execute_live --name #{name.to_s} --host \"#{host}\" --port #{port}",type, dir))
|
474
483
|
else
|
475
|
-
exec(command("--execute_live --name #{name.to_s}
|
484
|
+
exec(Zillabyte::Helpers.command("--execute_live --name #{name.to_s}",type, dir))
|
476
485
|
end
|
477
486
|
end
|
478
487
|
alias_command "live_run", "flows:live_run"
|
@@ -481,37 +490,47 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
481
490
|
#
|
482
491
|
# Outputs the info for the flow in the dir.
|
483
492
|
#
|
484
|
-
# --pretty
|
485
|
-
# --dir DIR
|
486
|
-
# --output_type OUTPUT_TYPE
|
493
|
+
# --pretty # Pretty prints the info output
|
494
|
+
# --dir DIR # Flow directory
|
495
|
+
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
496
|
+
# --host HOST # #HIDDEN
|
497
|
+
# --port PORT # #HIDDEN
|
487
498
|
#
|
488
499
|
def info
|
500
|
+
|
501
|
+
# Init
|
502
|
+
require 'socket'
|
503
|
+
type = options[:output_type]
|
489
504
|
dir = options[:dir] || shift_argument
|
490
505
|
if dir.nil?
|
491
506
|
dir = Dir.pwd
|
492
507
|
else
|
493
508
|
dir = File.expand_path(dir)
|
494
509
|
end
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
510
|
+
host = options[:host]
|
511
|
+
port = options[:port]
|
512
|
+
|
513
|
+
if (host && port)
|
514
|
+
|
515
|
+
# Do we have a host, port? If so, delegate to the multilang...
|
516
|
+
flow_info = Zillabyte::Helpers.get_info(dir, options)
|
517
|
+
socket = TCPSocket.new(host, port)
|
518
|
+
socket.write(flow_info.to_json)
|
519
|
+
socket.close()
|
520
|
+
exit
|
521
|
+
|
506
522
|
else
|
523
|
+
|
524
|
+
# Otherwise, start our own tcp socket and get the info, print to stdout
|
525
|
+
flow_info = Zillabyte::Helpers.get_info(dir, options)
|
507
526
|
if options[:pretty]
|
508
527
|
puts JSON.pretty_generate(JSON.parse(flow_info))
|
509
528
|
else
|
510
|
-
puts flow_info
|
529
|
+
puts flow_info.to_json
|
511
530
|
end
|
531
|
+
exit
|
532
|
+
|
512
533
|
end
|
513
|
-
|
514
|
-
exit
|
515
534
|
end
|
516
535
|
alias_command "info", "flows:info"
|
517
536
|
|
@@ -595,46 +614,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
595
614
|
end
|
596
615
|
|
597
616
|
|
598
|
-
def self.get_info(cmd, info_file, dir, options = {})
|
599
|
-
type = options[:output_type]
|
600
|
-
response = `#{cmd}`
|
601
|
-
if($?.exitstatus == 1)
|
602
|
-
|
603
|
-
File.delete("#{info_file}") if File.exists?(info_file)
|
604
|
-
|
605
|
-
if options[:output_type].nil?
|
606
|
-
exit(1)
|
607
|
-
else
|
608
|
-
Zillabyte::Helpers.error("error: #{response}", type)
|
609
|
-
end
|
610
|
-
end
|
611
|
-
|
612
|
-
flow_info = {}
|
613
|
-
File.open("#{info_file}", "r+").each do |line|
|
614
|
-
line = JSON.parse(line)
|
615
|
-
if(line["class"] == "node")
|
616
|
-
flow_info["nodes"] << line["info"]
|
617
|
-
elsif(line["class"] == "arc")
|
618
|
-
flow_info["arcs"] << line["info"]
|
619
|
-
else
|
620
|
-
flow_info = line
|
621
|
-
flow_info["nodes"] = []
|
622
|
-
flow_info["arcs"] = []
|
623
|
-
end
|
624
|
-
end
|
625
|
-
File.delete("#{info_file}")
|
626
|
-
|
627
|
-
flow_info = flow_info.to_json
|
628
|
-
if(File.exists?("#{dir}/info_to_java.in"))
|
629
|
-
java_pipe = open("#{dir}/info_to_java.in","w+")
|
630
|
-
java_pipe.puts(flow_info+"\n")
|
631
|
-
java_pipe.flush
|
632
|
-
java_pipe.close()
|
633
|
-
end
|
634
|
-
|
635
|
-
flow_info
|
636
|
-
end
|
637
|
-
|
638
617
|
|
639
618
|
# flows:local
|
640
619
|
#
|
@@ -657,7 +636,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
657
636
|
|
658
637
|
# Get the flow_type
|
659
638
|
require("zillabyte/api/flows")
|
660
|
-
meta =
|
639
|
+
meta = get_rich_info(dir, session, {:test => true})
|
661
640
|
if meta.nil? || meta["nodes"].nil?
|
662
641
|
error "this is not a valid zillabyte flow directory"
|
663
642
|
exit
|
@@ -755,46 +734,11 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
755
734
|
|
756
735
|
|
757
736
|
private
|
758
|
-
|
759
|
-
|
760
|
-
def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false)
|
761
|
-
require("zillabyte/cli/config")
|
762
|
-
meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
|
763
|
-
|
764
|
-
#meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
|
765
|
-
error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
|
766
|
-
error(meta["error_message"], type) if meta['status'] == "error"
|
767
|
-
full_script = File.join(dir, meta["script"])
|
768
|
-
stderr_opt = "2> /dev/null" if ignore_stderr
|
769
|
-
|
770
|
-
case meta["language"]
|
771
|
-
when "ruby"
|
772
|
-
# Execute in the bundler context
|
773
|
-
cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr_opt}"
|
774
|
-
|
775
|
-
when "python"#{
|
776
|
-
if(File.directory?("#{dir}/vEnv"))
|
777
|
-
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr_opt}"
|
778
|
-
else
|
779
|
-
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} #{stderr_opt}"
|
780
|
-
end
|
781
|
-
|
782
|
-
when "js"
|
783
|
-
# cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" #{arg}"
|
784
|
-
cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
|
785
|
-
|
786
|
-
else
|
787
|
-
error("no language specified", type)
|
788
|
-
end
|
789
|
-
|
790
|
-
return cmd
|
791
|
-
|
792
|
-
end
|
793
737
|
|
794
738
|
def read_name_from_conf(options = {})
|
795
739
|
type = options[:output_type]
|
796
740
|
require("zillabyte/api/apps")
|
797
|
-
hash =
|
741
|
+
hash = get_rich_info(Dir.pwd, options)
|
798
742
|
error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow.",type) if hash["error"]
|
799
743
|
hash["name"]
|
800
744
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
import zillabyte
|
2
|
+
|
3
|
+
def prep(controller):
|
4
|
+
return
|
5
|
+
|
6
|
+
# This is the heart of your algorithm. It's processed on every
|
7
|
+
# web page. This algorithm is run in parallel on possibly hundreds
|
8
|
+
# of machines.
|
9
|
+
def execute(controller, tup):
|
10
|
+
if("hello world" in tup.values["html"]):
|
11
|
+
controller.emit("has_hello_world",{"url":tup.values["url"]})
|
12
|
+
return
|
13
|
+
|
14
|
+
zillabyte.simple_function(\
|
15
|
+
# This directive instructs zillabyte to give your function every
|
16
|
+
# web page in our known universe. Your function will have access
|
17
|
+
# to two fields: URL and HTML
|
18
|
+
matches = "select * from web_pa", \
|
19
|
+
|
20
|
+
# This directive tells Zillabyte what kind of data your function
|
21
|
+
# produces. In this case, we're saying we will emit a tuple that
|
22
|
+
# is one-column wide and contains the field 'URL'
|
23
|
+
emits = [["has_hello_world", [{"url":"string"}]]], \
|
24
|
+
|
25
|
+
prepare = prep, \
|
26
|
+
execute = execute\
|
27
|
+
)
|
@@ -23,7 +23,13 @@ class LogFetcher
|
|
23
23
|
|
24
24
|
@color_map = {}
|
25
25
|
|
26
|
-
|
26
|
+
local_file = "/tmp/flows/f#{hash["flow"]}/flow_logs/flow_#{hash["flow"]}.log"
|
27
|
+
if(hash["server"] == "localhost" && File.exists?(local_file))
|
28
|
+
cmd = "tail -n 500 -F \"#{local_file}\""
|
29
|
+
else
|
30
|
+
cmd = "curl -G http://#{hash["flow"]}:#{hash["token"]}@#{ip}:#{hash["port"]}/getLogs?flow=#{hash["flow"]}"
|
31
|
+
end
|
32
|
+
|
27
33
|
|
28
34
|
begin
|
29
35
|
PTY.spawn( cmd ) do |r, w, pid|
|
data/lib/zillabyte/helpers.rb
CHANGED
@@ -151,12 +151,12 @@ module Zillabyte
|
|
151
151
|
end
|
152
152
|
|
153
153
|
|
154
|
-
def get_flow_name(dir = Dir.pwd)
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
end
|
154
|
+
# def get_flow_name(dir = Dir.pwd)
|
155
|
+
# require("zillabyte/api/flows")
|
156
|
+
# meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir)
|
157
|
+
# return meta["name"] if meta
|
158
|
+
# return nil
|
159
|
+
# end
|
160
160
|
|
161
161
|
|
162
162
|
# Format a message for display
|
@@ -190,6 +190,118 @@ module Zillabyte
|
|
190
190
|
error "expected remote server to return file manifest"
|
191
191
|
end
|
192
192
|
end
|
193
|
+
|
194
|
+
|
195
|
+
|
196
|
+
|
197
|
+
# Formerly: get_rich_meta_info_from_script.
|
198
|
+
# Combines the zilabyte.conf.yaml with 'zillabyte info'
|
199
|
+
def get_rich_info(dir, session = nil, options = {})
|
200
|
+
require("zillabyte/cli/config")
|
201
|
+
hash = Zillabyte::CLI::Config.get_config_info(dir, session, options)
|
202
|
+
if hash.nil?
|
203
|
+
return {"status" => "error", "error" => "invalid_directory", "error_message" => "The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file."}
|
204
|
+
end
|
205
|
+
|
206
|
+
hash["flow_type"] = "app" if hash["flow_type"].nil? # default to app
|
207
|
+
flow_info = get_info(dir, options)
|
208
|
+
|
209
|
+
return hash.merge(flow_info)
|
210
|
+
|
211
|
+
end
|
212
|
+
|
213
|
+
|
214
|
+
def get_info(dir, options = {})
|
215
|
+
|
216
|
+
require 'socket'
|
217
|
+
host = "0.0.0.0"
|
218
|
+
type = options[:output_type]
|
219
|
+
raw_lines = []
|
220
|
+
server = TCPServer.new(0)
|
221
|
+
port = server.addr[1]
|
222
|
+
|
223
|
+
server_thread = Thread.new do
|
224
|
+
|
225
|
+
client = server.accept()
|
226
|
+
loop do
|
227
|
+
raw_line = client.gets()
|
228
|
+
break if raw_line.nil?
|
229
|
+
raw_lines << raw_line
|
230
|
+
end
|
231
|
+
client.close()
|
232
|
+
end
|
233
|
+
|
234
|
+
cmd = command("--info --host \"#{host}\" --port #{port}", type, dir)
|
235
|
+
system(cmd)
|
236
|
+
server_thread.join()
|
237
|
+
|
238
|
+
if($?.exitstatus == 1)
|
239
|
+
if options[:output_type].nil?
|
240
|
+
exit(1)
|
241
|
+
else
|
242
|
+
Zillabyte::Helpers.error("error: #{response}", type)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
if (raw_lines.size == 0)
|
246
|
+
if options[:output_type].nil?
|
247
|
+
exit(1)
|
248
|
+
else
|
249
|
+
Zillabyte::Helpers.error("process did not emit any information", type)
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
flow_info = {}
|
254
|
+
raw_lines.each do |raw_line|
|
255
|
+
line = JSON.parse(raw_line)
|
256
|
+
if(line["class"] == "node")
|
257
|
+
flow_info["nodes"] << line["info"]
|
258
|
+
elsif(line["class"] == "arc")
|
259
|
+
flow_info["arcs"] << line["info"]
|
260
|
+
else
|
261
|
+
flow_info = line
|
262
|
+
flow_info["nodes"] = []
|
263
|
+
flow_info["arcs"] = []
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
flow_info
|
268
|
+
|
269
|
+
end
|
270
|
+
|
271
|
+
def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false, options = {})
|
272
|
+
require("zillabyte/cli/config")
|
273
|
+
meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
|
274
|
+
|
275
|
+
#meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
|
276
|
+
error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
|
277
|
+
error(meta["error_message"], type) if meta['status'] == "error"
|
278
|
+
full_script = File.join(dir, meta["script"])
|
279
|
+
stderr_opt = "2> /dev/null" if ignore_stderr
|
280
|
+
|
281
|
+
case meta["language"]
|
282
|
+
when "ruby"
|
283
|
+
# Execute in the bundler context
|
284
|
+
cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr_opt}"
|
285
|
+
|
286
|
+
when "python"#{
|
287
|
+
if(File.directory?("#{dir}/vEnv"))
|
288
|
+
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr_opt}"
|
289
|
+
else
|
290
|
+
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} #{stderr_opt}"
|
291
|
+
end
|
292
|
+
|
293
|
+
when "js"
|
294
|
+
# cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" #{arg}"
|
295
|
+
cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
|
296
|
+
|
297
|
+
else
|
298
|
+
error("no language specified", type)
|
299
|
+
end
|
300
|
+
|
301
|
+
return cmd
|
302
|
+
|
303
|
+
end
|
304
|
+
|
193
305
|
|
194
306
|
end
|
195
307
|
end
|
data/zillabyte-cli.gemspec
CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.add_dependency "bundler", "~> 1.3"
|
34
34
|
spec.add_dependency "colorize", "~> 0.6"
|
35
35
|
spec.add_dependency "indentation", "~> 0.1"
|
36
|
-
spec.add_dependency "
|
36
|
+
spec.add_dependency "eventmachine"
|
37
37
|
|
38
38
|
spec.post_install_message = """
|
39
39
|
Getting Started with Zillabyte
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-09-
|
11
|
+
date: 2014-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -151,7 +151,7 @@ dependencies:
|
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.1'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
|
-
name:
|
154
|
+
name: eventmachine
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
157
|
- - ">="
|
@@ -179,6 +179,7 @@ files:
|
|
179
179
|
- bin/zb
|
180
180
|
- bin/zbd
|
181
181
|
- bin/zillabyte
|
182
|
+
- lib/#zillabyte-cli.rb#
|
182
183
|
- lib/zillabyte-cli.rb
|
183
184
|
- lib/zillabyte-cli/version.rb
|
184
185
|
- lib/zillabyte/api.rb
|
@@ -197,6 +198,8 @@ files:
|
|
197
198
|
- lib/zillabyte/api/zillalogs.rb
|
198
199
|
- lib/zillabyte/auth.rb
|
199
200
|
- lib/zillabyte/cli.rb
|
201
|
+
- lib/zillabyte/cli/#logs.rb#
|
202
|
+
- lib/zillabyte/cli/#repl.rb#
|
200
203
|
- lib/zillabyte/cli/apps.rb
|
201
204
|
- lib/zillabyte/cli/auth.rb
|
202
205
|
- lib/zillabyte/cli/base.rb
|
@@ -233,6 +236,7 @@ files:
|
|
233
236
|
- lib/zillabyte/cli/templates/components/ruby/Gemfile
|
234
237
|
- lib/zillabyte/cli/templates/components/ruby/component.rb.erb
|
235
238
|
- lib/zillabyte/cli/templates/components/ruby/zillabyte.conf.yaml
|
239
|
+
- lib/zillabyte/cli/templates/python/#simple_function.py#
|
236
240
|
- lib/zillabyte/cli/untitled.md
|
237
241
|
- lib/zillabyte/cli/version.rb
|
238
242
|
- lib/zillabyte/cli/zillalogs.rb
|