zillabyte-cli 0.1.44 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/#zillabyte-cli.rb# +5 -0
- data/lib/zillabyte-cli/version.rb +1 -1
- data/lib/zillabyte/api/components.rb +2 -2
- data/lib/zillabyte/api/flows.rb +0 -58
- data/lib/zillabyte/cli/#logs.rb# +12 -0
- data/lib/zillabyte/cli/#repl.rb# +43 -0
- data/lib/zillabyte/cli/data.rb +36 -6
- data/lib/zillabyte/cli/flows.rb +47 -103
- data/lib/zillabyte/cli/templates/python/#simple_function.py# +27 -0
- data/lib/zillabyte/common/log_fetcher.rb +7 -1
- data/lib/zillabyte/helpers.rb +118 -6
- data/zillabyte-cli.gemspec +1 -1
- metadata +7 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eea29081785924f4123050342b0106d98355f0c8
|
4
|
+
data.tar.gz: 9e5e777aa135171b8230ced0011622519e450539
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9b6441517bb14bd08ce83ddc6502e99287434f80403e6a1ee634c9880546ac772c32ea49bffe320737d72b7c70f1135e65862026df7ae89df3542f61447d5788
|
7
|
+
data.tar.gz: ad27686e26e9ebe20d2a4e38879449108dc2f8df4be5c70fecd97617cd409ec2f457dc337a09f3b4772144d5cf586d0e71533150a3551d7d5b5e4537e73e7924
|
@@ -26,12 +26,12 @@ class Zillabyte::API::Components < Zillabyte::API::Flows
|
|
26
26
|
|
27
27
|
nodes = component["schema"]["nodes"]
|
28
28
|
nodes.each do |node|
|
29
|
-
if node["type"] == "source"
|
29
|
+
if node["type"] == "input" or node["type"] == "source"
|
30
30
|
inputs += "#{node["name"]}:\n"
|
31
31
|
node["fields"].each do |field|
|
32
32
|
inputs += " (#{field.keys[0]}, #{field.values[0].upcase})\n"
|
33
33
|
end
|
34
|
-
elsif node["type"] == "sink"
|
34
|
+
elsif node["type"] == "output" or node["type"] == "sink"
|
35
35
|
outputs += "#{node["name"]}:\n"
|
36
36
|
node["columns"].each do |field|
|
37
37
|
outputs += " (#{field.keys[0]}, #{field.values[0].upcase})\n"
|
data/lib/zillabyte/api/flows.rb
CHANGED
@@ -197,66 +197,8 @@ class Zillabyte::API::Flows < Zillabyte::API::Base
|
|
197
197
|
res.body
|
198
198
|
|
199
199
|
end
|
200
|
-
|
201
|
-
####################################################IMPORTANT####################################################
|
202
|
-
# #
|
203
|
-
# get_rich_meta_info_from_script only works with zillabyte apps:push now! The pipe "info_to_ruby.in" is #
|
204
|
-
# hardcoded as the meta info passing pipe, so if 2 processes are trying to access it simultaneously there might #
|
205
|
-
# be problems since it gets created/destroyed within this script (attempting to create a fifo that already #
|
206
|
-
# exists or deleting a fifo that still has information in it may occur). #
|
207
|
-
# #
|
208
|
-
####################################################IMPORTANT####################################################
|
209
|
-
def self.get_rich_meta_info_from_script(dir, session = nil, options = {})
|
210
|
-
require("zillabyte/cli/config")
|
211
|
-
hash = Zillabyte::CLI::Config.get_config_info(dir, session, options)
|
212
|
-
if hash.nil?
|
213
|
-
return {"status" => "error", "error" => "invalid_directory", "error_message" => "The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file."}
|
214
|
-
end
|
215
|
-
hash["flow_type"] = "app" if hash["flow_type"].nil? # default to app
|
216
200
|
|
217
|
-
type = options[:output_type]
|
218
|
-
|
219
|
-
full_script = File.join(dir, hash['script'])
|
220
|
-
command = nil
|
221
|
-
|
222
|
-
require("securerandom")
|
223
|
-
info_file = "#{dir}/#{SecureRandom.uuid}"
|
224
|
-
arg = "--info --file \"#{info_file}\""
|
225
|
-
|
226
|
-
|
227
|
-
case hash["language"]
|
228
|
-
when "ruby"
|
229
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
230
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
231
|
-
when "python"
|
232
|
-
if(File.directory?("#{dir}/vEnv"))
|
233
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}"
|
234
|
-
else
|
235
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}"
|
236
|
-
end
|
237
|
-
when "js"
|
238
|
-
require("zillabyte/api/settings")
|
239
|
-
#command = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" --info"
|
240
|
-
command = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg}"
|
241
|
-
else
|
242
|
-
session.error "unsupported language: #{hash["language"]}" if session
|
243
|
-
return nil
|
244
|
-
end
|
245
201
|
|
246
|
-
require("zillabyte/cli/flows")
|
247
|
-
results = Zillabyte::Command::Flows.get_info(command, info_file, dir, options)
|
248
202
|
|
249
|
-
begin
|
250
|
-
meta = JSON.parse(results.split("\n").first.strip) # Throws error if invalid json
|
251
|
-
rescue Exception => e
|
252
|
-
session.error("the underlying app did not return a valid result. command: #{command} error: #{e.message}", type) if session
|
253
|
-
exit
|
254
|
-
end
|
255
|
-
|
256
|
-
# Merge with the new info
|
257
|
-
return hash.merge(meta)
|
258
|
-
|
259
|
-
end
|
260
|
-
|
261
203
|
|
262
204
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require "zillabyte/cli/base"
|
2
|
+
require 'readline'
|
3
|
+
# REPL console for zillabyte commands
|
4
|
+
#
|
5
|
+
class Zillabyte::Command::Repl < Zillabyte::Command::Base
|
6
|
+
|
7
|
+
# repl
|
8
|
+
#
|
9
|
+
# start a console session for zillabyte
|
10
|
+
# --quiet # HIDDEN
|
11
|
+
# --history HISTORY# HIDDEN hack to allow history for readline
|
12
|
+
def index
|
13
|
+
if !options[:quiet]
|
14
|
+
v = `zillabyte version`
|
15
|
+
display "\n#{v}Type q,exit or Ctrl+D to quit\n\n"
|
16
|
+
end
|
17
|
+
server = `echo $ZILLABYTE_API_HOST` || ""
|
18
|
+
prompt = ""
|
19
|
+
if server && server.chomp.length > 0
|
20
|
+
prompt = "#{server.chomp} "
|
21
|
+
end
|
22
|
+
prompt += "zillabyte $ "
|
23
|
+
if options[:history]
|
24
|
+
#p options[:history]
|
25
|
+
history = JSON.parse(options[:history])
|
26
|
+
history.last(50).each do |his|
|
27
|
+
# TODO: Handle single quotes ??
|
28
|
+
Readline::HISTORY << his
|
29
|
+
end
|
30
|
+
end
|
31
|
+
# TODO: Add tab completion for basic commands, app/relation names etc.
|
32
|
+
while cmd = Readline.readline(prompt, true)
|
33
|
+
if cmd && cmd.length > 0
|
34
|
+
if cmd.downcase == "exit" || cmd.downcase == "q"
|
35
|
+
display "" # TODO Make Ctrl+D print a newline too
|
36
|
+
return
|
37
|
+
else
|
38
|
+
exec "zillabyte #{cmd}; zillabyte repl --quiet --history '#{Readline::HISTORY.to_a.to_json}'"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
data/lib/zillabyte/cli/data.rb
CHANGED
@@ -449,7 +449,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
449
449
|
aliases = []
|
450
450
|
end
|
451
451
|
|
452
|
-
valid_types=["string", "integer", "float"]
|
452
|
+
valid_types=["string", "integer", "float", "double", "boolean", "array", "map"]
|
453
453
|
meta_names=["id", "confidence", "since", "source"]
|
454
454
|
|
455
455
|
if description.nil?
|
@@ -484,7 +484,7 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
484
484
|
def get_dataset_properties(schema, is_public, description = "", aliases = nil)
|
485
485
|
|
486
486
|
description ||= ""
|
487
|
-
valid_types=["string", "integer", "float"]
|
487
|
+
valid_types=["string", "integer", "float", "double", "boolean", "array", "map"]
|
488
488
|
meta_names=["id", "confidence", "since", "source"]
|
489
489
|
if schema.nil?
|
490
490
|
schema = ""
|
@@ -545,13 +545,19 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
545
545
|
n_columns = dataset["columns"].size
|
546
546
|
float_cols = []
|
547
547
|
int_cols = []
|
548
|
+
array_cols = []
|
549
|
+
map_cols = []
|
548
550
|
(0...n_columns).each do |i|
|
549
551
|
value = dataset["columns"][i].values[0].downcase
|
550
552
|
# use integer instead of numeric to be consistent with multilang type declarations
|
551
553
|
if(value == 'integer')
|
552
554
|
int_cols << i
|
553
|
-
elsif(value == "float")
|
555
|
+
elsif(value == "float" or value == "double")
|
554
556
|
float_cols << i
|
557
|
+
elsif(value == "array")
|
558
|
+
array_cols << i
|
559
|
+
elsif(value == "map")
|
560
|
+
map_cols << i
|
555
561
|
end
|
556
562
|
end
|
557
563
|
|
@@ -561,18 +567,42 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
561
567
|
require("csv")
|
562
568
|
CSV.foreach(file) do |row|
|
563
569
|
if row.size != n_columns
|
564
|
-
error("
|
570
|
+
error("Dataset expects #{n_columns} column(s). Found a row with #{row.size}::\n #{row}", type)
|
565
571
|
end
|
566
572
|
|
567
573
|
float_cols.each do |i|
|
568
574
|
if (!(row[i] =~ /^[+-]?(\d+(.\d+)?)$/))
|
569
|
-
|
575
|
+
error("Column #{i+1} should be a FLOAT or DOUBLE in row ::\n #{row}", type)
|
570
576
|
end
|
571
577
|
end
|
572
578
|
|
573
579
|
int_cols.each do |i|
|
574
580
|
if (!(row[i] =~ /^[+-]?(\d+)$/))
|
575
|
-
|
581
|
+
error("Column #{i+1} should be an INTEGER in row ::\n #{row}", type)
|
582
|
+
end
|
583
|
+
end
|
584
|
+
|
585
|
+
array_cols.each do |i|
|
586
|
+
err = "Column #{i+1} should be an ARRAY (i.e. [v0,v1,...]) in row ::\n #{row}\n Please remember to escape strings in the array!"
|
587
|
+
begin
|
588
|
+
array = JSON.parse(row[i])
|
589
|
+
rescue
|
590
|
+
error(err, type)
|
591
|
+
end
|
592
|
+
if !array.instance_of?(Array)
|
593
|
+
error(err, type)
|
594
|
+
end
|
595
|
+
end
|
596
|
+
|
597
|
+
map_cols.each do |i|
|
598
|
+
err = "Column #{i+1} should be a MAP (i.e. {\"k1\":v1,\"k2\":v2,...}) in row ::\n #{row}\n Please remember to escape strings in the map!"
|
599
|
+
begin
|
600
|
+
map = JSON.parse(row[i])
|
601
|
+
rescue
|
602
|
+
error(err, type)
|
603
|
+
end
|
604
|
+
if !map.instance_of?(Hash)
|
605
|
+
error(err, type)
|
576
606
|
end
|
577
607
|
end
|
578
608
|
rows << row
|
data/lib/zillabyte/cli/flows.rb
CHANGED
@@ -78,7 +78,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
78
78
|
# Get the meta info
|
79
79
|
require('zillabyte/api/flows')
|
80
80
|
options[:local_flag] = 1
|
81
|
-
hash =
|
81
|
+
hash = get_rich_info(dir, session, options)
|
82
82
|
if hash.nil?
|
83
83
|
session.error("unable to extract #{flow_type} meta information", type) if session
|
84
84
|
end
|
@@ -332,7 +332,12 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
332
332
|
instances = res.body["instances"].select do |instance|
|
333
333
|
!instance.nil?
|
334
334
|
end
|
335
|
-
|
335
|
+
if instances.empty?
|
336
|
+
if type
|
337
|
+
display res.body.to_json
|
338
|
+
end
|
339
|
+
return
|
340
|
+
end
|
336
341
|
|
337
342
|
# a nested hash so can't map directly
|
338
343
|
# instead i'll map separately and then merge the arrays
|
@@ -445,9 +450,11 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
445
450
|
#
|
446
451
|
# Runs a local flow with live data.
|
447
452
|
#
|
448
|
-
# --config CONFIG_FILE
|
449
|
-
# --dir DIR
|
450
|
-
# --output_type OUTPUT_TYPE
|
453
|
+
# --config CONFIG_FILE # Use the given config file
|
454
|
+
# --dir DIR # Flow directory
|
455
|
+
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
456
|
+
# --host HOST # #HIDDEN
|
457
|
+
# --port PORT # #HIDDEN
|
451
458
|
#
|
452
459
|
# HIDDEN:
|
453
460
|
def live_run
|
@@ -455,7 +462,9 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
455
462
|
name = options[:name] || shift_argument
|
456
463
|
type = options[:output_type]
|
457
464
|
|
458
|
-
|
465
|
+
host = options[:host]
|
466
|
+
port = options[:port]
|
467
|
+
|
459
468
|
dir = options[:dir] || shift_argument
|
460
469
|
if dir.nil?
|
461
470
|
dir = Dir.pwd
|
@@ -469,10 +478,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
469
478
|
error("could not find meta information for: #{dir}", type)
|
470
479
|
end
|
471
480
|
|
472
|
-
if(
|
473
|
-
exec(command("--execute_live --name #{name.to_s}",type, dir))
|
481
|
+
if(host && port)
|
482
|
+
exec(Zillabyte::Helpers.command("--execute_live --name #{name.to_s} --host \"#{host}\" --port #{port}",type, dir))
|
474
483
|
else
|
475
|
-
exec(command("--execute_live --name #{name.to_s}
|
484
|
+
exec(Zillabyte::Helpers.command("--execute_live --name #{name.to_s}",type, dir))
|
476
485
|
end
|
477
486
|
end
|
478
487
|
alias_command "live_run", "flows:live_run"
|
@@ -481,37 +490,47 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
481
490
|
#
|
482
491
|
# Outputs the info for the flow in the dir.
|
483
492
|
#
|
484
|
-
# --pretty
|
485
|
-
# --dir DIR
|
486
|
-
# --output_type OUTPUT_TYPE
|
493
|
+
# --pretty # Pretty prints the info output
|
494
|
+
# --dir DIR # Flow directory
|
495
|
+
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
496
|
+
# --host HOST # #HIDDEN
|
497
|
+
# --port PORT # #HIDDEN
|
487
498
|
#
|
488
499
|
def info
|
500
|
+
|
501
|
+
# Init
|
502
|
+
require 'socket'
|
503
|
+
type = options[:output_type]
|
489
504
|
dir = options[:dir] || shift_argument
|
490
505
|
if dir.nil?
|
491
506
|
dir = Dir.pwd
|
492
507
|
else
|
493
508
|
dir = File.expand_path(dir)
|
494
509
|
end
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
510
|
+
host = options[:host]
|
511
|
+
port = options[:port]
|
512
|
+
|
513
|
+
if (host && port)
|
514
|
+
|
515
|
+
# Do we have a host, port? If so, delegate to the multilang...
|
516
|
+
flow_info = Zillabyte::Helpers.get_info(dir, options)
|
517
|
+
socket = TCPSocket.new(host, port)
|
518
|
+
socket.write(flow_info.to_json)
|
519
|
+
socket.close()
|
520
|
+
exit
|
521
|
+
|
506
522
|
else
|
523
|
+
|
524
|
+
# Otherwise, start our own tcp socket and get the info, print to stdout
|
525
|
+
flow_info = Zillabyte::Helpers.get_info(dir, options)
|
507
526
|
if options[:pretty]
|
508
527
|
puts JSON.pretty_generate(JSON.parse(flow_info))
|
509
528
|
else
|
510
|
-
puts flow_info
|
529
|
+
puts flow_info.to_json
|
511
530
|
end
|
531
|
+
exit
|
532
|
+
|
512
533
|
end
|
513
|
-
|
514
|
-
exit
|
515
534
|
end
|
516
535
|
alias_command "info", "flows:info"
|
517
536
|
|
@@ -595,46 +614,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
595
614
|
end
|
596
615
|
|
597
616
|
|
598
|
-
def self.get_info(cmd, info_file, dir, options = {})
|
599
|
-
type = options[:output_type]
|
600
|
-
response = `#{cmd}`
|
601
|
-
if($?.exitstatus == 1)
|
602
|
-
|
603
|
-
File.delete("#{info_file}") if File.exists?(info_file)
|
604
|
-
|
605
|
-
if options[:output_type].nil?
|
606
|
-
exit(1)
|
607
|
-
else
|
608
|
-
Zillabyte::Helpers.error("error: #{response}", type)
|
609
|
-
end
|
610
|
-
end
|
611
|
-
|
612
|
-
flow_info = {}
|
613
|
-
File.open("#{info_file}", "r+").each do |line|
|
614
|
-
line = JSON.parse(line)
|
615
|
-
if(line["class"] == "node")
|
616
|
-
flow_info["nodes"] << line["info"]
|
617
|
-
elsif(line["class"] == "arc")
|
618
|
-
flow_info["arcs"] << line["info"]
|
619
|
-
else
|
620
|
-
flow_info = line
|
621
|
-
flow_info["nodes"] = []
|
622
|
-
flow_info["arcs"] = []
|
623
|
-
end
|
624
|
-
end
|
625
|
-
File.delete("#{info_file}")
|
626
|
-
|
627
|
-
flow_info = flow_info.to_json
|
628
|
-
if(File.exists?("#{dir}/info_to_java.in"))
|
629
|
-
java_pipe = open("#{dir}/info_to_java.in","w+")
|
630
|
-
java_pipe.puts(flow_info+"\n")
|
631
|
-
java_pipe.flush
|
632
|
-
java_pipe.close()
|
633
|
-
end
|
634
|
-
|
635
|
-
flow_info
|
636
|
-
end
|
637
|
-
|
638
617
|
|
639
618
|
# flows:local
|
640
619
|
#
|
@@ -657,7 +636,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
657
636
|
|
658
637
|
# Get the flow_type
|
659
638
|
require("zillabyte/api/flows")
|
660
|
-
meta =
|
639
|
+
meta = get_rich_info(dir, session, {:test => true})
|
661
640
|
if meta.nil? || meta["nodes"].nil?
|
662
641
|
error "this is not a valid zillabyte flow directory"
|
663
642
|
exit
|
@@ -755,46 +734,11 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
755
734
|
|
756
735
|
|
757
736
|
private
|
758
|
-
|
759
|
-
|
760
|
-
def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false)
|
761
|
-
require("zillabyte/cli/config")
|
762
|
-
meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
|
763
|
-
|
764
|
-
#meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
|
765
|
-
error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
|
766
|
-
error(meta["error_message"], type) if meta['status'] == "error"
|
767
|
-
full_script = File.join(dir, meta["script"])
|
768
|
-
stderr_opt = "2> /dev/null" if ignore_stderr
|
769
|
-
|
770
|
-
case meta["language"]
|
771
|
-
when "ruby"
|
772
|
-
# Execute in the bundler context
|
773
|
-
cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr_opt}"
|
774
|
-
|
775
|
-
when "python"#{
|
776
|
-
if(File.directory?("#{dir}/vEnv"))
|
777
|
-
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr_opt}"
|
778
|
-
else
|
779
|
-
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} #{stderr_opt}"
|
780
|
-
end
|
781
|
-
|
782
|
-
when "js"
|
783
|
-
# cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" #{arg}"
|
784
|
-
cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
|
785
|
-
|
786
|
-
else
|
787
|
-
error("no language specified", type)
|
788
|
-
end
|
789
|
-
|
790
|
-
return cmd
|
791
|
-
|
792
|
-
end
|
793
737
|
|
794
738
|
def read_name_from_conf(options = {})
|
795
739
|
type = options[:output_type]
|
796
740
|
require("zillabyte/api/apps")
|
797
|
-
hash =
|
741
|
+
hash = get_rich_info(Dir.pwd, options)
|
798
742
|
error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow.",type) if hash["error"]
|
799
743
|
hash["name"]
|
800
744
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
import zillabyte
|
2
|
+
|
3
|
+
def prep(controller):
|
4
|
+
return
|
5
|
+
|
6
|
+
# This is the heart of your algorithm. It's processed on every
|
7
|
+
# web page. This algorithm is run in parallel on possibly hundreds
|
8
|
+
# of machines.
|
9
|
+
def execute(controller, tup):
|
10
|
+
if("hello world" in tup.values["html"]):
|
11
|
+
controller.emit("has_hello_world",{"url":tup.values["url"]})
|
12
|
+
return
|
13
|
+
|
14
|
+
zillabyte.simple_function(\
|
15
|
+
# This directive instructs zillabyte to give your function every
|
16
|
+
# web page in our known universe. Your function will have access
|
17
|
+
# to two fields: URL and HTML
|
18
|
+
matches = "select * from web_pa", \
|
19
|
+
|
20
|
+
# This directive tells Zillabyte what kind of data your function
|
21
|
+
# produces. In this case, we're saying we will emit a tuple that
|
22
|
+
# is one-column wide and contains the field 'URL'
|
23
|
+
emits = [["has_hello_world", [{"url":"string"}]]], \
|
24
|
+
|
25
|
+
prepare = prep, \
|
26
|
+
execute = execute\
|
27
|
+
)
|
@@ -23,7 +23,13 @@ class LogFetcher
|
|
23
23
|
|
24
24
|
@color_map = {}
|
25
25
|
|
26
|
-
|
26
|
+
local_file = "/tmp/flows/f#{hash["flow"]}/flow_logs/flow_#{hash["flow"]}.log"
|
27
|
+
if(hash["server"] == "localhost" && File.exists?(local_file))
|
28
|
+
cmd = "tail -n 500 -F \"#{local_file}\""
|
29
|
+
else
|
30
|
+
cmd = "curl -G http://#{hash["flow"]}:#{hash["token"]}@#{ip}:#{hash["port"]}/getLogs?flow=#{hash["flow"]}"
|
31
|
+
end
|
32
|
+
|
27
33
|
|
28
34
|
begin
|
29
35
|
PTY.spawn( cmd ) do |r, w, pid|
|
data/lib/zillabyte/helpers.rb
CHANGED
@@ -151,12 +151,12 @@ module Zillabyte
|
|
151
151
|
end
|
152
152
|
|
153
153
|
|
154
|
-
def get_flow_name(dir = Dir.pwd)
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
end
|
154
|
+
# def get_flow_name(dir = Dir.pwd)
|
155
|
+
# require("zillabyte/api/flows")
|
156
|
+
# meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir)
|
157
|
+
# return meta["name"] if meta
|
158
|
+
# return nil
|
159
|
+
# end
|
160
160
|
|
161
161
|
|
162
162
|
# Format a message for display
|
@@ -190,6 +190,118 @@ module Zillabyte
|
|
190
190
|
error "expected remote server to return file manifest"
|
191
191
|
end
|
192
192
|
end
|
193
|
+
|
194
|
+
|
195
|
+
|
196
|
+
|
197
|
+
# Formerly: get_rich_meta_info_from_script.
|
198
|
+
# Combines the zilabyte.conf.yaml with 'zillabyte info'
|
199
|
+
def get_rich_info(dir, session = nil, options = {})
|
200
|
+
require("zillabyte/cli/config")
|
201
|
+
hash = Zillabyte::CLI::Config.get_config_info(dir, session, options)
|
202
|
+
if hash.nil?
|
203
|
+
return {"status" => "error", "error" => "invalid_directory", "error_message" => "The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file."}
|
204
|
+
end
|
205
|
+
|
206
|
+
hash["flow_type"] = "app" if hash["flow_type"].nil? # default to app
|
207
|
+
flow_info = get_info(dir, options)
|
208
|
+
|
209
|
+
return hash.merge(flow_info)
|
210
|
+
|
211
|
+
end
|
212
|
+
|
213
|
+
|
214
|
+
def get_info(dir, options = {})
|
215
|
+
|
216
|
+
require 'socket'
|
217
|
+
host = "0.0.0.0"
|
218
|
+
type = options[:output_type]
|
219
|
+
raw_lines = []
|
220
|
+
server = TCPServer.new(0)
|
221
|
+
port = server.addr[1]
|
222
|
+
|
223
|
+
server_thread = Thread.new do
|
224
|
+
|
225
|
+
client = server.accept()
|
226
|
+
loop do
|
227
|
+
raw_line = client.gets()
|
228
|
+
break if raw_line.nil?
|
229
|
+
raw_lines << raw_line
|
230
|
+
end
|
231
|
+
client.close()
|
232
|
+
end
|
233
|
+
|
234
|
+
cmd = command("--info --host \"#{host}\" --port #{port}", type, dir)
|
235
|
+
system(cmd)
|
236
|
+
server_thread.join()
|
237
|
+
|
238
|
+
if($?.exitstatus == 1)
|
239
|
+
if options[:output_type].nil?
|
240
|
+
exit(1)
|
241
|
+
else
|
242
|
+
Zillabyte::Helpers.error("error: #{response}", type)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
if (raw_lines.size == 0)
|
246
|
+
if options[:output_type].nil?
|
247
|
+
exit(1)
|
248
|
+
else
|
249
|
+
Zillabyte::Helpers.error("process did not emit any information", type)
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
flow_info = {}
|
254
|
+
raw_lines.each do |raw_line|
|
255
|
+
line = JSON.parse(raw_line)
|
256
|
+
if(line["class"] == "node")
|
257
|
+
flow_info["nodes"] << line["info"]
|
258
|
+
elsif(line["class"] == "arc")
|
259
|
+
flow_info["arcs"] << line["info"]
|
260
|
+
else
|
261
|
+
flow_info = line
|
262
|
+
flow_info["nodes"] = []
|
263
|
+
flow_info["arcs"] = []
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
flow_info
|
268
|
+
|
269
|
+
end
|
270
|
+
|
271
|
+
def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false, options = {})
|
272
|
+
require("zillabyte/cli/config")
|
273
|
+
meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
|
274
|
+
|
275
|
+
#meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
|
276
|
+
error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
|
277
|
+
error(meta["error_message"], type) if meta['status'] == "error"
|
278
|
+
full_script = File.join(dir, meta["script"])
|
279
|
+
stderr_opt = "2> /dev/null" if ignore_stderr
|
280
|
+
|
281
|
+
case meta["language"]
|
282
|
+
when "ruby"
|
283
|
+
# Execute in the bundler context
|
284
|
+
cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg} #{stderr_opt}"
|
285
|
+
|
286
|
+
when "python"#{
|
287
|
+
if(File.directory?("#{dir}/vEnv"))
|
288
|
+
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg} #{stderr_opt}"
|
289
|
+
else
|
290
|
+
cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg} #{stderr_opt}"
|
291
|
+
end
|
292
|
+
|
293
|
+
when "js"
|
294
|
+
# cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" #{arg}"
|
295
|
+
cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg} #{stderr_opt}"
|
296
|
+
|
297
|
+
else
|
298
|
+
error("no language specified", type)
|
299
|
+
end
|
300
|
+
|
301
|
+
return cmd
|
302
|
+
|
303
|
+
end
|
304
|
+
|
193
305
|
|
194
306
|
end
|
195
307
|
end
|
data/zillabyte-cli.gemspec
CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.add_dependency "bundler", "~> 1.3"
|
34
34
|
spec.add_dependency "colorize", "~> 0.6"
|
35
35
|
spec.add_dependency "indentation", "~> 0.1"
|
36
|
-
spec.add_dependency "
|
36
|
+
spec.add_dependency "eventmachine"
|
37
37
|
|
38
38
|
spec.post_install_message = """
|
39
39
|
Getting Started with Zillabyte
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-09-
|
11
|
+
date: 2014-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -151,7 +151,7 @@ dependencies:
|
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.1'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
|
-
name:
|
154
|
+
name: eventmachine
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
157
|
- - ">="
|
@@ -179,6 +179,7 @@ files:
|
|
179
179
|
- bin/zb
|
180
180
|
- bin/zbd
|
181
181
|
- bin/zillabyte
|
182
|
+
- lib/#zillabyte-cli.rb#
|
182
183
|
- lib/zillabyte-cli.rb
|
183
184
|
- lib/zillabyte-cli/version.rb
|
184
185
|
- lib/zillabyte/api.rb
|
@@ -197,6 +198,8 @@ files:
|
|
197
198
|
- lib/zillabyte/api/zillalogs.rb
|
198
199
|
- lib/zillabyte/auth.rb
|
199
200
|
- lib/zillabyte/cli.rb
|
201
|
+
- lib/zillabyte/cli/#logs.rb#
|
202
|
+
- lib/zillabyte/cli/#repl.rb#
|
200
203
|
- lib/zillabyte/cli/apps.rb
|
201
204
|
- lib/zillabyte/cli/auth.rb
|
202
205
|
- lib/zillabyte/cli/base.rb
|
@@ -233,6 +236,7 @@ files:
|
|
233
236
|
- lib/zillabyte/cli/templates/components/ruby/Gemfile
|
234
237
|
- lib/zillabyte/cli/templates/components/ruby/component.rb.erb
|
235
238
|
- lib/zillabyte/cli/templates/components/ruby/zillabyte.conf.yaml
|
239
|
+
- lib/zillabyte/cli/templates/python/#simple_function.py#
|
236
240
|
- lib/zillabyte/cli/untitled.md
|
237
241
|
- lib/zillabyte/cli/version.rb
|
238
242
|
- lib/zillabyte/cli/zillalogs.rb
|