zillabyte-cli 0.0.24 → 0.1.0

data/lib/zillabyte/cli/flows.rb

@@ -0,0 +1,549 @@
+require "zillabyte/cli/base"
+require "zillabyte/cli/config"
+require "zillabyte/common"
+require "pty"
+require 'indentation'
+require 'open3'
+require 'securerandom'
+require 'colorize'
+require 'time_difference'
+require 'net/http'
+require "zillabyte/runner/component_runner"
+require "zillabyte/runner/app_runner"
+
+
+# HIDDEN: superclass for components and apps
+#
+class Zillabyte::Command::Flows < Zillabyte::Command::Base
+
+  # flows:status [DIR]
+  # 
+  # fetches detailed status of the flow
+  # 
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the flow
+  #
+  def status
+
+    id = options[:id] || shift_argument
+    type = options[:output_type]
+
+    if id.nil?
+      id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    res = api.request(
+      :expects => 200,
+      :method  => :get,
+      :path    => "/flows/#{id}"
+    )
+
+    res.body
+    if type == "json"
+      display res.body.to_json
+    else
+      # TODO
+      display res.body.to_json
+    end
+
+  end
+
+  # flows:pull ID DIR
+  #
+  # pulls a flow source to a directory.
+  #
+  # --force # pulls even if the directory exists
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the flow
+  #
+  # Examples:
+  #
+  # $ zillabyte flows:pull . 
+  #
+  def pull
+
+    flow_id = options[:id] || shift_argument
+    type = options[:output_type]
+
+    if !(flow_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    dir = options[:directory] || shift_argument
+    error("no directory given", type) if dir.nil?
+    dir = File.expand_path(dir)
+    
+    error("no id given", type) if flow_id.nil?
+    
+    # Create if not exists..
+    if File.exists?(dir)
+      if Dir.entries(dir).size != 2 and options[:force].nil?
+        error("target directory not empty. use --force to override", type)
+      end
+    else
+      FileUtils.mkdir_p(dir)
+    end
+    
+    res = api.flows.pull_to_directory flow_id, dir, session, options
+    
+    if res['error']
+      error("error: #{res['error_message']}", type)
+    else
+      if type == "json"
+        display "{}"
+      else
+        display "flow ##{res['id']} pulled to #{dir}"
+      end
+    end
+    
+  end
+  alias_command "pull", "flows:pull"
+
+  # flows:prep [DIR]
+  # 
+  # performs any necessary initialization for the flow
+  # 
+  # --directory DIR       # flow directory
+  # --output_type OUTPUT_TYPE           # specify an output type i.e. json
+  #   
+  def prep
+
+    type = options[:output_type]
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+    meta = Zillabyte::CLI::Config.get_config_info(dir)
+
+    if meta.nil?
+      error("The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file.", type)
+    end
+
+    case meta["language"]
+    when "ruby"
+
+      # Execute in the bundler context
+      full_script = File.join(dir, meta["script"])
+      cmd = "cd \"#{meta['home_dir']}\"; unset BUNDLE_GEMFILE; unset RUBYOPT; bundle install"
+      exec(cmd)
+      
+    when "python"
+      vDir = "#{meta['home_dir']}/vEnv"
+      lock_file = meta['home_dir']+"/zillabyte_thread_lock_file"
+      if File.exists?(lock_file)
+        sleep(1) while File.exists?(lock_file)
+      else
+        begin
+          cmd = "touch #{lock_file}; virtualenv --clear --system-site-packages #{vDir}; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{vDir}/bin/pip install -r #{meta['home_dir']}/requirements.txt"
+          system cmd, :out => :out
+        ensure
+          File.delete(lock_file)
+        end
+      end
+      
+    when "js"
+    end
+    
+  end
+  alias_command "prep", "flows:prep"
+
+  # flows:logs ID [OPERATION_NAME]
+  #
+  # streams logs from the distributed workers
+  # 
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  # --operation OPERATION_NAME # specify the operation to show logs for
+  # -v, --verbose LEVEL  # sets the verbosity (error, info, debug) (default: info)
+  #
+  def logs
+
+    flow_id = options[:id] || shift_argument
+    if flow_id.nil?
+      flow_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(flow_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    operation_id = options[:operation] || shift_argument || '_ALL_'
+    category = options[:verbose] || '_ALL_'
+    type = options[:output_type]
+
+    carry_settings = {
+      :category => category
+    }
+
+    display "Retrieving logs for flow ##{flow_id}...please wait..." if type.nil?
+    hash = self.api.logs.get(flow_id, operation_id, options)
+
+    fetch_logs(hash, operation_id)
+
+  end
+  alias_command "logs", "flows:logs"
+
+
+  # flows:errors ID
+  #
+  # show recent errors generated by the flow
+  # --output_type OUTPUT_TYPE           # specify an output type i.e. json
+  #
+  def errors
+
+    # Init
+    flow_id = options[:id] || shift_argument
+
+    # No name?
+    if flow_id.nil?
+      flow_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(flow_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    type = options[:output_type]
+
+    # Make the request
+    res = api.request(
+      :expects => 200,
+      :method => :get,
+      :body    => options.to_json,
+      :path => "/flows/#{CGI.escape(flow_id)}/errors"
+    )
+
+    # Render
+    display "Recent errors:" if type.nil?
+    headings = ["operation", "date", "error"]
+    rows = (res.body["recent_errors"] || []).map do |row|
+      if row['date']
+        d = Time.at(row['date']/1000)
+      else
+        d = nil
+      end
+      [row['name'], d, row['message']]
+    end
+    rows.sort! do |a,b|
+      a[1] <=> b[1]
+    end
+    color_map = {}
+    colors = LogFormatter::COLORS.clone
+    rows.each do |row|
+      name = row[0]
+      time = row[1]
+      message = row[2].strip
+      color_map[name] ||= colors.shift
+      if time
+        display "#{"* #{name} - #{time_ago_in_words(time)} ago".colorize(color_map[name])}:" if type.nil?
+      else
+        display "#{"* #{name}".colorize(color_map[name])}:" if type.nil?
+      end
+      message.split('\n').each do |sub_line|
+        display "    #{sub_line}" if type.nil?
+      end
+    end
+
+  end
+  alias_command "errors", "flows:errors"
+
+
+  # flows:live_run [OPERATION_NAME] [PIPE_NAME] [DIR]
+  #
+  # runs a local flow with live data
+  # 
+  # --config CONFIG_FILE # use the given config file
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the flow
+  #
+  def live_run
+
+    name = options[:name] || shift_argument
+    type = options[:output_type]
+
+    thread_id = options[:thread] || shift_argument || ""
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+
+    meta = Zillabyte::CLI::Config.get_config_info(dir, options)
+  
+    if meta.nil?
+      error("could not find meta information for: #{dir}", type)
+    end
+   
+    if(thread_id == "")
+      exec(command("--execute_live --name #{name.to_s}",type, dir))
+    else 
+      exec(command("--execute_live --name #{name.to_s} --pipe #{dir}/#{thread_id}",type, dir))
+    end
+  end 
+  alias_command "live_run", "flows:live_run"
+
+  # flows:info [DIR]
+  # 
+  # outputs the info for the flow in the dir.
+  # 
+  # --pretty # Pretty prints the info output
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the flow
+  #
+  def info
+
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+
+    info_file = "#{dir}/#{SecureRandom.uuid}"
+    type = options[:output_type]
+
+    cmd = command("--info --file #{info_file}", type, dir)
+    flow_info = Zillabyte::Command::Apps.get_info(cmd, info_file, dir, options)
+
+    if type == "json"
+      puts flow_info
+    else
+      if options[:pretty]
+        puts JSON.pretty_generate(JSON.parse(flow_info))
+      else
+        puts flow_info
+      end
+    end
+
+    exit
+  end
+  alias_command "info", "flows:info"
+
+  # flows:delete ID
+  #
+  # deletes a flow. if the flow is running, this command will kill it. 
+  #
+  # -f, --force # don't ask for confirmation
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  #
+  def delete
+    flow_id = options[:id] || shift_argument
+
+    if flow_id.nil?
+      flow_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(flow_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+    forced = options[:force]
+    type = options[:output_type]
+
+    if not forced
+
+      if !type.nil?
+        error("specify -f, --force to confirm deletion", type)
+      end
+
+      while true
+        display "This operation cannot be undone. Are you sure you want to delete this flow? (yes/no):", false
+        confirm = ask
+        break if confirm == "yes" || confirm == "no"
+        display "Please enter 'yes' to delete the flow or 'no' to exit"
+      end
+
+    end
+
+    confirmed = forced || confirm == "yes"
+
+    if confirmed
+      response = api.flows.delete(flow_id, options)
+      if type == "json"
+        display "{}"
+      else
+        display response["body"]
+      end
+    end
+
+  end
+
+
+  #
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  #
+  def self.get_info(cmd, info_file, dir, options = {})
+    type = options[:output_type]
+
+    response = `#{cmd}`
+    if($?.exitstatus == 1)
+
+      File.delete("#{info_file}") if File.exists?(info_file)
+
+      if options[:output_type].nil?
+        exit(1)
+      else
+        Zillabyte::Helpers.error("error: #{response}", type)
+      end
+    end
+
+    flow_info = {}
+    File.open("#{info_file}", "r+").each do |line|
+      line = JSON.parse(line)
+      if(line["class"] == "node")
+        flow_info["nodes"] << line["info"]
+      elsif(line["class"] == "arc")
+        flow_info["arcs"] << line["info"]
+      else
+        flow_info = line
+        flow_info["nodes"] = []
+        flow_info["arcs"] = []
+      end
+    end
+    File.delete("#{info_file}")
+
+    flow_info = flow_info.to_json
+    if(File.exists?("#{dir}/info_to_java.in"))
+      java_pipe = open("#{dir}/info_to_java.in","w+")
+      java_pipe.puts(flow_info+"\n")
+      java_pipe.flush
+      java_pipe.close()
+    end
+
+    flow_info
+  end
+
+
+
+
+  # flows:test
+  #
+  # tests a local flow with sample data
+  # 
+  # --config CONFIG_FILE            # use the given config file
+  # --input INPUT_FILE            # writes sink output to a file
+  # --output OUTPUT_FILE            # writes sink output to a file
+  # --cycles CYCLES                 # number of cycles to emit (default 1)
+  # --directory DIR                 # app directory
+  # -i, --interactive   # allow user to control input and read output
+  #
+  def test
+     dir = options[:directory]
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+    
+
+    # Get the class
+    flow_class = read_class_from_conf(options)
+
+    case flow_class
+    when "component"
+      runner = Zillabyte::Runner::ComponentRunner.new()
+    when "app"
+      runner = Zillabyte::Runner::AppRunner.new()
+    else
+      # Default to App
+      runner = Zillabyte::Runner::AppRunner.new()
+    end
+
+    # Start Runner
+    runner.run(dir, self, options)
+
+  end
+  alias_command "test", "flows:test"
+
+
+  private 
+  
+  #
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  #
+  def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false)
+    meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
+
+    #meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
+    error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
+    error(meta["error_message"], type) if meta['status'] == "error"
+    full_script = File.join(dir, meta["script"])
+    stderr_opt = "2> /dev/null" if ignore_stderr
+
+    case meta["language"]
+    when "ruby"
+      # Execute in the bundler context
+      cmd = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}  #{stderr_opt}"
+      
+    when "python"#{
+      if(File.directory?("#{dir}/vEnv"))
+        cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}  #{stderr_opt}"
+      else
+        cmd = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}  #{stderr_opt}"
+      end
+ 
+    when "js"
+#      cmd = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS}  \"#{full_script}\" #{arg}"
+      cmd = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN}  \"#{full_script}\" #{arg} #{stderr_opt}"
+
+    else
+      error("no language specified", type)
+    end
+    
+    return cmd
+
+  end
+
+  def read_name_from_conf(options = {})
+    type = options[:output_type]
+    hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
+    error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow.",type) if hash["error"]
+    hash["name"]
+  end
+
+  def read_class_from_conf(options = {})
+    type = options[:output_type]
+    hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
+    error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow.",type) if hash["error"]
+    hash["class"]
+  end
+
+  def fetch_logs(hash, operation=nil, exit_on=nil)
+    def color_for(operation_name)
+      @color_map[operation_name] ||= @all_colors[ @color_map.size % @all_colors.size ]
+      @color_map[operation_name]
+    end
+
+    @color_map = {}
+    @all_colors = [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
+    if(hash["server"] == "localhost")
+      cmd = "tail -n 500 -f /tmp/flows/f#{hash["flow"]}/flow_logs/flow_#{hash["flow"]}.log"
+    else
+      cmd = "curl -G http://#{hash["flow"]}:#{hash["token"]}@#{hash["server"]}:#{hash["port"]}/getLogs?flow=#{hash["flow"]}"
+    end
+
+    begin  
+      PTY.spawn( cmd ) do |r, w, pid|
+        r.each do |line|
+          begin
+            op = line.match(/\[(.*?)\]/)[0].to_s[1...-1]
+            op_name = op.split(".")[0]
+            next if op_name != operation and operation != "_ALL_"
+            line_split = line.split("[")
+            print line_split[0] + "[" + op.colorize(color_for(op)) + line_split[1..-1].join("[")[op.length..-1]
+            break if exit_on and line.include? exit_on
+          rescue Exception => e
+            next
+          end
+        end
+      end  
+    rescue PTY::ChildExited => e  
+    end  
+  end
+
+end

data/lib/zillabyte/cli/git.rb

@@ -0,0 +1,38 @@
+require "zillabyte/cli/base"
+
+# manage git for apps
+#
+class Zillabyte::Command::Git < Zillabyte::Command::Base
+
+  # git:remote [OPTIONS]
+  #
+  # adds a git remote to an app repo
+  #
+  # if OPTIONS are specified they will be passed to git remote add
+  #
+  # -r, --remote REMOTE        # the git remote to create, default "Zillabyte"
+  # -n, --name NAME            # the name of the app, default current directory's name
+  #
+  #Examples:
+  #
+  # $ zillabyte git:remote -a example
+  # Git remote zillabyte added
+  #
+  # $ zillabyte git:remote -a example
+  # !    Git remote zillabyte already exists
+  #
+  def remote
+    git_options = args.join(" ")
+    remote = options[:remote] || 'zillabyte'
+    app_name = options[:name] || File.basename(Dir.pwd)
+
+    if git('remote').split("\n").include?(remote)
+      # already exists.. remove it. 
+      git("remote remove #{remote}")
+    end
+
+    git_url = "git@#{Zillabyte::Auth.git_host}:#{Zillabyte::Auth.user}/#{app_name}"
+    create_git_remote(remote, git_url)
+  end
+
+end

data/lib/zillabyte/cli/help.rb

@@ -4,7 +4,7 @@ require "zillabyte/cli/base"
 #
 class Zillabyte::Command::Help < Zillabyte::Command::Base
 
-  PRIMARY_NAMESPACES = %w( relations query apps logs )
+  PRIMARY_NAMESPACES = %w( relations query apps components logs )
   
   # help
   def index(*direct_args)
@@ -85,13 +85,20 @@ private
 
   def summary_for_aliases(command_aliases)
     used_aliases = command_aliases.select{|c,a| !c.include?("-") and !c.include?(":") and !skip_command?(commands[a])}
-    a_size = longest(used_aliases.map { |al,al_to| al }) + 2
-    a_to_size = longest(used_aliases.map { |al,al_to| al_to }) + 2
+    a_size = longest(used_aliases.map { |al,al_to| al }) + 4
+    a_to_size = longest(used_aliases.map { |al,al_to| al_to }) + 4
     used_aliases.sort_by{|k,v| v}.each do |command_alias, alias_to|
+
       if command_alias.include?("-") or command_alias.include?(":")
         next #skip -h, -help and live-run etc.
       end
-      puts "#{command_alias.ljust(a_size)} --> #{alias_to.ljust(a_to_size)}  # #{commands[alias_to][:summary]}"
+
+      if alias_to.include?("flows")
+        puts "#{command_alias.ljust(a_size)} --> #{alias_to.gsub("flows", "apps").ljust(a_to_size)}  # #{commands[alias_to][:summary].gsub("flow", "app")}"
+        puts "#{" ".ljust(a_size)} --> #{alias_to.gsub("flows", "components").ljust(a_to_size)}  # #{commands[alias_to][:summary].gsub("flow", "component")}"
+      else
+        puts "#{command_alias.ljust(a_size)} --> #{alias_to.ljust(a_to_size)}  # #{commands[alias_to][:summary]}"
+      end
     end
   end