zillabyte-cli 0.0.24 → 0.1.0

data/lib/zillabyte/cli/keys.rb

@@ -0,0 +1,177 @@
+require "zillabyte/cli/base"
+require "zillabyte/cli/helpers/table_output_builder"
+require "cgi"
+require "csv"
+require "open-uri"
+require "json"
+
+
+# manage authentication keys
+#
+class Zillabyte::Command::Keys < Zillabyte::Command::Base
+
+  SHORT_KEY_SPLIT = 18
+  SHORT_KEY_MAX = 64
+
+  # keys:add NAME [KEY]
+  #
+  #  add a key for the current user
+  #
+  #  if no KEY is specified, will try to find ~/.ssh/id_rsa.pub
+  #
+  #  --output_type OUTPUT_TYPE # The output format type
+  #
+  def add()
+    type = options[:output_type] || nil
+    name = shift_argument()
+    if name.nil?
+      error("no name given", type)
+    end
+    keypath = shift_argument() || "~/.ssh/id_rsa.pub"
+    begin
+      key = File.binread(File.expand_path(keypath))
+    rescue => e
+      error(e.message, type)
+    end
+    message = api.keys.add(name, key)
+    if type == "json"
+      display("{}")
+    else
+      display(message)
+    end
+  end
+
+  # keys:show NAME
+  #
+  #  show a key for the current user
+  #
+  #  --output_type OUTPUT_TYPE # The output format type
+  #
+  def show()
+    type = options[:output_type] || nil
+    name = shift_argument()
+    if name.nil?()
+      error("no name given", type)
+    end
+
+    key = api.keys.show(name)
+    if type == "json"
+      display({"key" => key}.to_json())
+    else
+      display(key)
+    end
+  end
+
+  # keys:remove [NAME]
+  #
+  #  remove a key from the current user
+  #
+  #  --output_type OUTPUT_TYPE # The output format type
+  #  -l, --long                # display extended information for each key
+  #  -f, --force               # Force -- remove key without prompting
+  #
+  def remove()
+    type = options[:output_type] || nil
+    long = options[:long]
+    force = options[:force]
+    name = shift_argument()
+    key_data = if name.nil?()
+      if force
+        error("no name given", type)
+      end
+      keys = api.keys.list()
+      display("=== Keys")
+      display(format_keys(keys, long, nil))
+      # NOTE: it may not be the most efficient thing to use a hash here, unless
+      # we expect users to routinely try to remove invalid keys.
+      keymap = Hash[*keys.flatten()]
+      loop do
+        display("Please enter the name of the key you wish to remove (case-sensitive):")
+        name = ask().chomp()
+        if name.empty?()
+          error("Aborting.", type)
+        end
+        if keymap.has_key?(name)
+          break(keymap[name])
+        end
+        display("Key not found: `#{name}`")
+      end
+    else
+      key = api.keys.show(name)
+      if force
+        key
+      else
+        display(format_keys([[name, key]], long, nil))
+        loop do
+          display("Proceed with removal? (yes or no)")
+          choice = ask().chomp()
+          if choice == "y" || choice == "yes"
+            break(key)
+          elsif choice == "n" || choice == "no"
+            error("Aborting.", type)
+          else
+            display("Invalid response.")
+          end
+        end
+      end
+      key
+    end
+
+    message = api.keys.remove(name, key_data)
+    if type == "json"
+      display("{}")
+    else
+      display(message)
+    end
+  end
+
+  # keys
+  #
+  #  display keys for the current user
+  #
+  #  --output_type OUTPUT_TYPE # The output format type
+  #  -l, --long                # display extended information for each key
+  def index()
+    type = options[:output_type] || nil
+    long = options[:long]
+
+    keys = api.keys.list()
+    if type.nil?()
+      display("=== Keys")
+    end
+    display(format_keys(keys, long, type))
+  end
+
+  # keys:clear
+  #
+  #  remove all authentication keys from the current user
+  #
+  #  --output_type OUTPUT_TYPE # The output format type
+  #
+  def clear()
+    type = options[:output_type] || nil
+
+    message = api.keys.clear()
+    if type == "json"
+      display("{}")
+    else
+      display(message)
+    end
+  end
+
+  private
+  def format_keys(keys, long, type)
+    outlong = long || type == "json"
+    rows = keys.map() { |row|
+      name = row[0]
+      key = row[1]
+      if outlong || key.length() < SHORT_KEY_MAX
+        [key, name]
+      else
+        ["#{key[0...SHORT_KEY_SPLIT]}...#{key[SHORT_KEY_SPLIT - SHORT_KEY_MAX + 2..-1]}", name]
+      end
+    }
+    headings = ["key", "name"]
+    TableOutputBuilder.build_table(headings, rows, type)
+  end
+end

data/lib/zillabyte/cli/query.rb

@@ -264,7 +264,6 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
     )
     res = res.body
     
-    p res
     if(res["uri"])
       display "Waiting for download." if type.nil?
       File.open(file, "w") do |f|

data/lib/zillabyte/cli/relations.rb

@@ -520,6 +520,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
     case filetype
     when "csv"
       begin
+
         CSV.foreach(file) do |row|
           if row.size != n_columns
             error("relation expect #{n_columns} column(s). Found a row with #{row.size}::\n #{row}", type)
@@ -539,7 +540,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
           rows << row
         end
       rescue Exception => e
-        error("unable to parse csv", type)
+        error("error when parsing csv : #{e}", type)
       end
       
     else

data/lib/zillabyte/cli/templates/apps/python/app.py

@@ -0,0 +1,17 @@
+import zillabyte
+
+def prep(controller):
+  return
+
+# This is the heart of your algorithm.  It's processed on every
+# web page.  This algorithm is run in parallel on possibly hundreds
+# of machines.
+def execute(controller, tup):
+  if("hello world" in tup["html"]):
+    controller.emit("has_hello_world",{"url":tup["url"]})
+  return
+
+app = zillabyte.app(name = "hello_world")
+app.source(matches = "select * from web_pages")\
+   .each(execute = execute)\
+   .sink(name = "has_hello_world", columns = [{"url":"string"}])

data/lib/zillabyte/cli/templates/{python → apps/python}/zillabyte.conf.yaml

@@ -1,4 +1,4 @@
 language: python
-script: simple_function.py
+script: app.py
 ignore_files:
   - ./vEnv

data/lib/zillabyte/cli/templates/{ruby → apps/ruby}/app.rb

@@ -3,7 +3,7 @@
 
 require 'zillabyte' 
 
-Zillabyte.app("hello_world_app")
+Zillabyte.app("<%= name %>")
   .source("select * from web_pages")
   .each{ |page|
     if page['html'].include? "hello world"

data/lib/zillabyte/cli/templates/python/{simple_function.py → #simple_function.py#}

@@ -7,18 +7,15 @@ def prep(controller):
 # web page.  This algorithm is run in parallel on possibly hundreds
 # of machines.
 def execute(controller, tup):
-  if("hello world" in tup["html"]):
-    controller.emit("has_hello_world",{"url":tup["url"]})
+  if("hello world" in tup.values["html"]):
+    controller.emit("has_hello_world",{"url":tup.values["url"]})
   return
 
 zillabyte.simple_function(\
-  # This specifies the function's name and is mandatory.
-  name = "simple_function", \
-
   # This directive instructs zillabyte to give your function every
   # web page in our known universe.  Your function will have access
   # to two fields: URL and HTML
-  matches = "select * from web_pages", \
+  matches = "select * from web_pa", \
 
   # This directive tells Zillabyte what kind of data your function
   # produces.  In this case, we're saying we will emit a tuple that

data/lib/zillabyte/common/session.rb

@@ -6,7 +6,9 @@ module Zillabyte::Common
     end
     
     def display(*args)
-      puts *args
+      args.each do |arg|
+        STDOUT.syswrite(arg + "\n")
+      end
     end
     
     def error(*args)

data/lib/zillabyte/helpers.rb

@@ -7,7 +7,7 @@ module Zillabyte
     
     def display(msg="", new_line=true)
       if new_line
-        puts(msg)
+       puts(msg)
       else
         print(msg)
         $stdout.flush
@@ -50,6 +50,69 @@ module Zillabyte
         # fails on windows
       end
     end
+    
+    def has_git?
+      %x{ git --version }
+      $?.success?
+    end
+    
+    def git(args)
+      return "" unless has_git?
+      flattened_args = [args].flatten.compact.join(" ")
+      %x{ git #{flattened_args} 2>&1 }.strip
+    end
+    
+    def create_git_remote(remote, url)
+      return if git('remote').split("\n").include?(remote)
+      return unless File.exists?(".git")
+      git "remote add #{remote} #{url}"
+      display "Git remote #{remote} added"
+    end
+    
+    
+    
+    def app
+      @app ||= if options[:confirm].is_a?(String)
+        if options[:app] && (options[:app] != options[:confirm])
+          error("Mismatch between --app and --confirm")
+        end
+        options[:confirm]
+      elsif options[:app].is_a?(String)
+        options[:app]
+      elsif ENV.has_key?('ZILLABYTE_APP')
+        ENV['ZILLABYTE_APP']
+      elsif app_from_dir = extract_app_in_dir(Dir.pwd)
+        app_from_dir
+      else
+        # raise instead of using error command to enable rescuing when app is optional
+        raise Zillabyte::Command::CommandFailed.new("No app specified.\nRun this command from an app folder or specify which app to use with --app APP.") unless options[:ignore_no_app]
+      end
+    end
+    
+    
+
+    def extract_app_in_dir(dir)
+      return unless remotes = git_remotes(dir)
+
+      if remote = options[:remote]
+        remotes[remote]
+      elsif remote = extract_app_from_git_config
+        remotes[remote]
+      else
+        apps = remotes.values.uniq
+        if apps.size == 1
+          apps.first
+        else
+          raise(Heroku::Command::CommandFailed, "Multiple apps in folder and no app specified.\nSpecify app with --app APP.") unless options[:ignore_no_app]
+        end
+      end
+    end
+
+    def extract_app_from_git_config
+      remote = git("config zillabyte.remote")
+      remote == "" ? nil : remote
+    end
+    
 
   end
 end

data/lib/zillabyte/runner/app_runner.rb

@@ -0,0 +1,226 @@
+require "zillabyte/cli/base"
+require "zillabyte/runner"
+require "zillabyte/runner/multilang_operation"
+require 'thread'
+
+
+# HIDDEN:
+class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base 
+  include Zillabyte::Helpers
+
+  def run (dir = Dir.pwd, session = nil, options = {})
+
+    if session.nil?
+      return
+    end
+
+    @session = session
+    @colors = {}
+    output = options[:output]
+    otype = options[:output_type] 
+    interactive = options[:interactive]
+    cycles = (options[:cycles] || "1").to_i
+
+    # Get app metadata 
+    meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, @session, {:test => true})
+    if meta.nil? || meta["nodes"].nil?
+      error "this is not a valid zillabyte app directory"
+      exit
+    end 
+
+    # Check that multilang version is atleast 0.1.0
+    version = meta["multilang_version"] || "0.0.0"
+    version_arr = version.split('.').map {|v| v.to_i}
+    if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
+      display "The version of zillabyte used in your application is outdated."
+      display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
+      return
+    end
+
+    # Show the user what we know about their app... 
+    display "inferring your app details..."
+    describe_app(meta)
+
+    # Setup streams
+    @nodes = meta["nodes"]
+
+    # Index stream consummers and emitters by stream name
+    @arcs = meta["arcs"]
+
+    # Organize component pipes
+    @operations = {}
+    @operation_pipes = {}
+
+    # On each cycle, setup and tear down a test harness
+    (1..cycles).each do |cycle|
+
+      display "starting cycle #{cycle}" unless  cycles == 1
+      
+      begin
+
+        # Setup component pipes
+        @nodes.each do |n|
+
+          name = n["name"]
+          type = n["type"]
+          emits = n["emits"]
+          if n["type"] ==  "source"
+            options[:end_cycle_policy] = n["end_cycle_policy"]
+          end
+          
+          # Create two new pipes in the parent.
+          rd_child, wr_parent = IO.pipe()
+          rd_parent, wr_child = IO.pipe()
+
+          @operation_pipes[name] = {
+            :rd_child => rd_child,      
+            :wr_child => wr_child,
+            :rd_parent => rd_parent,        
+            :wr_parent => wr_parent
+          }
+        end
+
+        # Maps origin => {stream => [destinations]}
+        @arc_map = {}
+        @arcs.each do |a|
+          origin = a["origin"]
+          name = a["name"]
+          dest = a["dest"]
+          @arc_map[origin] ||= {}
+          @arc_map[origin][name] ||= []
+          @arc_map[origin][name] << a["dest"]
+        end
+
+        # Spawn component threads
+        @nodes.each do |n|
+
+          name = n["name"]
+          type = n["type"]
+          emits = n["emits"]
+
+          pipes = @operation_pipes[name] 
+          rd_child = pipes[:rd_child]
+          wr_child = pipes[:wr_child]
+          rd_parent = pipes[:rd_parent]        
+          wr_parent = pipes[:wr_parent]
+
+          # Fork.
+          pid = fork()
+          if pid # In parent
+            # Close the reading end of the child so we can write to the child.
+            rd_child.close()
+            # Close the writing end of the child so we can read from the child.
+            wr_child.close()
+
+          else # in child
+            # Close the writing end of the parent so we can read from the parent.
+            wr_parent.close()
+            # Close the reading end of the parent so we can write to the parent.
+            rd_parent.close()
+            begin
+
+
+              # Setup reading and writing pipes for communicating with consumee component
+              in_pipe = {:rd_child => @operation_pipes[name][:rd_child], :wr_child => @operation_pipes[name][:wr_child]}
+
+              # Index consumer pipes by stream name, consumer_name
+              out_pipes = {}
+
+
+              if type != "sink"
+                @arc_map[name].each_pair do |stream, destinations|
+                  out_pipes[stream] ||= {}
+                  destinations.each do |dest|
+                    out_pipes[stream][dest] = {:wr_parent => @operation_pipes[dest][:wr_parent], :rd_parent => @operation_pipes[dest][:rd_parent] }
+                  end
+                end
+              end
+
+              # Run the child process 
+              Zillabyte::Runner::MultilangOperation.run(n, dir, in_pipe, out_pipes, self, meta, options)
+    
+            ensure
+              # Close the reading end of the child
+              rd_child.close()
+              # Close the writing end of the child
+              wr_child.close()
+              exit!(-1)
+            end
+
+          end #end child
+        end 
+
+        if interactive
+          display ""
+          while true
+            display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
+            msg = ask
+
+            begin
+              JSON.parse(msg)
+            rescue JSON::ParserError
+              display "Received invalid JSON object"
+              next
+            end
+            # Send tuple to source
+           @operation_pipes["source_1"][:wr_parent].puts msg
+          end
+        end 
+
+      ensure
+        Process.waitall()
+        @operation_pipes.each do |name, pipes|
+          #Close the  writing end of the parent
+          pipes[:wr_parent].close()
+          # Close the reading end of the parent
+          pipes[:rd_parent].close()
+        end
+      end
+
+    end 
+  end
+
+  def session
+    @session
+  end
+
+
+  def cdisplay(name, message)
+
+      color = @colors[name] || :default
+      if message == ""
+        display ""
+      else
+        display "#{name} - #{message}".colorize(color)
+      end
+  end
+
+
+  def query_agnostic(query)
+    @session.api.query.agnostic(query)
+  end
+
+  def display(message, newline = true)
+    @session.display(message, newline)
+  end
+
+
+  def describe_app(meta)
+    colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
+    rjust = 20
+    display "#{'app name'.rjust(rjust)}: #{meta['name']}"
+    display "#{'app language'.rjust(rjust)}: #{meta['language']}"
+    meta['nodes'].each_with_index do |node, index|
+      @colors[node['name']] ||= colors[index % colors.length]
+      color = @colors[node['name']]
+      display (("="*rjust + "  operation ##{index}").colorize(color))
+      display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
+      display "#{"type".rjust(rjust)}: #{node['type'].to_s.colorize(color)}"
+      display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
+      display "#{"consumes".rjust(rjust)}: #{node['consumes'].to_s.colorize(color)}" if node['consumes']
+      display "#{"emits".rjust(rjust)}: #{JSON.pretty_generate(node['emits']).indent(rjust+2).lstrip.colorize(color)}" if node['emits']
+    end
+  end
+
+
+end