zillabyte-cli 0.1.0 → 0.1.1

data/lib/zillabyte/cli/flows.rb

@@ -15,6 +15,9 @@ require "zillabyte/runner/app_runner"
 # HIDDEN: superclass for components and apps
 #
 class Zillabyte::Command::Flows < Zillabyte::Command::Base
+  
+  MAX_POLL_SECONDS = 60 * 15
+  POLL_SLEEP = 2.0
 
   # flows:status [DIR]
   # 
@@ -95,7 +98,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
       if type == "json"
         display "{}"
       else
-        display "flow ##{res['id']} pulled to #{dir}"
+        display "Flow ##{res['id']} pulled to #{dir}"
       end
     end
     
@@ -333,12 +336,12 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   # --output_type OUTPUT_TYPE # specify an output type i.e. json
   #
   def delete
-    flow_id = options[:id] || shift_argument
+    id = options[:id] || shift_argument
 
-    if flow_id.nil?
-      flow_id = read_name_from_conf(options)
+    if id.nil?
+      id = read_name_from_conf(options)
       options[:is_name] = true
-    elsif !(flow_id =~ /^\d*$/)
+    elsif !(id =~ /^\d*$/)
       options[:is_name] = true
     end
     forced = options[:force]
@@ -362,12 +365,48 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     confirmed = forced || confirm == "yes"
 
     if confirmed
-      response = api.flows.delete(flow_id, options)
-      if type == "json"
-        display "{}"
+      display "Destroying flow ##{id}...please wait..." if type.nil?
+      response = api.flows.create_delete(id, options)
+
+      if response["job_id"]
+        options[:job_id] = response["job_id"]
+        flow_id = response["flow_id"]
+        options.delete :is_name
+
+        start = Time.now.utc
+        display "Destroy request sent. Please wait..." if type.nil?
+
+        while(Time.now.utc < start + MAX_POLL_SECONDS) do
+
+        # Poll
+          res = self.api.flows.poll_delete(flow_id, options)
+
+          case res['status']
+          when 'completed'
+            if res['return']
+              if type == "json"
+                display "{}"
+              else
+                display "Flow ##{id} destroyed"
+              end
+            else
+              throw "something is wrong: #{res}"
+            end
+            # success! continue below
+            break
+          when 'running'
+            sleep(POLL_SLEEP)
+          else
+            throw "unknown status: #{res}"
+          end
+
+        end
+      elsif response["error"]
+        error(response["error"], type)
       else
-        display response["body"]
+        error("remote server error (f413)", type)
       end
+
     end
 
   end
@@ -428,7 +467,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
   # --input INPUT_FILE            # writes sink output to a file
   # --output OUTPUT_FILE            # writes sink output to a file
   # --cycles CYCLES                 # number of cycles to emit (default 1)
-  # --directory DIR                 # app directory
+  # --directory DIR                 # flow directory
   # -i, --interactive   # allow user to control input and read output
   #
   def test
@@ -440,11 +479,23 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     end
     options[:directory] = dir
     
+    # Get the flow_type
+    meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, session, {:test => true})
+    if meta.nil? || meta["nodes"].nil?
+      error "this is not a valid zillabyte app directory"
+      exit
+    end
 
-    # Get the class
-    flow_class = read_class_from_conf(options)
+    # Check that multilang version is atleast 0.1.0
+    version = meta["multilang_version"] || "0.0.0"
+    version_arr = version.split('.').map {|v| v.to_i}
+    if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
+      display "The version of zillabyte used in your application is outdated."
+      display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
+      return
+    end
 
-    case flow_class
+    case meta["flow_type"]
     when "component"
       runner = Zillabyte::Runner::ComponentRunner.new()
     when "app"
@@ -455,12 +506,76 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     end
 
     # Start Runner
-    runner.run(dir, self, options)
+    runner.run(meta, dir, self, options)
 
   end
   alias_command "test", "flows:test"
 
 
+  # flows:kill ID
+  #
+  # kills the given flow
+  # 
+  # --config CONFIG_FILE # use the given config file
+  # --output_type OUTPUT_TYPE # specify an output type i.e. json
+  #
+  def kill
+    id = options[:id] || shift_argument
+    type = options[:output_type]
+
+    if id.nil?
+      id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    display "Killing flow ##{id}...please wait..." if type.nil?
+    response = api.flows.create_kill(id, options)
+
+    if response["job_id"]
+      options[:job_id] = response["job_id"]
+      flow_id = response["flow_id"]
+      options.delete :is_name
+
+      start = Time.now.utc
+      display "Kill request sent. Please wait..." if type.nil?
+
+      while(Time.now.utc < start + MAX_POLL_SECONDS) do
+
+        # Poll
+        res = self.api.flows.poll_kill(flow_id, options)
+
+        case res['status']
+        when 'completed'
+          if res['return']
+            if type == "json"
+              display "{}"
+            else
+              display "Flow ##{id} killed"
+            end
+          else
+            throw "something is wrong: #{res}"
+          end
+          # success! continue below
+          break
+        when 'running'
+          sleep(POLL_SLEEP)
+       #   display ".", false
+        else
+          throw "unknown status: #{res}"
+        end
+
+      end
+    elsif response["error"]
+      error(response["error"], type)
+    else
+      error("remote server error (f569)", type)
+    end
+
+  end
+
+
   private 
   
   #
@@ -506,13 +621,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     hash["name"]
   end
 
-  def read_class_from_conf(options = {})
-    type = options[:output_type]
-    hash = Zillabyte::API::Apps.get_rich_meta_info_from_script Dir.pwd, options
-    error("No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow.",type) if hash["error"]
-    hash["class"]
-  end
-
   def fetch_logs(hash, operation=nil, exit_on=nil)
     def color_for(operation_name)
       @color_map[operation_name] ||= @all_colors[ @color_map.size % @all_colors.size ]

data/lib/zillabyte/cli/git.rb

@@ -22,17 +22,14 @@ class Zillabyte::Command::Git < Zillabyte::Command::Base
   # !    Git remote zillabyte already exists
   #
   def remote
+    
     git_options = args.join(" ")
     remote = options[:remote] || 'zillabyte'
-    app_name = options[:name] || File.basename(Dir.pwd)
-
-    if git('remote').split("\n").include?(remote)
-      # already exists.. remove it. 
-      git("remote remove #{remote}")
-    end
-
-    git_url = "git@#{Zillabyte::Auth.git_host}:#{Zillabyte::Auth.user}/#{app_name}"
-    create_git_remote(remote, git_url)
+    name = options[:name] || get_flow_name()
+    error "could not infer app name" if name.nil? 
+    
+    add_git_remote(name, remote)
+    
   end
 
 end

data/lib/zillabyte/cli/templates/components/js/simple_function.js

@@ -0,0 +1,38 @@
+var zillabyte = require('zillabyte');
+
+function prep(controller) {
+}
+
+/** 
+ * This is the heart of your algorithm.  It's processed on every
+ * web page.  This algorithm is run in parallel on possibly hundreds
+ * of machines.
+ */
+function exec(controller, tuple) {
+  if(tuple["html"].indexOf("hello world") !== -1) {
+    controller.emit("has_hello_world",{"url":tuple["url"]});
+  }
+}
+
+zillabyte.simple_function({
+/**
+ * This specifies the function's name and is mandatory.
+ */
+  name: "simple_function",
+
+/** 
+ * This directive instructs zillabyte to give your function every
+ * web page in our known universe.  Your function will have access
+ * to two fields: URL and HTML
+ */
+  matches: "select * from web_pages",
+
+/**
+ * This directive tells Zillabyte what kind of data your function
+ * produces.  In this case, we're saying we will emit a tuple that
+ * is one-column wide and contains the field 'URL' 
+ */
+  emits: [["has_hello_world", [{"url":"string"}]]],
+  prepare: prep,
+  execute: exec
+});

data/lib/zillabyte/cli/templates/components/js/zillabyte.conf.yaml

@@ -0,0 +1,2 @@
+language: js
+script: simple_function.js

data/lib/zillabyte/cli/templates/components/python/component.py

@@ -0,0 +1,17 @@
+import zillabyte
+
+# This function takes a tuple and emits its url field
+def execute(controller, tuple):
+    controller.emit("urls",{"url":tuple["url"]})
+
+component = zillabyte.component(name = "hello_world")
+
+
+# Declare the schema for inputs to the component
+input_stream = component.inputs(name = "url_stream", fields = [{"url" : "string", "html" : "string"}])
+
+# Extract urls from the input_stream
+url_stream = input_stream.each(execute = execute)
+
+# Declare the output schema of the component
+url_stream.outputs(name = "urls", fields = [{"url":"string"}])

data/lib/zillabyte/cli/templates/components/python/requirements.txt

@@ -0,0 +1,7 @@
+# Indicate any required libraries here
+# Default libraries installed:
+# - numpy 1.8.0
+# - scipy 0.13.0
+# For all other libraries, enter one per line in the following format: LIBRARY==VERSION, e.g. numpy==1.8.0
+# MAKE SURE LIBRARY DEPENDENCIES ARE ALSO INCLUDED, FOR EXAMPLE IF YOUR LIBRARY NEEDS MATPLOTLIB FUNCTIONS, THEN MATPLOTLIB
+# MUST ALSO BE LISTED BELOW EVEN IF YOU DO NOT CALL ANY OF ITS FUNCTIONS EXPLICITLY.

data/lib/zillabyte/cli/templates/components/python/zillabyte.conf.yaml

@@ -0,0 +1,4 @@
+language: python
+script: component.py
+ignore_files:
+  - ./vEnv

data/lib/zillabyte/cli/templates/components/ruby/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem "zillabyte"

data/lib/zillabyte/cli/templates/components/ruby/component.rb.erb

@@ -0,0 +1,20 @@
+require 'zillabyte'
+
+comp = Zillabyte.component("<%= name %>")
+
+# Declare the schema for inputs to the component
+url_stream = comp.inputs do
+  name "urls"
+  field "url", :string
+end
+
+# This component strips HTTP and HTTPS prefixes
+suffix_streams = url_stream.each do |tuple|
+    emit :urls => tuple["url"].gsub(/^http(s)?:\/\//,"")
+end
+
+# Declare the output schema for the component
+suffix_streams.outputs do
+  name "suffix_urls"
+  field "urls", :string
+end

data/lib/zillabyte/cli/templates/components/ruby/zillabyte.conf.yaml

@@ -0,0 +1,5 @@
+language: ruby
+script: component.rb
+flow_type: component
+
+

data/lib/zillabyte/helpers.rb

@@ -66,7 +66,7 @@ module Zillabyte
       return if git('remote').split("\n").include?(remote)
       return unless File.exists?(".git")
       git "remote add #{remote} #{url}"
-      display "Git remote #{remote} added"
+      display "git remote #{remote} added"
     end
     
     
@@ -113,6 +113,46 @@ module Zillabyte
       remote == "" ? nil : remote
     end
     
+    
+    def add_git_remote(name, remote = "zillabyte")
+      
+      remotes = {}
+      remotes[remote] = Zillabyte::Auth.git_host
+      
+      if ENV["ZILLABYTE_EXTRA_GIT_REMOTES"]
+        # This is for dev purposes -- auto init various git env endpoints
+        JSON.parse(ENV["ZILLABYTE_EXTRA_GIT_REMOTES"]).each_pair do |remote, host|
+          remotes[remote] = host
+        end
+      end
+      
+      # Maybe initialize a git directory at the app root... 
+      if File.exists?(File.join(Dir.pwd, ".git")) == false && File.exists?(File.join(Dir.pwd, "zillabyte.conf.yaml"))
+        git("init .")
+      end
+      
+      # Install each git remote
+      remotes.each_pair do |remote, host|
+      
+        if git('remote').split("\n").include?(remote)
+          # already exists.. remove it. 
+          git("remote remove #{remote}")
+        end
+
+        git_url = "git@#{host}:#{Zillabyte::Auth.user}/#{name}"
+        create_git_remote(remote, git_url)
+        
+      end
+      
+    end
+    
+    
+    def get_flow_name(dir = Dir.pwd)
+      meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir)
+      return meta["name"] if meta
+      return nil
+    end
+    
 
   end
 end

data/lib/zillabyte/runner/app_runner.rb

@@ -8,35 +8,20 @@ require 'thread'
 class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base 
   include Zillabyte::Helpers
 
-  def run (dir = Dir.pwd, session = nil, options = {})
+  def run (meta, dir = Dir.pwd, session = nil, options = {})
 
-    if session.nil?
+    if meta.nil? or session.nil?
       return
     end
 
     @session = session
+
     @colors = {}
     output = options[:output]
     otype = options[:output_type] 
     interactive = options[:interactive]
     cycles = (options[:cycles] || "1").to_i
 
-    # Get app metadata 
-    meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, @session, {:test => true})
-    if meta.nil? || meta["nodes"].nil?
-      error "this is not a valid zillabyte app directory"
-      exit
-    end 
-
-    # Check that multilang version is atleast 0.1.0
-    version = meta["multilang_version"] || "0.0.0"
-    version_arr = version.split('.').map {|v| v.to_i}
-    if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
-      display "The version of zillabyte used in your application is outdated."
-      display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
-      return
-    end
-
     # Show the user what we know about their app... 
     display "inferring your app details..."
     describe_app(meta)
@@ -151,7 +136,11 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
         end 
 
         if interactive
+          display "To view results:  Enter 'end'    "
+          display "To exit the test: Enter 'Ctrl-C' "
           display ""
+
+  
           while true
             display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
             msg = ask
@@ -223,4 +212,4 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
   end
 
 
-end
+end