zillabyte-cli 0.0.20 → 0.0.21

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjAzZDgxMjBiY2M0NDU3ODEyNjEyMmJhZjA4MWQ1MjU2NjE0Zjk2YQ==
+    M2YyYjcyMWJlMGExNmUzOGI3NjE1N2UzMThiZjU4ZjZkYzc4ZDgyYg==
   data.tar.gz: !binary |-
-    ZjM2ZjY2NTRiZGU3ZmMwMzI1ZDQ5NjNkOTZmMTEzMzAxMzJmZmZkYw==
+    MWIyNjZmMjVlYWJhYzM2OTdhOGU2NzlhY2U2NTg4YTRkMjJjMjJkMQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZmJhYzkwZWZiMGNkMGJlNzY1N2NlNzdkMjA0ZTU4NzhmNGM3YTBhZjFkZWMz
-    Nzc0ZmJlOGQ3NjQ3NTM3YzE5NDIwNTYxZGI5ZDI4NWY2NjZiOTRkMTY0ZmYz
-    NGI2YjljM2RjNzc0ZGQ3ZTc5N2Y2YmMxYTZiNTkwNjQ5MWExMWY=
+    MWFmY2FiMzUzNjM0MmY3OGE2NzA3YjEwYTY4MzQ2NWMwZTdmNGE4NDA5ZDgy
+    NjdmNDI4YjllMjQyMGFiNDY1ZGQzYzY1OTliY2Y1YTRkMGJlMzQyM2VkZDA4
+    NzY5ZWFmMThkNjFjNDhmYzZmODM3Yzk3ZjQ0OGRkMDZkNjE1Njg=
   data.tar.gz: !binary |-
-    MjIxYjU5Mzg3MTYzYTA5OGNkZTNlOGI5ZTFhMjVjNWY2YjE2ZDg1MmQ0YjRl
-    NWFmZTNhNmJhODY5ODgzOTM4YzhjYTI1ZmMyMTRjMjM5ZWJjNTk5YTcyMDc1
-    Yzk0ZTEzZTNkMzY0M2JhMTI2OTBhNDQ2YzY0Yjg0ZjgyNzEyNWI=
+    YjA3ZGIxNDU4NjVkNDViZGNmN2M5MTg5YmNkYTQ0YzQzODU3NDJkMTNmMzZi
+    MDNjZTI3YmViMDhiM2JlNzA4NGFiNDVhZjk3MTg2ZmY0NmI1NDYzMmNlNDNi
+    YmIzYWY2ZjcyNGZjYTI0NmVkNzdhM2ZhMDEyNTY1OGYyNWI2YjA=

data/lib/zillabyte/api/apps.rb

@@ -203,6 +203,7 @@ class Zillabyte::API::Apps < Zillabyte::API::Base
       :body     => options.to_json
     )
 
+
     if(res.body['uri'])
       uri = URI(res.body['uri'])
       try_again = 1
@@ -246,9 +247,11 @@ class Zillabyte::API::Apps < Zillabyte::API::Base
     full_script = File.join(dir, hash['script'])
     command = nil
     
+
     info_file = "#{dir}/#{SecureRandom.uuid}"
     arg = "--info --file #{info_file}"
-    
+
+
     case hash["language"]
     when "ruby"
       command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
@@ -267,7 +270,7 @@ class Zillabyte::API::Apps < Zillabyte::API::Base
       return nil
     end
 
-    results = Zillabyte::Command::Apps.get_info(command, info_file, options)
+    results = Zillabyte::Command::Apps.get_info(command, info_file, dir, options)
 
     begin
       meta = JSON.parse(results.split("\n").first.strip)   # Throws error if invalid json

data/lib/zillabyte/api.rb

@@ -87,6 +87,9 @@ class Zillabyte::API
         # Executed on each chunk excon bubbles up
         params[:response_block] = lambda do |chunk, remaining_bytes, total_bytes|
 
+          # Debug helper
+          puts chunk if ENV["ZB_DEBUG"]
+
           # assume single-line json, split by '\n'
           last_string += chunk
           ary = last_string.split("\n", -1)

data/lib/zillabyte/cli/apps.rb

@@ -23,6 +23,8 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   def index
     self.list
   end
+  
+
 
   # apps
   #
@@ -37,15 +39,143 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
         headings = row.keys 
         headings.delete("rel_dir")
       end
+      
       v = row.values_at *headings
       v
     end
 
     display "apps:\n" if type.nil?
     display TableOutputBuilder.build_table(headings, rows, type)
-    display "Total number of apps: "+rows.length.to_s if type.nil?
+    display "Total number of apps: " + rows.length.to_s if type.nil?
   end
 
+  # apps:details ID
+  #
+  # list details for an app, including percentage complete
+  # if the source is an existing relation
+  def details
+    
+    app_id = options[:id] || shift_argument
+    type = options[:type]
+    
+    if app_id.nil?
+      app_id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(app_id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+        
+    res = api.request(
+      :expects => 200,
+      :method => :get,
+      :path => "/flows/#{app_id}/details",
+      :body => options.to_json
+    )
+    res.body
+    
+    # a nested hash so can't map directly
+    # instead i'll map separately and then merge the arrays
+    heading1 = ["type", "name"]
+    
+    # a level deeper
+    heading2 = ["consumed", "emitted", "errors"]
+    
+    # for the table output
+    headings = heading1 + heading2 
+    
+    # get the values for the headings
+    type_name = res.body.map do |instance|
+  
+      heading1.map do |heading|
+        instance[heading]
+      end  
+  
+    end
+    
+    # get the values for the headings
+    stats = res.body.map do |instance|
+  
+      heading2.map do |heading|
+        instance["stats"][heading]
+      end  
+  
+    end
+    
+    # combine the arrays
+    rows = type_name.zip(stats).map{|x,y| x.concat y}
+    
+    # check if it's from a relation.
+    # if the source is a relation, there's a sharder. 
+    # remove source types, and hardcode sharder-named instances as source
+    
+    from_relation = false
+        
+    rows.each do | row |
+      if row[1].scan("sharder").length != 0
+        from_relation = true
+        break
+      end
+    end
+
+    if from_relation == true
+      
+      rows = rows.each do |row|
+        if row[0].scan("source").length != 0 
+          rows.delete(row)
+        end
+        
+      end
+      
+    
+      rows = rows.each do |row|
+        if row[1].scan("sharder").length != 0
+          row[0] = "source"
+        end
+      end
+        
+    end    
+    
+    # rows to be output to the CLI
+    rows = rows.group_by{ |type, name, consumed, emitted, errors| name }.map{ |hashkey, array| [array[0][0], hashkey, array.inject(0){ |sum, i | sum + i[2].to_i }, array.inject(0){ |sum, i | sum + i[3].to_i },array.inject(0){ |sum, i | sum + i[4].to_i }]    }
+    # example sums: [["sink", "has_facebook_likebox_and_vwo", 73, 0, 0], ["source", "sharder.1", 16, 10367872, 1], ["each", "each_3", 10367872, 73, 0]]
+    
+    # calculate percentage for the user to see
+    
+    attempts = 0
+    total = 0
+
+    if from_relation == true
+
+      rows.each do |instance|
+        if instance[0] == "source"
+          total = instance[3]
+        end
+      end
+
+
+      rows.each do |instance|
+        if instance[0] == "each"
+          attempts = instance[2] 
+        end
+      end
+    end
+    
+    if attempts > 0 && total > 0 
+      completion = attempts/total*100
+    else 
+      completion = 0
+    end
+
+    # output on the CLI
+    display "details:"
+    display TableOutputBuilder.build_table(headings, rows, type)
+    if from_relation == true
+      display "percent complete: #{completion}%\n"
+    end    
+  end
+
+
+
 
   # apps:push [DIR]
   #
@@ -62,7 +192,13 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   def push
     
     since = Time.now.utc.to_s
-    dir = options[:directory] || shift_argument || Dir.pwd
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
     type = options[:type]
 
     res = api.apps.push_directory dir, session, options
@@ -89,8 +225,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
       # exit(0)  if (hash['line'] || '').downcase.include?("app deployed")
       
     end
-
-    display {}.to_json if type == "json"
+    display "{}" if type == "json"
       
   end
   alias_command "push", "apps:push"
@@ -103,8 +238,9 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   #
   # --force # pulls even if the directory exists
   # --type TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the app
   #
-  #Examples:
+  # Examples:
   #
   # $ zillabyte apps:pull . 
   #
@@ -117,10 +253,12 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
     end
 
     dir = options[:directory] || shift_argument
+    error("no directory given", type) if dir.nil?
+    dir = File.expand_path(dir)
+
     type = options[:type]
     
     error("no id given", type) if app_id.nil?
-    error("no directory given", type) if dir.nil?
     
     # Create if not exists..
     if File.exists?(dir)
@@ -137,7 +275,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
       error("error: #{res['error_message']}", type)
     else
       if type == "json"
-        display {}.to_json
+        display "{}"
       else
         display "app ##{res['id']} pulled to #{dir}"
       end
@@ -186,7 +324,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
     if confirmed
       response = api.apps.delete(app_id, options)
       if type == "json"
-        display {}.to_json
+        display "{}"
       else
         display response["body"]
       end
@@ -196,23 +334,31 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
 
   
   # apps:prep [DIR]
-  #
-  # prepares an app for execution
+  # 
+  # performs any necessary initialization for the app
+  # 
   # --directory DIR       # app directory
   # --type TYPE           # specify an output type i.e. json
-  #
+  #  
   def prep
-   
-    type = options[:type] 
-    dir = options[:directory] || shift_argument || Dir.pwd
+
+    type = options[:type]
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
     meta = Zillabyte::CLI::Config.get_config_info(dir)
+
     if meta.nil?
       error("The specified directory (#{dir}) does not appear to contain a valid Zillabyte configuration file.", type)
     end
 
     case meta["language"]
     when "ruby"
-      
+
       # Execute in the bundler context
       full_script = File.join(dir, meta["script"])
       cmd = "cd \"#{meta['home_dir']}\"; unset BUNDLE_GEMFILE; unset RUBYOPT; bundle install"
@@ -247,6 +393,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   # [LANG] defaults to ruby, and [DIR] to the current directory
   #
   # --type TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the app
   #
   #Examples:
   #
@@ -255,7 +402,12 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   def init
     
     lang = options[:lang] || shift_argument || "ruby"
-    dir = options[:dir] || shift_argument || Dir.pwd
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
     type = options[:type]
 
     languages = ["ruby","python", "js"]
@@ -311,7 +463,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
 
   # apps:errors ID
   #
-  # Show recent errors generated by the app
+  # show recent errors generated by the app
   # --type TYPE           # specify an output type i.e. json
   #
   def errors
@@ -432,7 +584,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
         when 'completed'
           if res['return']
             if type == "json"
-              return {}.to_json
+              return "{}"
             else
               display res['return']
             end
@@ -521,8 +673,15 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
     end
 
 
+
     # INIT
-    dir = options[:directory] || shift_argument || Dir.pwd
+     dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
 
     meta = Zillabyte::API::Apps.get_rich_meta_info_from_script(dir, self, {:test => true})
     if meta.nil?
@@ -598,7 +757,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
           next 
           
         else
-          
+
           # A regular source.. 
           stream_messages[default_stream] ||= []
           stream_messages[default_stream] << "{\"command\": \"begin_cycle\"}\n"
@@ -705,6 +864,8 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
             handshake stdin, stdout, node, color
             write_queue = []
             read_queue = []
+            mutex = Mutex.new 
+            signal = ConditionVariable.new
           
             if consumes.nil?
               # Assume default stream (this should only happen for the source)
@@ -716,7 +877,6 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
             stream_messages.delete(stream_name)
           
             # Start writing the messages... 
-            stuff_to_read = false
             writing_thread = Thread.start do 
 
               while(true)
@@ -733,8 +893,10 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
                 end
 
                 # Make sure we're not reading anything... 
-                while(stuff_to_read)   # TODO: semaphores
-                  sleep 0.5 # spin wait
+                while(write_queue.size == 0)
+                  mutex.synchronize do
+                    signal.wait(mutex)
+                  end
                 end
                 
                 # Get next mesage
@@ -749,13 +911,12 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
                   last_call_next_tuple = true
                   op_display.call "getting next set of tuples in the batch"
                 else
-                  puts write_json
+                  # puts write_json
                 end
                 
                 # Actually send it to the process 
                 begin
                   write_message stdin, write_msg
-                  stuff_to_read = true
                   sleep 0.1
                 rescue Exception => e
                   puts "Error running #{cmd}: #{e}"
@@ -771,25 +932,30 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
                 # If the end cycle command is received, we either trigger the next cycle if the number of emitted
                 # cycles is less than what the user requested, or we break
                 if type == "source" and end_cycle_received
-                  write_queue << "{\"command\": \"begin_cycle\"}\n"
+                  
+                  mutex.synchronize do
+                    write_queue << "{\"command\": \"begin_cycle\"}\n"
+                    signal.signal()
+                  end
+                  
                   n_batches_emitted += 1
                   end_cycle_received = false
                   last_call_next_tuple = false
-                  stuff_to_read = false
                   break if n_batches_emitted > batches
                   sleep 0.5
                   next
                 end
 
                 # Get next message
-                read_msg = read_message(stdout, color)
+                read_msg = read_message(stdout, color);
+                
                 if read_msg == "done" || read_msg.nil?
-                  stuff_to_read = false
 
                   # For sources, if we receive a "done", check to see if any of the streams emitted by the source has
                   # increased in size since the last call to next_tuple. If so, the cycle isn't over, otherwise, the
                   # current call to next_tuple emitted nothing and if the end_cycle_policy is set to null_emit, this
                   # should end the current cycle.
+                  
                   if type == "source"
                     if last_call_next_tuple and node["end_cycle_policy"] == "null_emit"
                       end_cycle_received = true
@@ -801,7 +967,11 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
                     end 
 
                     # If the policy isn't "null_emit", then just request next_tuple again
-                    write_queue << "{\"command\": \"next\"}\n"
+                    mutex.synchronize do
+                      write_queue << "{\"command\": \"next\"}\n"
+                      signal.signal()
+                    end
+                    
                   end
   
                   # For other operations, if the queue is empty then we're done
@@ -812,7 +982,6 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
                     next
                   end
                 end
-                stuff_to_read = true
                 
                 # Process message
                 obj = JSON.parse(read_msg)
@@ -859,6 +1028,13 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
                 elsif obj['command'] == 'fail'
                   op_display.call "error: #{obj['msg']}", :red
                   exit(1)
+                elsif obj['ping']
+                  
+                  mutex.synchronize do
+                    write_queue << "{\"pong\": \"#{Time.now.utc.to_f}\"}\n"
+                    signal.signal()
+                  end
+                  
                 else
                   error "unknown message: #{read_msg}"
                 end
@@ -924,7 +1100,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
     api.apps.kill(id, options)
 
     if type == "json"
-      display {}.to_json
+      display "{}"
     else
       display "App ##{id} killed"
     end
@@ -938,7 +1114,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   # 
   # --config CONFIG_FILE # use the given config file
   # --type TYPE # specify an output type i.e. json
-  # --directory DIR # directory of app
+  # --directory DIR # Directory of the app
   #
   # HIDDEN:
   def live_run
@@ -947,22 +1123,68 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
     type = options[:type]
 
     thread_id = options[:thread] || shift_argument || ""
-    dir = options[:directory] || shift_argument || Dir.pwd
-    meta = Zillabyte::CLI::Config.get_config_info(dir)
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+
+    meta = Zillabyte::CLI::Config.get_config_info(dir, options)
   
     if meta.nil?
       error("could not find meta information for: #{dir}", type)
     end
    
     if(thread_id == "")
-      exec(command("--execute_live --name #{name.to_s}", type, dir))
+      exec(command("--execute_live --name #{name.to_s}",type, dir))
     else 
-      exec(command("--execute_live --name #{name.to_s} --pipe #{thread_id}", type, dir))
+      exec(command("--execute_live --name #{name.to_s} --pipe #{dir}/#{thread_id}",type, dir))
     end
   end  
   alias_command "live_run", "apps:live_run"
   
   
+  
+  # apps:status [DIR]
+  # 
+  # fetches detailed status of the app
+  # 
+  # --type TYPE # specify an output type i.e. json
+  # --directory DIR # Directory of the app
+  #
+  def status
+
+    id = options[:id] || shift_argument
+    type = options[:type]
+
+    if id.nil?
+      id = read_name_from_conf(options)
+      options[:is_name] = true
+    elsif !(id =~ /^\d*$/)
+      options[:is_name] = true
+    end
+
+    res = api.request(
+      :expects => 200,
+      :method  => :get,
+      :path    => "/flows/#{id}"
+    )
+    res.body
+
+    if type == "json"
+      display res.body.to_json
+    else
+      # TODO
+      display res.body.to_json
+    end
+    
+  end
+
+  
+  
+  
   # apps:info [DIR]
   # 
   # outputs the info for the app in the dir.
@@ -972,12 +1194,20 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   # --directory DIR # Directory of the app
   #
   def info
-    dir = options[:directory] || shift_argument || Dir.pwd
-    type = options[:type]
+
+    dir = options[:directory] || shift_argument
+    if dir.nil?
+      dir = Dir.pwd
+    else
+      dir = File.expand_path(dir)
+    end
+    options[:directory] = dir
+
     info_file = "#{dir}/#{SecureRandom.uuid}"
+    type = options[:type]
 
     cmd = command("--info --file #{info_file}", type, dir)
-    app_info = Zillabyte::Command::Apps.get_info(cmd, info_file)
+    app_info = Zillabyte::Command::Apps.get_info(cmd, info_file, dir, options)
 
     if type == "json"
       puts app_info
@@ -997,11 +1227,14 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
   #
   # --type TYPE # specify an output type i.e. json
   #
-  def self.get_info(cmd, info_file, options = {})
+  def self.get_info(cmd, info_file, dir, options = {})
     type = options[:type]
+
     response = `#{cmd}`
     if($?.exitstatus == 1)
+
       File.delete("#{info_file}") if File.exists?(info_file)
+
       if options[:type].nil?
         exit(1)
       else
@@ -1022,8 +1255,8 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
     File.delete("#{info_file}")
 
     app_info = app_info.to_json
-    if(File.exists?("info_to_java.in"))
-      java_pipe = open("info_to_java.in","w+")
+    if(File.exists?("#{dir}/info_to_java.in"))
+      java_pipe = open("#{dir}/info_to_java.in","w+")
       java_pipe.puts(app_info+"\n")
       java_pipe.flush
       java_pipe.close()
@@ -1035,13 +1268,12 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Base
 
   private 
   
-  #
   #
   # --type TYPE # specify an output type i.e. json
   #
   def command(arg="--execute_live", type = nil, dir = Dir.pwd, ignore_stderr = false)
-
     meta = Zillabyte::CLI::Config.get_config_info(dir, self, options)
+
     #meta = Zillabyte::API::Functions.get_rich_meta_info_from_script(dir, self)
     error("could not extract meta information. missing zillabyte.conf.yml?", type) if meta.nil?
     error(meta["error_message"], type) if meta['status'] == "error"