zillabyte-cli 0.1.37 → 0.1.38

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OGUxZDA3NGNiZWIyOWUzNmE3ZmMxYTUzYmUwMzNjYzExZmQ5MjM2Ng==
+    ODc2YjU3ZTY2MWI5YWUxZjQwNjc3YjljN2FjNGYwN2U1ZjQwNjkwZA==
   data.tar.gz: !binary |-
-    MWVjMThkNzRjZGRhZmM4NGY0YjIyYzg1ODcxZmJiYjVhMWIwZWYyZg==
+    NzhlMDhkODI1MzY4OTVjNzRhNzVlZWVjYWQ2NTM2OWYwNGNlNTQ3Nw==
 SHA512:
   metadata.gz: !binary |-
-    NDU0YjQzYTFmYmM4ZmViOGEzNzkwYzY2ZWQzNjJhOTY5ZmU0YTBkNjg0Y2Vl
-    YWI3Y2U4OTE5MWIwMmI2OWQ4ODRjMDZlYjAzMjcxMjMwNzEzMDU1NmVhZGFk
-    MjJkM2QwOGM1YmE5OTA4MGUxM2UxZDRkMGIwYjNkY2Y4MzkwN2Y=
+    YzM4NWFkNzZmOTYwYjJmMGZmYTdiMTRkYmQ3NjlkYWVhMjFjMmZiYTU0MzUx
+    ZGM1ZjRkZGFhMWIxMWMxZThiOWNhNzUwMmFkMWY5NDhjZTYyMjBkMzgyMzBj
+    NmY4NmMzOWE0NDE5ZTMzZmEwYzViNzhkNjYzYmUyN2YzNzcxYzE=
   data.tar.gz: !binary |-
-    MWE4ZTg4NGI4Yjk3MDNhZTk3NDNkOWNiMjRmNmMzN2Q3NTQzZDUxM2JiYTA2
-    ZWI4ZGI5ZmJlMGVmMWIxYTdiMTI4Mzk2NzViNDFiYjY2ZmE4OTdjZGY1Y2Zi
-    N2Q5ZDFiODdlOTY2Y2RjODQyN2FmMThlOTdiOTA4MTVhMDM3ZTI=
+    ODQ2ZDQ1N2M1ZTU4YjY3NjQyYzhlNmRiYjNkMTliMjMzOWFhYzhmYzA1ZWMx
+    MGFiOGE5ZGU1ODE3ZGQ5ODQwNGM1OWVkMDBkNzQzNTEwZWFmMjM2YTIzNTky
+    NjQwYjMzMTIyNjlkNjRjNDFkZWU4NDQ5YTBlZTVkOWNiNzdhYjU=

data/lib/zillabyte/cli/flows.rb

@@ -319,10 +319,20 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
       :body => options.to_json
     )
 
+    return if res.body.nil? or res.body["state"].nil?
     flow_state = res.body["state"]
+    if type.nil?
+      display "State: #{flow_state}"
+    elsif ["KILLED", "RETIRED"].member? flow_state
+      display res.body.to_json
+      return
+    end
+
+    return if res.body["instances"].nil?
     instances = res.body["instances"].select do |instance|
       !instance.nil?
     end
+    return if instances.empty?
 
     # a nested hash so can't map directly
     # instead i'll map separately and then merge the arrays
@@ -420,7 +430,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
     # output on the CLI
     if type.nil?
       require("zillabyte/cli/helpers/table_output_builder")
-      display "State: #{flow_state}"
       display TableOutputBuilder.build_terminal_table(headings, rows)
       if from_dataset == true
         display "percent complete: #{completion}%\n"

data/lib/zillabyte/runner/app_runner.rb

@@ -4,6 +4,8 @@ require "zillabyte/runner/multilang_operation"
 class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base 
   include Zillabyte::Helpers
 
+  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
+
   def run (meta, dir = Dir.pwd, session = nil, options = {})
 
     if meta.nil? or session.nil?
@@ -197,6 +199,7 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
               display e.message
               display e.backtrace
             ensure
+
               # Close the reading end of the child
               pipes["rd_child_1"].close()
               # Close the writing end of the child
@@ -214,7 +217,6 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
 
         if interactive
           display "To view results:  Enter 'end'    "
-          display "To exit the test: Enter 'Ctrl-C' "
           display ""
 
   
@@ -222,11 +224,16 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
             display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
             msg = ask
 
-            begin
-              JSON.parse(msg)
-            rescue JSON::ParserError
-              display "Received invalid JSON object"
-              next
+            
+            if msg == "end"
+              msg = END_CYCLE_MESSAGE
+            else
+              begin
+                JSON.parse(msg)
+              rescue JSON::ParserError
+                display "Received invalid JSON object"
+                next
+              end
             end
             # Send tuple to source
            @operation_pipes["source_1"]["wr_parent_1"].puts msg

data/lib/zillabyte/runner/component_operation.rb

@@ -6,6 +6,7 @@ class Zillabyte::Runner::ComponentOperation
 
   NEXT_MESSAGE = "{\"command\": \"next\"}\n"
   DONE_MESSAGE = "{\"command\": \"done\"}\n"
+  KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
   END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
   ENDMARKER = "\nend\n"
 
@@ -49,7 +50,6 @@ class Zillabyte::Runner::ComponentOperation
       cdisplay e.message
       cdisplay e.backtrace
     end
-
   end
 
 
@@ -104,22 +104,28 @@ class Zillabyte::Runner::ComponentOperation
             consumer = consumer_hash[r][:consumer]
 
             # Consumer is ready for next message
-            if msg["command"] && msg["command"] == "next"
-
-              @__emit_queues[stream][consumer][:ready] = true
-              tuple_json = get_consumer_tuple(stream, consumer)
-
-              # If all messages have been sent to consumer, end their cycle
-              if tuple_json.nil?
-                write_stream = get_write_stream(stream, consumer)
-                cdisplay "ending cycle for #{consumer}"
-                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-                send_command_tuple(stream, consumer, DONE_MESSAGE)
+            if msg["command"]
+              case msg["command"]
+              when "next"
+
+                @__emit_queues[stream][consumer][:ready] = true
+                tuple_json = get_consumer_tuple(stream, consumer)
+
+                # If all messages have been sent to consumer, end their cycle
+                if tuple_json.nil?
+                  write_stream = get_write_stream(stream, consumer)
+                  cdisplay "ending cycle for #{consumer}"
+                  send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                  send_command_tuple(stream, consumer, DONE_MESSAGE)
+
+                else
+                  # Emit tuple to consumer
+                  emit_consumer_tuple(stream, consumer, tuple_json)
+                  emitted = true
+                end
 
-              else
-                # Emit tuple to consumer
-                emit_consumer_tuple(stream, consumer, tuple_json)
-                emitted = true
+              when "kill_cycle"
+                send_to_consumers(KILL_CYCLE_MESSAGE)                
               end
             end
           end
@@ -136,28 +142,23 @@ class Zillabyte::Runner::ComponentOperation
       loop do 
 
         msg = stdin.gets
-        if msg == "end\n"
-          @__consumer_pipes.each_pair do |stream, consumers|
-            consumers.each_pair do |consumer, pipe| 
-              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-              send_command_tuple(stream, consumer, DONE_MESSAGE)
-            end
+        if msg == END_CYCLE_MESSAGE
+          send_to_consumers(END_CYCLE_MESSAGE)
+          send_to_consumers(DONE_MESSAGE)
+          return
+        else
+          # Build tuples
+          args = msg.scan(/(?:\w|"[^"]*")+/).map {|s| s.gsub(/[\'\"]/, '')}
+          component_args = []
+
+          while(true) do
+            break if args.empty?
+            tuple = {}
+            fields.each {|f| tuple[f] = args.shift}
+            tuple_json = build_tuple_json(tuple)
+            display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
+            send_to_consumers(tuple_json)
           end
-
-          break
-        end
-
-        # Build tuples
-        args = msg.scan(/(?:\w|"[^"]*")+/).map {|s| s.gsub(/[\'\"]/, '')}
-        component_args = []
-
-        while(true) do
-          break if args.empty?
-          tuple = {}
-          fields.each {|f| tuple[f] = args.shift}
-          tuple_json = build_tuple_json(tuple)
-          display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-          send_to_consumers(tuple_json)
         end
       end
     end
@@ -173,11 +174,12 @@ class Zillabyte::Runner::ComponentOperation
     # Keep track of how many consumers to handle before exiting
     consumers_running = consumer_hash.keys.length
 
+    # Kill the cycle on error
+    cycle_killed = false
+
     # Begin cycle
     end_cycle_received = false
 
-
-    # TODO multiple inputs
     # The input component(singular at the moment)
     read_streams = consumer_hash.keys.concat [@__consumee_pipes["rd_child_1"]]
 
@@ -204,155 +206,172 @@ class Zillabyte::Runner::ComponentOperation
 
     fields = source_nodes[0]["fields"]
 
-    # Receive and handle messages
-    loop do
-
-      # Read from a stream
-      rs = select_read_streams(read_streams)
-      rs.each do |r|
-        # Read a message
-        obj = read_message(r)
-
-        # Handle tuple through RPC
-        if obj['tuple']
-          cdisplay("error: The number of inputs to the component does not match the declared number for stream #{source_nodes[0]["name"]}.") if obj['tuple'].size != fields.size
-          rpc_inputs = []
-          fields.each do |field|
-            rpc_inputs << obj['tuple'][field.keys[0]]
-          end
-
-          display_json = Hash[obj['tuple'].map{|k, v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
-          
-
-          # Send RPC call
-          cdisplay "sending RPC call..."
-          call_options = {
-            :rpc_inputs => [rpc_inputs],
-            :output_format => output_format
-          }
+    begin
 
-          res = caller.rpc(component_id, call_options)
-          cdisplay "received API response : #{res}"
+      # Receive and handle messages
+      loop do
+
+        # Read from a stream
+        rs = select_read_streams(read_streams)
+        rs.each do |r|
+          # Read a message
+          obj = read_message(r)
+
+          # Handle tuple through RPC
+          if obj['tuple']
+            cdisplay("error: The number of inputs to the component does not match the declared number for stream #{source_nodes[0]["name"]}.") if obj['tuple'].size != fields.size
+            rpc_inputs = []
+            fields.each do |field|
+              rpc_inputs << obj['tuple'][field.keys[0]]
+            end
 
-          #TODO handle errors
-          if res['error']
-            cdisplay("error: #{res['error']}")
-            next
-          end
-          run_ids = []
-          res['execute_ids'].each do |q, qid|
-            run_ids << qid
-          end
-          status = res['status']
+            display_json = Hash[obj['tuple'].map{|k, v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
+            
 
+            # Send RPC call
+            cdisplay "sending RPC call..."
+            call_options = {
+              :rpc_inputs => [rpc_inputs],
+              :output_format => output_format
+            }
 
-          # Send result call
-          loop do
+            res = caller.rpc(component_id, call_options)
+            cdisplay "received API response : #{res}"
 
-            # If a call for results is not ready, save the execution ID for the next cycle
-            ids_in_progress = []
-            res = caller.get_rpc_results(component_id, {:execute_ids => run_ids})
+            #TODO handle errors
             if res['error']
               cdisplay("error: #{res['error']}")
               next
             end
+            run_ids = []
+            res['execute_ids'].each do |q, qid|
+              run_ids << qid
+            end
+            status = res['status']
+
+
+            # Send result call
+            loop do
+
+              # If a call for results is not ready, save the execution ID for the next cycle
+              ids_in_progress = []
+              res = caller.get_rpc_results(component_id, {:execute_ids => run_ids})
+              if res['error']
+                cdisplay("error: #{res['error']}")
+                next
+              end
 
-        
-            # check results
-            res['results'].each do |run_id, hash|
-              
-              # We have results
-              if hash['status'] == "complete"
-
-                # Handle Tuple in here
-                data_streams = hash['data']
-
-                # if stream_hash.nil?
-                #   cdisplay("error : no results!")
-                #   next
-                # else
-                #   stream_hash.each_pair
-                # end
-
-                # Handle each resulting tuple
-                data_streams.each_pair do |stream, data_tuples|
-                  data_tuples.each do |data_tuple|
-                    tuple_json = build_tuple_json(data_tuple)
-
-                    #send or enqueue the tuple to all consumers of the stream
-                    @__emit_queues.each_pair do |stream, consumers|
-                      consumers.each_pair do |consumer, emitter|
-                        if emitter[:ready]
-                          emit_consumer_tuple(stream, consumer, tuple_json)
-                        else
-                          @__emit_queues[stream][consumer][:write_queue] << tuple_json
+          
+              # check results
+              res['results'].each do |run_id, hash|
+                
+                # We have results
+                if hash['status'] == "complete"
+
+                  # Handle Tuple in here
+                  data_streams = hash['data']
+
+                  # if stream_hash.nil?
+                  #   cdisplay("error : no results!")
+                  #   next
+                  # else
+                  #   stream_hash.each_pair
+                  # end
+
+                  # Handle each resulting tuple
+                  data_streams.each_pair do |stream, data_tuples|
+                    data_tuples.each do |data_tuple|
+                      tuple_json = build_tuple_json(data_tuple)
+
+                      #send or enqueue the tuple to all consumers of the stream
+                      @__emit_queues.each_pair do |stream, consumers|
+                        consumers.each_pair do |consumer, emitter|
+                          if emitter[:ready]
+                            emit_consumer_tuple(stream, consumer, tuple_json)
+                          else
+                            @__emit_queues[stream][consumer][:write_queue] << tuple_json
+                          end
                         end
                       end
-                    end
 
+                    end
                   end
+
+                else
+                  cdisplay "RPC #{run_id} has not completed... "
+                  ids_in_progress << run_id
                 end
+              end
+              run_ids = ids_in_progress
 
-              else
-                cdisplay "RPC #{run_id} has not completed... "
-                ids_in_progress << run_id
+              # If no more IDs to run, we are done
+              if run_ids.empty?
+                break
               end
-            end
-            run_ids = ids_in_progress
 
-            # If no more IDs to run, we are done
-            if run_ids.empty?
-              break
+              # Dont spam the API
+              sleep(2)
             end
 
-            # Dont spam the API
-            sleep(2)
-          end
+            # Ask for next tuple
+            write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
-          # Ask for next tuple
-          write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
+          # End cycle
+          elsif obj['command'] 
+            case obj["command"]
 
-        # End cycle
-        elsif obj['command'] 
-          case obj["command"]
+            # Consumer is ready for a message
+            when "next"
+              stream = consumer_hash[r][:stream]
+              consumer = consumer_hash[r][:consumer]
 
-          # Consumer is ready for a message
-          when "next"
-            stream = consumer_hash[r][:stream]
-            consumer = consumer_hash[r][:consumer]
+              @__emit_queues[stream][consumer][:ready] = true
+              tuple_json = get_consumer_tuple(stream, consumer)
 
-            @__emit_queues[stream][consumer][:ready] = true
-            tuple_json = get_consumer_tuple(stream, consumer)
+              # End cycle for consumer if it has processed all tuples
+              if tuple_json.nil? && end_cycle_received
+                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                consumers_running -= 1
+                if consumers_running == 0
+                  break
+                end
 
-            # End cycle for consumer if it has processed all tuples
-            if tuple_json.nil? && end_cycle_received
-              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-              consumers_running -= 1
-              if consumers_running == 0
-                break
+                # TODO break if last consumer
+              elsif !tuple_json.nil?
+                # Emit tuple to consumer
+                emit_consumer_tuple(stream, consumer, tuple_json)
+                emitted = true
               end
 
-              # TODO break if last consumer
-            elsif !tuple_json.nil?
-              # Emit tuple to consumer
-              emit_consumer_tuple(stream, consumer, tuple_json)
-              emitted = true
+            when "end_cycle"
+              end_cycle_received = true
+            when "kill_cycle"
+              cycle_killed = true
+              return
             end
-
-          when "end_cycle"
-            end_cycle_received = true
           end
         end
+      
 
+        # Exit after ending consumer cycles
+        if consumers_running == 0
+          break
+        end
       end
 
-      # Exit after ending consumer cycles
-      if consumers_running == 0
-        break
+    rescue => e
+      cdisplay e.message
+      cdisplay e.backtrace
+    ensure
+      # cleanup
+      if cycle_killed
+        send_to_consumers(KILL_CYCLE_MESSAGE, false)
+        send_to_consumees(KILL_CYCLE_MESSAGE, false)
       end
     end
 
+
   end
 
 
@@ -475,6 +494,7 @@ class Zillabyte::Runner::ComponentOperation
     return consumer_hash
   end
 
+
   # Send object to every consumer of the operation, regardless of stream
   def self.send_to_consumers(json_obj)
     @__consumer_pipes.each_pair do |stream, consumers|
@@ -489,11 +509,29 @@ class Zillabyte::Runner::ComponentOperation
           write_message(write_stream, json_obj)
         end
 
-        cdisplay "emitted #{json_obj} to #{consumer}"
+        cdisplay "emitted #{json_obj.chomp} to #{consumer}"
       end
     end
   end
 
+
+  # Send object to every consumer of the operation, regardless of stream
+  def self.send_to_consumees(json_obj)
+    pipes = @__consumee_pipes
+    # Left hand(or singular) input
+    if (pipes.has_key? "wr_parent_1")
+      write_stream = get_write_stream(stream, consumer, 1)
+      write_message(write_stream, json_obj)
+    end
+
+    # Right hand input 
+    if (pipes.has_key? "wr_parent_2")
+      write_stream = get_write_stream(stream, consumer, 2)
+      write_message(write_stream, json_obj)
+    end
+  end
+
+
   # Get the write pipe of the stream consumer
   def self.get_write_stream(stream, consumer, number=1)
      wr_pipe = "wr_parent_" + number.to_s

data/lib/zillabyte/runner/component_runner.rb

@@ -5,6 +5,8 @@ require "zillabyte/runner/component_operation"
 class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base 
   include Zillabyte::Helpers
 
+  END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
+
   def run (meta, dir = Dir.pwd, session = nil, options = {})
 
     if meta.nil? or session.nil?
@@ -226,7 +228,6 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
           
         display ""
         display "To view results:  Enter 'end'    "
-        display "To exit the test: Enter 'Ctrl-C' "
         display ""
 
         while true
@@ -234,8 +235,9 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
           fields = @node_map[source]['fields'].map {|h| h.keys[0].upcase }
           display "Enter an input tuple in the form : #{fields.join(' ')}"
           msg = ask
-
-          if msg != "end"
+          if msg == 'end'
+            msg = END_CYCLE_MESSAGE
+          else
             args = msg.scan(/(?:\w|"[^"]*")+/)
             if (args.length % fields.length != 0)
               display "Error: Argument length must be a multiple of the schema length"

data/lib/zillabyte/runner/multilang_operation.rb

@@ -7,6 +7,7 @@ module Zillabyte; module Runner; class MultilangOperation
   NEXT_MESSAGE = "{\"command\": \"next\"}\n"
   BEGIN_CYCLE_MESSAGE = "{\"command\": \"begin_cycle\"}\n"
   END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
+  KILL_CYCLE_MESSAGE = "{\"command\": \"kill_cycle\"}\n"
   PONG_PREFIX = "{\"pong\": \""
   PONG_SUFFIX = "\"}\n"  
   ENDMARKER = "\nend\n"
@@ -61,8 +62,8 @@ module Zillabyte; module Runner; class MultilangOperation
       end
     rescue => e
       cdisplay e.message
-      cdisplay e.backtrace
     end
+
   end
 
 
@@ -76,17 +77,24 @@ module Zillabyte; module Runner; class MultilangOperation
 
         msg = @__consumee_pipes["rd_child_1"].gets
 
+        if msg == END_CYCLE_MESSAGE            
+          send_to_consumers(END_CYCLE_MESSAGE)
+          send_to_consumers(DONE_MESSAGE)
+          return
+        else
           # Build tuple
           begin
-            tuple = JSON.parse(msg)
+            obj = JSON.parse(msg)
           rescue JSON::ParserError 
             cdisplay "Error: invalid JSON"           
             next
           end
-
-          tuple_json = build_tuple_json(tuple)
+                  
+          tuple_json = build_tuple_json(obj)
           display_json = Hash[JSON.parse(tuple_json)["tuple"].map {|k,v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
           send_to_consumers(tuple_json)   
+          
+        end
       end
 
     # Source from relation
@@ -141,21 +149,28 @@ module Zillabyte; module Runner; class MultilangOperation
           consumer = consumer_hash[r][:consumer]
 
           # Consumer is ready for next message
-          if msg["command"] && msg["command"] == "next"
-
-            @__emit_queues[stream][consumer][:ready] = true
-            tuple_json = get_consumer_tuple(stream, consumer)
-
-            # If all messages have been sent to consumer, end their cycle
-            if tuple_json.nil?
-              write_stream = get_write_stream(stream, consumer)
-              cdisplay "ending cycle for #{consumer}"
-              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-              send_command_tuple(stream, consumer, DONE_MESSAGE)
-            else
-              # Emit tuple to consumer
-              emit_consumer_tuple(stream, consumer, tuple_json)
-              emitted = true
+          if msg["command"] 
+
+            case msg["command"]
+            when "next"
+
+              @__emit_queues[stream][consumer][:ready] = true
+              tuple_json = get_consumer_tuple(stream, consumer)
+
+              # If all messages have been sent to consumer, end their cycle
+              if tuple_json.nil?
+                write_stream = get_write_stream(stream, consumer)
+                cdisplay "ending cycle for #{consumer}"
+                send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+                send_command_tuple(stream, consumer, DONE_MESSAGE)
+              else
+                # Emit tuple to consumer
+                emit_consumer_tuple(stream, consumer, tuple_json)
+                emitted = true
+              end
+            when "kill_cycle"
+              send_to_consumers(KILL_CYCLE_MESSAGE, false)
+              return
             end
           end
         end
@@ -169,13 +184,15 @@ module Zillabyte; module Runner; class MultilangOperation
     # Custom source
     else
 
-
       # Index streams and consumers by their pipes for lookup
       consumer_hash = build_consumer_hash()
 
       # Keep track of how many consumers to handle before exiting
       consumers_running = consumer_hash.keys.length
 
+      # Kill the cycle on error
+      cycle_killed = false
+
       # Setup multilang pipe
       ml_pipe = "#{@__name}_pipe"
       if File.exists?("#{ml_pipe}.in")
@@ -343,7 +360,9 @@ module Zillabyte; module Runner; class MultilangOperation
                       next
                     end
                     end_cycle_received = true
-
+                  when "kill_cycle"
+                    cycle_killed = true
+                    return
                   end
                 
                 # Multilang sent a ping
@@ -358,21 +377,33 @@ module Zillabyte; module Runner; class MultilangOperation
               end
               
             end
-          rescue Errno::EIO
-             cdisplay("Errno:EIO error")
+          rescue => e
+            cycle_killed = true
+            cdisplay e.message
+            cdisplay e.backtrace
           ensure
+
             # cleanup
+            if cycle_killed
+              send_to_consumers(KILL_CYCLE_MESSAGE, false)
+            end
             pid = wait_thread[:pid] 
             ml_input.close
-            ml_output.close
-            File.delete("#{ml_pipe}.in")
+            if File.exists?("#{ml_pipe}.in")
+              ml_output.close
+              File.delete("#{ml_pipe}.in")
+            end
             stdout.close
             stderr.close
             Process.kill('INT', pid)
+            Process.exit!(true)
           end
         end
+
       rescue PTY::ChildExited
-        cdisplay("The child process exited!")
+        if File.exists?("#{ml_pipe}.in")
+          File.delete("#{ml_pipe}.in")
+        end
       end
     end
   end
@@ -387,6 +418,9 @@ module Zillabyte; module Runner; class MultilangOperation
     # Keep track of how many consumers to handle before exiting
     consumers_running = consumer_hash.keys.length
 
+    # Kill the cycle on error
+    cycle_killed = false
+
     # Setup multilang pipe
     ml_pipe = "#{@__name}_pipe"
     if File.exists?("#{ml_pipe}.in")
@@ -528,14 +562,16 @@ module Zillabyte; module Runner; class MultilangOperation
                           send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
                           consumers_running -= 1
                           if consumers_running == 0
-                            break
+                            return
                           end
                         end
                       end
                     end
-
                   end
-
+       
+                when "kill_cycle"
+                  cycle_killed = true
+                  return
                 end
 
               # Received a tuple from consumee
@@ -557,7 +593,7 @@ module Zillabyte; module Runner; class MultilangOperation
 
             # Exit after ending consumer cycles
             if consumers_running == 0
-              break
+              return
             end
 
           end
@@ -566,16 +602,29 @@ module Zillabyte; module Runner; class MultilangOperation
            cdisplay e.backtrace
         ensure
           # cleanup
+          if cycle_killed
+            send_to_consumers(KILL_CYCLE_MESSAGE, false)
+            send_to_consumees(KILL_CYCLE_MESSAGE, false)
+          end
           pid = wait_thread[:pid] 
-          ml_input.close  
-          ml_output.close
-          File.delete("#{ml_pipe}.in")
+          ml_input.close
+          if File.exists?("#{ml_pipe}.in")
+            ml_output.close
+            File.delete("#{ml_pipe}.in")
+          end
           stdout.close
           stderr.close
+          Process.kill('INT', pid)
+          Process.exit!(true)
         end
       end
     rescue PTY::ChildExited
+     
       cdisplay("The child process exited!")
+    ensure
+      if File.exists?("#{ml_pipe}.in")
+        File.delete("#{ml_pipe}.in")
+      end
     end
   end
 
@@ -588,6 +637,9 @@ module Zillabyte; module Runner; class MultilangOperation
     # Keep track of how many consumers to handle before exiting
     consumers_running = consumer_hash.keys.length
 
+    # Kill the cycle on error
+    cycle_killed = false
+
     # Setup groups
     group_by = @__node['group_by']
     group_tuples = {}
@@ -696,7 +748,11 @@ module Zillabyte; module Runner; class MultilangOperation
                       write_message(ml_input, tuple_json)
                     end
                   end
-
+              
+                # An error has occured
+                when "kill_cycle"
+                  cycle_killed = true
+                  return
                 end
 
               # Received a tuple from operation
@@ -790,13 +846,17 @@ module Zillabyte; module Runner; class MultilangOperation
             elsif consumers_running == 0
               break
             end
-
-            
+          
           end
         rescue Errno::EIO
            cdisplay("Errno:EIO error")
         ensure
           # cleanup
+          if cycle_killed
+            send_to_consumers(KILL_CYCLE_MESSAGE, false)
+            send_to_consumers(KILL_CYCLE_MESSAGE, false)
+          end
+
           pid = wait_thread[:pid]  
           ml_input.close 
           ml_output.close
@@ -804,9 +864,13 @@ module Zillabyte; module Runner; class MultilangOperation
           stdout.close
           stderr.close
           Process.kill('INT', pid)
+          Process.exit!(true)
         end  
       end
     rescue PTY::ChildExited
+      if File.exists?("#{ml_pipe}.in")
+        File.delete("#{ml_pipe}.in")
+      end
       cdisplay("The child process exited!")
     end
   end
@@ -828,6 +892,9 @@ module Zillabyte; module Runner; class MultilangOperation
     # Index the consumee streams for left and right sides
     consumer_hash = build_consumer_hash()
 
+    # Kill the cycle on error
+    cycle_killed = false
+
     # read all tuples from lefthand and right hand streams
     read_streams = [@__consumee_pipes["rd_child_1"], @__consumee_pipes["rd_child_2"]]
 
@@ -879,6 +946,10 @@ module Zillabyte; module Runner; class MultilangOperation
             if left_end_cycle_received && right_end_cycle_received
               break
             end
+         
+          when "kill_cycle"
+            cycle_killed = true
+            return
           end
 
          # Received a tuple from consumee
@@ -1115,21 +1186,27 @@ module Zillabyte; module Runner; class MultilangOperation
         stream = consumer_hash[r][:stream]
 
         # Consumer is ready for next message
-        if msg["command"] && msg["command"] == "next"
+        if msg["command"] 
+          case msg["command"] 
+          when"next"
 
-          @__emit_queues[stream][consumer][:ready] = true
-          tuple_json = get_consumer_tuple(stream, consumer)
+            @__emit_queues[stream][consumer][:ready] = true
+            tuple_json = get_consumer_tuple(stream, consumer)
 
-          # If all messages have been sent to a consumer, end its cycle
-          if tuple_json.nil?
-            send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
-            consumers_running -= 1
-            if consumers_running == 0
-              return
+            # If all messages have been sent to a consumer, end its cycle
+            if tuple_json.nil?
+              send_command_tuple(stream, consumer, END_CYCLE_MESSAGE)
+              consumers_running -= 1
+              if consumers_running == 0
+                return
+              end
+            else
+              # Emit tuple to consumer
+              emit_consumer_tuple(stream, consumer, tuple_json)
             end
-          else
-            # Emit tuple to consumer
-            emit_consumer_tuple(stream, consumer, tuple_json)
+          when "kill_cycle"
+            cycle_killed = true
+            return
           end
         end
       end
@@ -1148,7 +1225,8 @@ module Zillabyte; module Runner; class MultilangOperation
     type_map = {
       "string" => String,
       "double" => Float,
-      "integer" => Integer
+      "integer" => Integer,
+      "float" => Float
     }
 
     col_map = {}
@@ -1158,13 +1236,12 @@ module Zillabyte; module Runner; class MultilangOperation
       col_map[key] = type_map[type]
     end
 
-    messages = []
+    tuples = []
 
     output = @__options["output"]
     loop do
       # Read messages
-      msg = read_message(@__consumee_pipes["rd_child_1"])
-      obj = msg
+      obj = read_message(@__consumee_pipes["rd_child_1"])
 
       # Add row
       if obj['tuple']
@@ -1172,7 +1249,7 @@ module Zillabyte; module Runner; class MultilangOperation
         tuple = obj['tuple']
         display_json = Hash[obj['tuple'].map{|k, v| [Zillabyte::Runner::Operation.truncate_message(k), Zillabyte::Runner::Operation.truncate_message(v)]}].to_json
 
-        if col_map.keys.length != tuple.keys.length
+        if col_map.keys.length > tuple.keys.length
           cdisplay "Error: invalid keys for sink tuple : Expected #{col_map.keys} , got: #{tuple.keys}"
           cdisplay("\n \nPress Ctrl-C to exit", false)
           return
@@ -1199,24 +1276,27 @@ module Zillabyte; module Runner; class MultilangOperation
 
         if columns_to_check != 0
           cdisplay "Error: invalid schema for sink tuple #{display_json}"
-          cdisplay("\n \nPress Ctrl-C to exit", false)
           return
         end
-        messages << msg
+        tuples << obj
         if @__options[:interactive]
           cdisplay "received #{display_json}"
         end
         write_message(@__consumee_pipes["wr_child_1"], NEXT_MESSAGE)
 
       # End cycle
-      elsif obj['command'] && obj['command'] == "end_cycle"
-        break
+      elsif obj['command']
+        case obj['command']
+        when "end_cycle" 
+          break
+        when "kill_cycle"
+          break
+        end
       end
     end
 
-    if messages.empty?
-      cdisplay "no tuples"
-      cdisplay "use Ctrl-C to exit"
+    if tuples.empty?
+      cdisplay "No tuples received"
       return
     end
 
@@ -1226,7 +1306,7 @@ module Zillabyte; module Runner; class MultilangOperation
     require("csv")
     csv_str = CSV.generate do |csv|
       header_written = false;
-      messages.each do |obj|
+      tuples.each do |obj|
         begin
 
           t = obj['tuple']
@@ -1257,7 +1337,7 @@ module Zillabyte; module Runner; class MultilangOperation
 
     # Output table
     cdisplay("\n#{table.to_s}")
-    cdisplay "#{messages.length} rows"
+    cdisplay "#{tuples.length} rows"
       
     # Write file
     if output
@@ -1267,7 +1347,6 @@ module Zillabyte; module Runner; class MultilangOperation
       f.close()
       cdisplay("output written to #{filename}")
     end
-    cdisplay("\n \nPress Ctrl-C to exit", false)
   end
 
 
@@ -1299,7 +1378,6 @@ module Zillabyte; module Runner; class MultilangOperation
   # Read a JSON message
   def self.read_message(read_stream)
 
-
     @__read_buffers[read_stream] ||= ""
     @__read_buffered_messages[read_stream] ||= []
     if !@__read_buffered_messages[read_stream].empty?
@@ -1373,17 +1451,23 @@ module Zillabyte; module Runner; class MultilangOperation
 
   # Instruct multilang to begin cycle
   def self.begin_cycle(write_stream, read_stream)
-    begin
-      write_message(write_stream, BEGIN_CYCLE_MESSAGE)
+    write_message(write_stream, BEGIN_CYCLE_MESSAGE)
+   
+    while 1 do
+
       msg = read_message(read_stream)
       obj = Hash[msg]
-      if obj["command"] != "done"
-        raise "Invalid response from multilang #{msg}"
-      end
-    rescue => e
-      cdisplay("Error beginning cycle")
-      raise e
-    end
+      case obj["command"]
+        when "log"
+          cdisplay "LOG: #{obj['msg']}"
+        when "done"
+          break
+        else
+          cdisplay("Error beginning cycle")
+          raise obj["msg"]
+        end
+     end
+  
   end
 
   # Build the hash of consumer streams for lookup when receiving responses
@@ -1407,12 +1491,27 @@ module Zillabyte; module Runner; class MultilangOperation
     return consumer_hash
   end
 
+  # Send object to every consumer of the operation, regardless of stream
+  def self.send_to_consumees(json_obj)
+    pipes = @__consumee_pipes
+    # Left hand(or singular) input
+    if (pipes.has_key? "wr_parent_1")
+      write_stream = get_write_stream(stream, consumer, 1)
+      write_message(write_stream, json_obj)
+    end
+
+    # Right hand input 
+    if (pipes.has_key? "wr_parent_2")
+      write_stream = get_write_stream(stream, consumer, 2)
+      write_message(write_stream, json_obj)
+    end
+  end
 
   # Send object to every consumer of the operation, regardless of stream
-  def self.send_to_consumers(json_obj)
+  def self.send_to_consumers(json_obj, display = false)
     @__consumer_pipes.each_pair do |stream, consumers|
       consumers.each_pair do |consumer, pipe| 
-
+        @__consumee_pipes["wr_child_1"]
         # Single or Left hand pipe
         if (pipe.has_key? "wr_parent_1")
           write_stream = get_write_stream(stream, consumer, 1)
@@ -1421,8 +1520,9 @@ module Zillabyte; module Runner; class MultilangOperation
           write_stream = get_write_stream(stream, consumer, 2)
           write_message(write_stream, json_obj)
         end
-
-        cdisplay "emitted #{json_obj} to #{consumer}"
+        if display
+          cdisplay "emitted #{json_obj.chomp} to #{consumer}"
+        end
       end
     end
   end
@@ -1441,6 +1541,7 @@ module Zillabyte; module Runner; class MultilangOperation
   end
 
 
+
   # Send a command message to a consumer
   def self.send_command_tuple(stream, consumer, json_obj)
     pipe = @__consumer_pipes[stream][consumer]

data/lib/zillabyte-cli/version.rb

@@ -1,5 +1,5 @@
 module Zillabyte
   module CLI
-    VERSION = "0.1.37"
+    VERSION = "0.1.38"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.37
+  version: 0.1.38
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-29 00:00:00.000000000 Z
+date: 2014-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake