zillabyte-cli 0.1.0 → 0.1.1

data/lib/zillabyte/runner/component_operation.rb

@@ -11,6 +11,7 @@ require 'zillabyte/api/components'
 class Zillabyte::Runner::ComponentOperation
 
   NEXT_MESSAGE = "{\"command\": \"next\"}\n"
+  DONE_MESSAGE = "{\"command\": \"done\"}\n"
   END_CYCLE_MESSAGE = "{\"command\": \"end_cycle\"}\n"
   ENDMARKER = "\nend\n"
 
@@ -38,15 +39,14 @@ class Zillabyte::Runner::ComponentOperation
         @__emit_queues[stream][consumer] = {:write_queue => [], :ready => true}
       end
     end
-
     begin
       case @__type
       when "source"
         self.run_input()
-      when "sink"
-        self.run_output()
       when "component"
         self.run_rpc_component()
+
+      # Component outputs act in same manner as sinks
       else
         Zillabyte::Runner::MultilangOperation.run(node, dir, consumee, consumer_pipes, tester, meta, options)
       end
@@ -64,24 +64,111 @@ class Zillabyte::Runner::ComponentOperation
 
     # Read input from file
     if input
+      messages = []
       cdisplay "reading from file...."
       csv_rows = CSV.read("#{input}")
+      fields = @__node["fields"].map {|f| f.keys[0]}
+      csv_rows.each do |row|
+        tuple = {}
+        fields.each {|f| tuple[f] = row.shift}
 
-    else
+        tuple_json = build_tuple_json(tuple)
+
+        @__emit_queues.each_pair do |stream, consumers|
+          consumers.each_pair do |consumer, emitter|
+            emitter[:write_queue] << tuple_json
+          end
+        end
+
+        # Index streams and consumers by their pipes for lookup
+        consumer_hash = {}
+        @__emit_queues.each_pair do |stream, consumers|
+          consumers.each_key do |consumer|
+            read_stream = @__consumer_pipes[stream][consumer][:rd_parent]
+            consumer_hash[read_stream] = {:stream => stream, :consumer => consumer}
+          end
+        end
+
+
+        # Send first tuple
+        @__emit_queues.each_pair do |stream, consumers|
+          consumers.each_key do |consumer|
+            tuple_json = get_consumer_tuple(stream, consumer)
+            emit_consumer_tuple(stream, consumer, tuple_json)
+          end
+        end
+
+        # Sent tuples to consumers as appropriate
+        loop do 
+
+          # Retrieve messages from consumers
+          rs, ws, es = IO.select(consumer_hash.keys, [], [])
 
+          # Emit tuples to consumers
+          emitted = false
+          rs.each do |r|
+
+            # Read from consumer
+            msg = read_message(r)
+
+            stream = consumer_hash[r][:stream]
+            consumer = consumer_hash[r][:consumer]
+
+            # Consumer is ready for next message
+            if msg["command"] && msg["command"] == "next"
+
+              @__emit_queues[stream][consumer][:ready] = true
+              tuple_json = get_consumer_tuple(stream, consumer)
+
+              # If all messages have been sent to consumer, end their cycle
+              if tuple_json.nil?
+                write_stream = get_write_stream(stream, consumer)
+                cdisplay "ending cycle for #{consumer}"
+                write_message(write_stream, END_CYCLE_MESSAGE)
+                write_message(write_stream, DONE_MESSAGE)
+
+              else
+                # Emit tuple to consumer
+                emit_consumer_tuple(stream, consumer, tuple_json)
+                emitted = true
+              end
+            end
+          end
+
+          # Exit when done emitting
+          if !emitted
+            return
+          end
+        end
+      end
+      
+    else
       stdin = @__consumee[:rd_child]
       loop do 
 
         msg = stdin.gets
+        if msg == "end\n"
+          @__consumer_pipes.each_pair do |stream, consumers|
+            consumers.each_pair do |consumer, pipe| 
+              write_stream = pipe[:wr_parent]
+              write_message(write_stream, END_CYCLE_MESSAGE)
+              write_message(write_stream, DONE_MESSAGE)
+            end
+          end
+
+          break
+        end
 
         # Build tuple
         begin
+
           tuple = JSON.parse(msg)
         rescue JSON::ParserError 
           cdisplay "Error: invalid JSON"           
           next
         end
 
+
         # Check input for correct fields
         has_fields = true
         fields = @__node['fields'].flat_map {|h| h.keys}
@@ -124,6 +211,28 @@ class Zillabyte::Runner::ComponentOperation
     # The input component(singular at the moment)
     read_streams = consumer_hash.keys.concat [@__consumee[:rd_child]]
 
+    # Start communication with API
+    api = @__tester.session.api
+    caller = Zillabyte::API::Components.new(api)
+    component_id = @__node["id"]
+    output_format = @__node["output_format"]
+    component_info = api.request(
+      :expects  => 200,
+      :method   => :get,
+      :path     => "/flows/#{component_id}"
+    )
+    component_schema = component_info.body["schema"]
+    cdisplay("error: The requested component is not properly registered.") if component_info.nil?
+
+    component_nodes = component_schema["nodes"]
+    source_nodes = []
+    component_nodes.each do |node|
+      source_nodes << node if node["type"] == "source"
+    end
+    cdisplay("error: This component has multiple input streams. Currently we only support single input streams, sorry!") if source_nodes.size > 1
+
+    fields = source_nodes[0]["fields"]
+
     # Receive and handle messages
     loop do
 
@@ -135,21 +244,19 @@ class Zillabyte::Runner::ComponentOperation
 
         # Handle tuple through RPC
         if obj['tuple']
+          cdisplay("error: The number of inputs to the component does not match the declared number for stream #{source_nodes[0]["name"]}.") if obj['tuple'].size != fields.size
+          rpc_inputs = []
+          fields.each do |field|
+            rpc_inputs << obj['tuple'][field.keys[0]]
+          end
 
           display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
           
 
-          # Start communication with API
-          api = "API #{@__tester.session.api}"
-          caller = Zillabyte::API::Components.new(api)
-          component_id = @__node["id"]
-          output_format = @__node["output_format"]
-
           # Send RPC call
-          tuple = obj['tuple']
           cdisplay "sending RPC call..."
           call_options = {
-            :rpc_inputs => [tuple],
+            :rpc_inputs => [rpc_inputs],
             :output_format => output_format
           }
 
@@ -161,7 +268,10 @@ class Zillabyte::Runner::ComponentOperation
             cdisplay("error: #{res['error']}")
             next
           end
-          run_ids = res['run_ids']
+          run_ids = []
+          res['execute_ids'].each do |q, qid|
+            run_ids << qid
+          end
           status = res['status']
 
 
@@ -275,60 +385,6 @@ class Zillabyte::Runner::ComponentOperation
 
   end
 
-  # Run a component output
-  def self.run_output()
-
-
-    output = @__options[:output]
-    header_written = false
-    read_stream = @__consumee[:rd_child]
-
-    loop do
-      # Read a tuple
-      obj = read_message(read_stream)
-
-
-      # Add row
-      if obj['tuple']
-
-        if output
-          filename = "#{output}.csv"
-          t = obj['tuple']
-          m = obj['meta'] || {}
-
-          # Write header
-          if header_written == false
-            keys = [t.keys, m.keys].flatten
-            csv_str = keys
-            f = File.open(filename, "w")
-            f.write(csv_str)
-            f.close()
-            header_written = true
-          end
-
-          # Write CSV row
-          vals = [t.values, m.values].flatten
-          csv_str = vals.to_csv
-          
-          f = File.open(filename, "w")
-          f.write(csv_str)
-          f.close()
-        else
-
-          display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
-          cdisplay "received #{display_json}"
-        end
-
-        # Ready for next message
-        write_message(@__consumee[:wr_child], NEXT_MESSAGE)
-
-      # End cycle
-      elsif obj['command'] && obj['command'] == "end_cycle"
-        break
-      end
-    end
-
-  end
 
 
   private
@@ -526,4 +582,4 @@ class Zillabyte::Runner::ComponentOperation
   end
 
 
-end
+end

data/lib/zillabyte/runner/component_runner.rb

@@ -8,9 +8,9 @@ require 'thread'
 class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base 
   include Zillabyte::Helpers
 
-  def run (dir = Dir.pwd, session = nil, options = {})
+  def run (meta, dir = Dir.pwd, session = nil, options = {})
 
-    if session.nil?
+    if meta.nil? or session.nil?
       return
     end
 
@@ -22,23 +22,6 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
     output = options[:output]
     otype = options[:output_type] 
 
-    # Get component metadata
-    meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, @session, {:test => true})
-    if meta.nil? || meta["nodes"].nil?
-      error "this is not a valid zillabyte app directory"
-      exit
-    end
-
-
-    # Check that multilang version is atleast 0.1.0
-    version = meta["multilang_version"] || "0.0.0"
-    version_arr = version.split('.').map {|v| v.to_i}
-    if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
-      display "The version of zillabyte used in your application is outdated."
-      display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
-      return
-    end
-
     # Show the user what we know about their app... 
     display "inferring your app details..."
     describe_component(meta)
@@ -156,8 +139,26 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
       # TODO handle inputs
       if input.nil?
 
+        source = ""
+        @nodes.each do |n|
+          name = n["name"]
+          type = n["type"]
+
+          if type == "source"
+            if source == ""
+              source = name
+            else
+              display "Cannot run component with multiple input sources without input files"
+              return
+            end
+          end
+        end
+          
+        display ""
+        display "To view results:  Enter 'end'    "
+        display "To exit the test: Enter 'Ctrl-C' "
+        display ""
 
-        display "Use Ctrl-C to exit"
         while true
 
           # TODO this doesnt handle multiple sources
@@ -165,15 +166,8 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
 
           display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
           msg = ask
-
-          begin
-            JSON.parse(msg)
-          rescue JSON::ParserError
-            display "Received invalid JSON object"
-            next
-          end
           # Send tuple to source
-          @operation_pipes["stream_1"][:wr_parent].puts msg
+          @operation_pipes[source][:wr_parent].puts msg
         end
       end
    
@@ -212,8 +206,8 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
     colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
     rjust = 20
 
-    display "#{'app name'.rjust(rjust)}: #{meta['name']}"
-    display "#{'app language'.rjust(rjust)}: #{meta['language']}"
+    display "#{'component name'.rjust(rjust)}: #{meta['name']}"
+    display "#{'component language'.rjust(rjust)}: #{meta['language']}"
     meta['nodes'].each_with_index do |node, index|
       color = @colors[node['name']] ||= colors[index % colors.length]
       display (("="*rjust + "  operation ##{index}").colorize(color))
@@ -241,4 +235,4 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
 
 
 
-end
+end

data/lib/zillabyte/runner/multilang_operation.rb

@@ -361,7 +361,7 @@ class Zillabyte::Runner::MultilangOperation
                 
                 # Multilang sent a ping
                 elsif obj['ping']
-                  write_message(to_ml, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
+                  write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
                 end
               end
 
@@ -433,6 +433,7 @@ class Zillabyte::Runner::MultilangOperation
           mutlilang_count = 0
           end_cycle_received = false
 
+
           # Receive and handle messages
           loop do
 
@@ -444,12 +445,13 @@ class Zillabyte::Runner::MultilangOperation
               if r == stdout
                 msg = r.gets
                 msg = msg.sub(/\n/, "")
-                cdisplay("log: #{msg}")
+                cdisplay("LOG: #{msg}")
                 next
               end
 
               # Receive an object
               obj = read_message(r)
+
               if obj["command"] 
                 case obj["command"]
 
@@ -540,7 +542,6 @@ class Zillabyte::Runner::MultilangOperation
                   end_cycle_received = true
 
                   if mutlilang_count == 0
-                    
                     @__emit_queues.each_pair do |stream, consumers|
                       consumers.each_pair do |consumer, emitter|                      
                         if emitter[:ready]
@@ -571,7 +572,7 @@ class Zillabyte::Runner::MultilangOperation
               
               # Multilang sent a ping
               elsif obj['ping']
-                write_message(to_ml, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
+                write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
               end
             end
 
@@ -751,6 +752,9 @@ class Zillabyte::Runner::MultilangOperation
                 # Ask operation for next tuple
                 write_message(@__consumee[:wr_child], NEXT_MESSAGE)
 
+              # Multilang sent a ping
+              elsif obj['ping']
+                write_message(ml_input, PONG_PREFIX + "#{Time.now.utc.to_f}" + PONG_SUFFIX)
               end
             end
 
@@ -866,6 +870,7 @@ class Zillabyte::Runner::MultilangOperation
 
     if messages.empty?
       cdisplay "empty relation"
+      cdisplay "use Ctrl-C to exit"
       return
     end
 
@@ -913,6 +918,7 @@ class Zillabyte::Runner::MultilangOperation
       f.close()
       cdisplay("output written to #{filename}")
     end
+    cdisplay "\n \nPress Ctrl-C to exit"
   end
 
 
@@ -956,6 +962,7 @@ class Zillabyte::Runner::MultilangOperation
     loop do
 
       while !@__read_buffers[read_stream].include? ENDMARKER
+
         segment = read_stream.sysread(BUFSIZE)
         @__read_buffers[read_stream] << segment
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-26 00:00:00.000000000 Z
+date: 2014-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -274,7 +274,6 @@ files:
 - lib/zillabyte/cli/components.rb
 - lib/zillabyte/cli/config.rb
 - lib/zillabyte/cli/counters.rb
-- lib/zillabyte/cli/executes.rb
 - lib/zillabyte/cli/flows.rb
 - lib/zillabyte/cli/git.rb
 - lib/zillabyte/cli/help.rb
@@ -294,8 +293,16 @@ files:
 - lib/zillabyte/cli/templates/apps/python/requirements.txt
 - lib/zillabyte/cli/templates/apps/python/zillabyte.conf.yaml
 - lib/zillabyte/cli/templates/apps/ruby/Gemfile
-- lib/zillabyte/cli/templates/apps/ruby/app.rb
+- lib/zillabyte/cli/templates/apps/ruby/app.rb.erb
 - lib/zillabyte/cli/templates/apps/ruby/zillabyte.conf.yaml
+- lib/zillabyte/cli/templates/components/js/simple_function.js
+- lib/zillabyte/cli/templates/components/js/zillabyte.conf.yaml
+- lib/zillabyte/cli/templates/components/python/component.py
+- lib/zillabyte/cli/templates/components/python/requirements.txt
+- lib/zillabyte/cli/templates/components/python/zillabyte.conf.yaml
+- lib/zillabyte/cli/templates/components/ruby/Gemfile
+- lib/zillabyte/cli/templates/components/ruby/component.rb.erb
+- lib/zillabyte/cli/templates/components/ruby/zillabyte.conf.yaml
 - lib/zillabyte/cli/templates/python/#simple_function.py#
 - lib/zillabyte/cli/version.rb
 - lib/zillabyte/cli/zillalogs.rb