zillabyte-cli 0.0.24 → 0.1.0

data/lib/zillabyte/runner/component_runner.rb

@@ -0,0 +1,244 @@
+require "zillabyte/cli/base"
+require "zillabyte/runner"
+require "zillabyte/runner/multilang_operation"
+require "zillabyte/runner/component_operation"
+require 'thread'
+
+# HIDDEN:
+class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base 
+  include Zillabyte::Helpers
+
+  def run (dir = Dir.pwd, session = nil, options = {})
+
+    if session.nil?
+      return
+    end
+
+    @session = session
+    @colors = {}
+
+    # Get options
+    input = options[:input]
+    output = options[:output]
+    otype = options[:output_type] 
+
+    # Get component metadata
+    meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, @session, {:test => true})
+    if meta.nil? || meta["nodes"].nil?
+      error "this is not a valid zillabyte app directory"
+      exit
+    end
+
+
+    # Check that multilang version is atleast 0.1.0
+    version = meta["multilang_version"] || "0.0.0"
+    version_arr = version.split('.').map {|v| v.to_i}
+    if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
+      display "The version of zillabyte used in your application is outdated."
+      display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
+      return
+    end
+
+    # Show the user what we know about their app... 
+    display "inferring your app details..."
+    describe_component(meta)
+
+    # Setup streams
+    @nodes = meta["nodes"]
+
+    # Index stream consummers and emitters by stream name
+    @arcs = meta["arcs"]
+
+    # Organize component pipes
+    @operations = {}
+    @operation_pipes = {}
+
+
+    # Start component
+    begin
+
+      # Setup operation pipes
+      @nodes.each do |n|
+
+        name = n["name"]
+        type = n["type"]
+        if n["type"] ==  "source"
+          fields = n["fields"]
+        end
+        
+        # Create two new pipes in the parent.
+        rd_child, wr_parent = IO.pipe()
+        rd_parent, wr_child = IO.pipe()
+
+        @operation_pipes[name] = {
+          :node => n,
+          :rd_child => rd_child,      
+          :wr_child => wr_child,
+          :rd_parent => rd_parent,        
+          :wr_parent => wr_parent
+        }
+      end
+
+      # Maps origin => {stream => [destinations]}
+      @arc_map = {}
+      @arcs.each do |a|
+        origin = a["origin"]
+        name = a["name"]
+        dest = a["dest"]
+        @arc_map[origin] ||= {}
+        @arc_map[origin][name] ||= []
+        @arc_map[origin][name] << a["dest"]
+      end
+
+
+      # Spawn component threads
+      @nodes.each do |n|
+
+        name = n["name"]
+        type = n["type"]
+        emits = n["emits"]
+
+        pipes = @operation_pipes[name] 
+        rd_child = pipes[:rd_child]
+        wr_child = pipes[:wr_child]
+        rd_parent = pipes[:rd_parent]        
+        wr_parent = pipes[:wr_parent]
+
+        # Fork.
+        pid = fork()
+        if pid # In parent
+          # Close the reading end of the child so we can write to the child.
+          rd_child.close()
+          # Close the writing end of the child so we can read from the child.
+          wr_child.close()
+
+        else # in child
+          # Close the writing end of the parent so we can read from the parent.
+          wr_parent.close()
+          # Close the reading end of the parent so we can write to the parent.
+          rd_parent.close()
+          begin
+
+            # Setup reading and writing pipes for communicating with consumee component
+            in_pipe = {:rd_child => @operation_pipes[name][:rd_child], :wr_child => @operation_pipes[name][:wr_child]}
+
+            # Index consumer pipes by consumer_name
+            out_pipes = {}
+
+            if type != "sink"
+              @arc_map[name].each_pair do |stream, destinations|
+                out_pipes[stream] ||= {}
+                destinations.each do |dest|
+                  out_pipes[stream][dest] = {:wr_parent => @operation_pipes[dest][:wr_parent], :rd_parent => @operation_pipes[dest][:rd_parent] }
+                end
+              end
+            end
+
+            # Run the child process 
+            Zillabyte::Runner::ComponentOperation.run(n, dir, in_pipe, out_pipes, self, meta, options)
+
+         rescue => e
+          display e
+
+          ensure
+            # Close the reading end of the child
+            rd_child.close()
+            # Close the writing end of the child
+            wr_child.close()
+            exit!(-1)
+          end
+
+        end #end child
+      end 
+
+
+      # If no input file, read from STDIN
+      # TODO handle inputs
+      if input.nil?
+
+
+        display "Use Ctrl-C to exit"
+        while true
+
+          # TODO this doesnt handle multiple sources
+          #source = @operation_pipes["source_1"][:node]
+
+          display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
+          msg = ask
+
+          begin
+            JSON.parse(msg)
+          rescue JSON::ParserError
+            display "Received invalid JSON object"
+            next
+          end
+          # Send tuple to source
+          @operation_pipes["stream_1"][:wr_parent].puts msg
+        end
+      end
+   
+    ensure
+      Process.waitall()
+      @operation_pipes.each do |name, pipes|
+        #Close the  writing end of the parent
+        pipes[:wr_parent].close()
+        # Close the reading end of the parent
+        pipes[:rd_parent].close()
+      end
+    end
+
+  end
+
+  def session
+    @session
+  end
+
+
+  def cdisplay(name, message)
+     color = @colors[name] || :default
+      if message == ""
+        display ""
+      else
+        display "#{name} - #{message}".colorize(color)
+      end
+  end
+
+
+  def display(message, newline = true)
+    @session.display(message, newline)
+  end
+
+  def describe_component(meta)
+    colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
+    rjust = 20
+
+    display "#{'app name'.rjust(rjust)}: #{meta['name']}"
+    display "#{'app language'.rjust(rjust)}: #{meta['language']}"
+    meta['nodes'].each_with_index do |node, index|
+      color = @colors[node['name']] ||= colors[index % colors.length]
+      display (("="*rjust + "  operation ##{index}").colorize(color))
+      display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
+
+      # Convert metadata typing to that of components
+      if node['type'] == "source"
+        type = "input"
+        display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
+        display "#{"fields".rjust(rjust)}: #{node['fields'].to_s.colorize(color)}"
+        display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
+      elsif node['type'] == "sink"
+        type = "output"
+        display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
+        display "#{"columns".rjust(rjust)}: #{node['columns'].to_s.colorize(color)}"
+
+      else
+        type = node['type']
+        display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
+      end
+    end
+  end
+
+
+
+
+
+end