wukong 3.0.0.pre2 → 3.0.0.pre3

data/lib/wukong/runner.rb

@@ -0,0 +1,144 @@
+module Wukong
+  module CommandlineRunner
+            
+    def exit_with_status(status, options = {})
+      warn options[:msg] if options[:msg]
+      @env.dump_help     if options[:show_help]
+      exit(status)
+    end    
+
+    def env= settings
+      @env = settings
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+    
+    module ClassMethods
+      
+      def usage(usg = nil)
+        return @usage if usg.nil?
+        @usage = usg
+      end
+
+      def desc(dsc = nil)
+        return @description if dsc.nil?
+        @decription = desc
+      end
+
+      def add_param(*args)
+        defined_params << args
+      end
+      
+      def defined_params
+        @defined_params ||= []
+      end
+
+      def base_config(conf = nil)
+        return @base_configuration if conf.nil?
+        @base_configuration = conf
+      end
+
+      def decorate_environment! env
+        usg = self.usage
+        env.define_singleton_method(:usage){ usg }
+        env.description = self.desc
+        defined_params.each{ |params| env.send(:define, *params) }
+      end
+
+      def in_deploy_pack?
+        return @in_deploy_pack unless @in_deploy_pack.nil?
+        @in_deploy_pack = (find_deploy_pack_dir != '/')
+      end
+
+      def find_deploy_pack_dir
+        return @deploy_pack_dir if @deploy_pack_dir
+        wd     = Dir.pwd
+        parent = File.dirname(wd)
+        until wd == parent
+          return wd if File.exist?(File.join(wd, 'Gemfile')) && File.exist?(File.join(wd, 'config', 'environment.rb'))
+          wd     = parent
+          parent = File.dirname(wd)
+        end
+        @deploy_pack_dir = wd
+      end
+
+      def run!(*run_params)
+        settings   = base_configuration || Configliere::Param.use(:commandline)
+        boot_environment(settings) if in_deploy_pack?
+        runner     = new(*run_params)
+        runner.env = settings.resolve!
+        runner.run(*settings.rest)
+      end
+      
+    end    
+  end
+  
+  class LocalRunner
+    include CommandlineRunner
+    base_configuration
+
+    usage 'usage: wu-local PROCESSOR|FLOW [ --param=value | -p value | --param | -p]'
+    desc  <<EOF
+ wu-local is a tool for running Wukong processors and flows locally on
+ the command-line.  Use wu-local by passing it a processor and feeding
+ in some data:
+
+   $ echo 'UNIX is Clever and Fun...' | wu-local tokenizer.rb
+   UNIX
+   is
+   Clever
+   and
+   Fun
+
+ If your processors have named fields you can pass them in as
+ arguments:
+
+   $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4
+   UNIX
+   Clever
+
+ You can chain processors and calls to wu-local together:
+
+   $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4 | wu-local downcaser.rb
+   unix
+   clever
+
+ Which is a good way to develop a combined data flow which you can
+ again test locally:
+
+   $ echo 'UNIX is clever and fun...' | wu-local tokenize_and_downcase_big_words.rb
+   unix
+   clever
+EOF
+    
+    add_param :run,        description: "Name of the processor or dataflow to use. Defaults to basename of the given path.", flag: 'r'
+    add_param :tcp_server, description: "Run locally as a server using provided TCP port", default: false,                   flag: 't'
+
+    def run *args
+      arg = args.first
+      case
+      when arg.nil?
+        exit_with_status(1, show_help: true, msg: "Must pass a processor name or path to a processor file. Got <#{arg}>")
+      when Wukong.registry.registered?(arg.to_sym)
+        processor = arg.to_sym
+      when File.exist?(arg)
+        load arg
+        processor = @env.run || File.basename(arg, '.rb')
+      else
+        exit_with_status(2, show_help: true, msg: "Must pass a processor name or path to a processor file. Got <#{arg}>")
+      end     
+      run_em_server(processor, @env)
+    end
+    
+    def run_em_server(processor, env)
+      EM.run do 
+        env.tcp_server ? Wu::TCPServer.start(processor, env) : Wu::StdioServer.start(processor, env)
+      end
+    rescue Wu::Error => e
+      exit_with_status(3, msg: e.backtrace.join("\n"))
+    end
+
+  end
+end

data/lib/wukong/server.rb

@@ -0,0 +1,119 @@
+module Wukong
+  module EventMachineServer
+    include DriverMethods
+
+    def self.included klass
+      klass.class_eval do
+        attr_accessor :dataflow, :settings  
+
+        def self.add_signal_traps
+          Signal.trap('INT')  { log.info 'Received SIGINT. Stopping.'  ; EM.stop }
+          Signal.trap('TERM') { log.info 'Received SIGTERM. Stopping.' ; EM.stop }                  
+        end
+      end
+    end
+
+    def initialize(label, settings)
+      super
+      @settings = settings      
+      @dataflow = construct_dataflow(label, settings)
+    end
+      
+  end
+
+  class StdioServer < EM::P::LineAndTextProtocol
+    include EventMachineServer
+    include Processor::StdoutProcessor
+    include Logging
+    
+    def self.start(label, settings = {})
+      EM.attach($stdin, self, label, settings)
+    end
+
+    def post_init      
+      self.class.add_signal_traps
+      setup_dataflow
+    end
+
+    def receive_line line
+      driver.send_through_dataflow(line)
+    end
+
+    def unbind
+      finalize_and_stop_dataflow
+      EM.stop
+    end
+    
+  end
+
+  class TCPServer < EM::P::LineAndTextProtocol
+    include EventMachineServer
+    include Processor::BufferedProcessor
+    include Logging
+       
+    def self.start(label, settings = {})
+      host = settings[:host] || Socket.gethostname
+      port = settings[:port] || 9000
+      EM.start_server(host, port, self, label, settings)
+      log.info "Server started on #{host} on port #{port}"
+      add_signal_traps
+    end
+
+    def post_init
+      port, ip = Socket.unpack_sockaddr_in(get_peername)
+      log.info "Connected to #{ip} on #{port}"
+      setup_dataflow
+    end
+
+    def receive_line line
+      @buffer = []      
+      operation = proc { driver.send_through_dataflow(line) }
+      callback  = proc { flush_buffer @buffer }
+      EM.defer(operation, callback)
+    end
+
+    def flush_buffer records
+      send_data(records.join("\n") + "\n")
+      records.clear
+    end
+
+    def unbind
+      EM.stop
+    end
+    
+  end
+end
+
+class StupidServer
+
+  attr_accessor :dataflow, :settings
+  
+  def initialize(label, settings)
+    @settings = settings
+    builder   = (Wukong.registry.retrieve(label.to_sym) or raise Wukong::Error.new("No such processor or dataflow: #{label}"))
+    dataflow  = builder.build(settings)
+    @dataflow = dataflow.respond_to?(:stages) ? dataflow.directed_sort.map{ |name| dataflow.stages[name] } : [ dataflow ]
+    @dataflow << self
+  end
+
+  def dumb_driver
+    @dumb_driver ||= Wukong::Driver.new(dataflow)
+  end
+
+  def stop() ; end
+  def setup() ; end
+  def process(record) $stdout.puts record ; end
+
+  def run!
+    dataflow.each(&:setup)
+
+    while line = $stdin.readline.chomp rescue nil do
+      dumb_driver.send_through_dataflow(line)
+    end
+    dataflow.each do |stage|
+      stage.finalize(&dumb_driver.advance(stage)) if stage.respond_to?(:finalize)
+      stage.stop
+    end      
+
+  end  
+end

data/lib/wukong/spec_helpers.rb

@@ -79,3 +79,4 @@ module Wukong
 
   Processor.class_eval { include SpecHelpers::ProcessorSpecMethods }
 end
+

data/lib/wukong/spec_helpers/integration_driver.rb

@@ -87,15 +87,17 @@ module Wukong
       # @return [true, false]
       def run!
         return false if ran?
-        Open3.popen3(env, cmd) do |i, o, e, wait_thr|
-          self.pid = wait_thr.pid
-
-          @inputs.each { |input| i.puts(input) }
-          i.close
-
-          self.stdout    = o.read
-          self.stderr    = e.read
-          self.exit_code = wait_thr.value.to_i
+        FileUtils.cd(cwd) do
+          Open3.popen3(env, cmd) do |i, o, e, wait_thr|
+            self.pid = wait_thr.pid
+            
+            @inputs.each { |input| i.puts(input) }
+            i.close
+            
+            self.stdout    = o.read
+            self.stderr    = e.read
+            self.exit_code = wait_thr.value.to_i
+          end
         end
         @ran = true
       end
@@ -116,6 +118,17 @@ module Wukong
         @inputs.concat(events)
         self
       end
+      alias_method :<, :on
+
+      def in dir
+        @cwd = dir
+        self
+      end
+
+      def using env
+        @env = env
+        self
+      end
 
       def env
         ENV.to_hash.merge(@env || {})

data/lib/wukong/spec_helpers/integration_driver_matchers.rb

@@ -80,7 +80,7 @@ module Wukong
 
       # :nodoc:
       def failure_message
-        "Ran\n\n  #{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n  #{failed_expectation}"
+        "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n  #{failed_expectation}#{formatted_error_output}"
       end
 
       # :nodoc:
@@ -93,9 +93,26 @@ module Wukong
         output.split("\n").map { |line| '  ' + line }.join("\n")
       end
 
+      # :nodoc:
+      def formatted_error_output
+        output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{driver.stdout}" : "\n\nSTDERR was\n\n#{driver.stderr}"
+      end
+
       # :nodoc:
       def formatted_command
-        "$ #{driver.cmd}"
+        "  $ #{driver.cmd}"
+      end
+
+      # :nodoc:
+      def formatted_env
+        ['  {'].tap do |lines|
+          driver.env.each_pair do |key, value|
+            if key =~ /^(BUNDLE_GEMFILE|PATH|RUBYLIB)$/
+              lines << "    #{key} => #{value},"
+            end
+          end
+          lines << '  }'
+        end.join("\n")
       end
 
       # :nodoc:
@@ -182,12 +199,12 @@ module Wukong
 
       # :nodoc:
       def failure_message
-        "Ran\n\n  #{formatted_command}\n\nexpecting #{expected_exit_code_description}  Got #{driver.exit_code} instead."
+        "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description}  Got #{driver.exit_code} instead.#{formatted_error_output}"
       end
 
       # :nodoc:
       def negative_failure_message
-        "Ran\n\n  #{formatted_command}\n\nNOT expecting #{expected_exit_code_description}."
+        "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
       end
 
       # :nodoc:
@@ -214,6 +231,11 @@ module Wukong
         "exit with #{expected_exit_code_description}"
       end
 
+      # :nodoc:
+      def output_description
+        "STDOUT"
+      end
+
     end
   end
 end

data/lib/wukong/spec_helpers/processor_helpers.rb

@@ -54,18 +54,13 @@ module Wukong
       #     ...
       #   end
       def processor *args, &block
-        case
-        when args.empty?
-          create_processor(self.class.description, {}, &block)
-        when args.first.is_a?(Hash)
-          create_processor(self.class.description, args.first, &block)
-        else
-          create_processor(args[0], (args[1] || {}), &block)
-        end
+        options = args.extract_options!
+        name    = args.first || self.class.description
+        create_processor(name, options, &block)
       end
       alias_method :flow, :processor
 
-      # Is the given `klass` a Wukong::Processor?
+      # Is the given +klass+ a Wukong::Processor?
       #
       # @param [Class] klass
       # @return [true, false]
@@ -92,4 +87,3 @@ module Wukong
     end
   end
 end
-

data/lib/wukong/spec_helpers/shared_examples.rb

@@ -1,15 +1,14 @@
-shared_examples_for 'a processor' do |options={}|
-  name = options[:named]
-  if name
-    it "is registered with the name '#{name}'" do
-      Wukong.registry.retrieve(name.to_sym).should_not be_nil
-    end
-    it{ create_processor(name).should respond_to(:setup)    }
-    it{ create_processor(name).should respond_to(:process)  }
-    it{ create_processor(name).should respond_to(:finalize) }
-    it{ create_processor(name).should respond_to(:stop)     }
-    it{ create_processor(name).should respond_to(:notify)   }
-  else
-    warn "Must supply a name for a processor you want to test"
+shared_examples_for 'a processor' do |options = {}|
+  let(:processor_name){ options[:named] || self.class.top_level_description }
+  subject             { create_processor(processor_name, on_error: :skip)   }
+
+  it 'is registered' do
+    Wukong.registry.retrieve(processor_name.to_sym).should_not be_nil
   end
+  
+  it{ should respond_to(:setup)    }
+  it{ should respond_to(:process)  }
+  it{ should respond_to(:finalize) }
+  it{ should respond_to(:stop)     }
+  it{ should respond_to(:notify)   }
 end

data/lib/wukong/version.rb

@@ -1,4 +1,4 @@
 module Wukong
   # The current version of Wukong.
-  VERSION = '3.0.0.pre2'
+  VERSION = '3.0.0.pre3'
 end

data/lib/wukong/widget/processors.rb

@@ -152,5 +152,18 @@ module Wukong
       end
       register
     end
+
+    # Mixin processor behavior
+    module BufferedProcessor
+      def setup()                           ; end
+      def process(record) @buffer << record ; end
+      def stop()                            ; end
+    end
+  
+    module StdoutProcessor
+      def setup()         $stdout.sync        ; end
+      def process(record) $stdout.puts record ; end
+      def stop()                              ; end
+    end
   end
 end