wukong 3.0.0.pre3 → 3.0.0

data/bin/wu-local

@@ -1,77 +1,4 @@
 #!/usr/bin/env ruby
 
 require 'wukong'
-
-settings = Wukong::Local::Configuration
-settings.use(:commandline)
-
-def settings.usage
-  "usage: #{File.basename($0)} PROCESSOR|FLOW [ --param=value | -p value | --param | -p]"
-end
-
-settings.description = <<-EOF
-wu-local is a tool for running Wukong processors and flows locally on
-the command-line.  Use wu-local by passing it a processor and feeding
-in some data:
-
-  $ echo 'UNIX is Clever and Fun...' | wu-local tokenizer.rb
-  UNIX
-  is
-  Clever
-  and
-  Fun
-
-If your processors have named fields you can pass them in as
-arguments:
-
-  $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4
-  UNIX
-  Clever
-
-You can chain processors and calls to wu-local together:
-
-  $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4 | wu-local downcaser.rb
-  unix
-  clever
-
-Which is a good way to develop a combined data flow which you can
-again test locally:
-
-  $ echo 'UNIX is clever and fun...' | wu-local tokenize_and_downcase_big_words.rb
-  unix
-  clever
-EOF
-
-settings.define :run,        description: "Name of the processor or dataflow to use.  Defaults to basename of the given path.", flag: 'r'
-# settings.define :tcp_server, description: "Run locally as a tcp server on a specified port", default: false,                    flag: 't'
-require 'wukong/boot' ; Wukong.boot!(settings)
-
-thing = settings.rest.first
-case
-when thing.nil?
-  settings.dump_help
-  exit(1)
-when Wukong.registry.registered?(thing.to_sym)
-  processor = thing.to_sym
-when File.exist?(thing)
-  load thing
-  processor = settings.run || File.basename(thing, '.rb')
-else
-  settings.dump_help
-  exit(2)
-end
-
-
-
-begin
-  # EM.run do
-  #   settings.tcp_server ? Wu::TCPServer.start(processor.to_sym, settings) : Wu::StdioServer.start(processor.to_sym, settings)
-  # end
-  StupidServer.new(processor.to_sym, settings).run!
-rescue Wu::Error => e
-  $stderr.puts e.message
-  exit(3)
-end
-
-# One day, it will be this easy...
-# Wukong::LocalRunner.run!
+Wukong::Local::LocalRunner.run

data/diagrams/wu_local.dot

@@ -0,0 +1,39 @@
+digraph WuLocalControlFlow {
+	size ="100,100";
+	stdin [label=<
+	       <TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+	       <TR><TD><FONT FACE="BOLD" POINT-SIZE="20">STDIN</FONT></TD></TR>
+	       <TR><TD><FONT POINT-SIZE="10">The line of input text</FONT></TD></TR>
+	       <TR><TD><FONT FACE="MONOSPACE">Shall I compare thee to a summers day?<BR/>Thou art more lovely and more temperate<BR/>...</FONT></TD></TR>
+	       
+	       </TABLE>>];
+	command [shape=diamond,label=<
+		 <TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		 <TR><TD><FONT FACE="BOLD" POINT-SIZE="20">Command</FONT></TD></TR>
+		 <TR><TD><FONT POINT-SIZE="10">A UNIX process launched on the command line</FONT></TD></TR>
+		 <TR><TD><FONT FACE="MONOSPACE">wu-local word_counter</FONT></TD></TR>
+		 </TABLE>>];
+	runner [shape=box,label=<
+		<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		<TR><TD><FONT FACE="BOLD" POINT-SIZE="20">Runner</FONT></TD></TR>
+		<TR><TD><FONT POINT-SIZE="10">Loads plugins and code, configures and resolves settings, boots plugins, validates command line, then runs.</FONT></TD></TR>
+		<TR><TD><FONT FACE="MONOSPACE">Wukong::Local::LocalRunner</FONT></TD></TR>
+		</TABLE>>];
+	driver [shape=box,label=<
+		<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		<TR><TD><FONT FACE="BOLD" POINT-SIZE="20">Driver</FONT></TD></TR>
+		<TR><TD><FONT POINT-SIZE="10">Passes input to processor, handles output</FONT></TD></TR>
+		<TR><TD><FONT FACE="MONOSPACE">Wukong::Local::StdioDriver</FONT></TD></TR>
+		</TABLE>>];
+	stdout [label=<
+		<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		<TR><TD><FONT FACE="BOLD" POINT-SIZE="20">STDOUT</FONT></TD></TR>
+		<TR><TD><FONT POINT-SIZE="10">The resulting output lines</FONT></TD></TR>
+		<TR><TD><FONT FACE="MONOSPACE">8<BR/>7<BR/>...</FONT></TD></TR>
+		</TABLE>>];
+
+		command -> runner [label="Implemented By"];
+		runner -> driver [label="Instantiates"];
+		stdin -> driver [label="Reads 1 Line"];
+		driver -> stdout [label="Writes N Lines"];
+	}

data/examples/loadable.rb

@@ -0,0 +1,2 @@
+# Just a file to test that we can correctly load Ruby files from the
+# command-line.

data/examples/string_reverser.rb

@@ -0,0 +1,7 @@
+# A simple processor in its own little file.
+class StringReverser < Wukong::Processor
+  def process line
+    yield line.reverse
+  end
+  register(:string_reverser)
+end

data/lib/hanuman/stage.rb

@@ -25,7 +25,7 @@ module Hanuman
     include Gorillib::Model
     extend  StageClassMethods
 
-    field :label, Symbol
+    field :label, Symbol, :doc => false
   end  
 
   class StageBuilder
@@ -43,7 +43,7 @@ module Hanuman
     end
 
     def build(options = {})      
-      for_class.receive self.serialize.merge(options)
+      for_class.receive self.serialize.merge(options).merge(options[label] || {})
     end
     
     def handle_extra_attributes(attrs)

data/lib/wukong.rb

@@ -4,16 +4,14 @@ require 'multi_json'
 require 'eventmachine'
 require 'log4r'
 
-require 'hanuman'
+require_relative 'hanuman'
+require_relative 'wukong/logger' 
+require_relative 'wukong/processor'
+require_relative 'wukong/dataflow'
+require_relative 'wukong/plugin'
+require_relative 'wukong/driver'
+require_relative 'wukong/runner'
 
-require 'wukong/logger' 
-require 'wukong/processor'
-require 'wukong/dataflow'
-require 'wukong/configuration'
-require 'wukong/widgets'
-require 'wukong/driver'
-require 'wukong/server'
-# require 'wukong/runner'
 
 # The Wukong module will contain all code for Wukong's core (like
 # Processors and Dataflows) as well as all plugins.
@@ -25,7 +23,15 @@ module Wukong
 
   # A common error class intended to be raised by code within Wukong
   # or its plugins.
-  Error = Class.new(StandardError)
+  class Error < StandardError
+    def initialize msg_or_error
+      if msg_or_error.respond_to?(:message) && msg_or_error.respond_to?(:backtrace)
+        super([msg_or_error.message, msg_or_error.backtrace].compact.join("\n"))
+      else
+        super(msg_or_error)
+      end
+    end
+  end
   
   add_shortcut_method_for(:processor, ProcessorBuilder)
   add_shortcut_method_for(:dataflow,  DataflowBuilder)
@@ -34,3 +40,8 @@ end
 
 # Alias module name for shorter namespaces
 Wu = Wukong
+
+require_relative 'wukong/widgets'
+require_relative 'wukong/local'
+
+

data/lib/wukong/dataflow.rb

@@ -1,11 +1,8 @@
 module Wukong
   class DataflowBuilder < Hanuman::GraphBuilder
 
-    def describe desc
-      @description = desc
-    end
-
-    def description
+    def description desc=nil
+      @description = desc if desc
       @description
     end
     

data/lib/wukong/doc_helpers.rb

@@ -0,0 +1,14 @@
+require 'yard'
+require 'wukong'
+require_relative('doc_helpers/processor_handler')
+require_relative('doc_helpers/dataflow_handler')
+require_relative('doc_helpers/field_handler')
+
+module Wukong
+
+  # This module defines several YARD handlers for processing inline
+  # documentation written around Wukong processors and dataflows and
+  # Gorillib fields when using the DSL.
+  module DocHelpers
+  end
+end

data/lib/wukong/doc_helpers/dataflow_handler.rb

@@ -0,0 +1,29 @@
+module Wukong
+  module DocHelpers
+
+    # Handles the Wukong.dataflow syntax.
+    class DataflowHandler < YARD::Handlers::Ruby::ClassHandler
+
+      handles method_call(:dataflow)
+
+      # :nodoc:
+      def base_dataflow_class
+        @base_dataflow_class ||= YARD::CodeObjects::ClassObject.new(namespace, "Wukong::Dataflow")
+      end
+
+      # :nodoc:
+      def process
+        dataflow_name  = statement.parameters.first.jump(:tstring_content, :ident).source
+        class_name     = Gorillib::Inflector.camelize(dataflow_name)
+        dataflow_class = create_class(class_name, base_dataflow_class)
+        dataflow_body  = statement.last.last
+
+        push_state(:owner => dataflow_class, :scope => :class, :namespace => dataflow_class) do
+          parse_block(dataflow_body)
+        end
+      end
+      
+    end
+  end
+end
+

data/lib/wukong/doc_helpers/field_handler.rb

@@ -0,0 +1,91 @@
+module Wukong
+  module DocHelpers
+
+    # Handles the syntax
+    #
+    #     class Foo
+    #       include Gorillib::Model
+    #       field :bar, Integer, :default => 3
+    #     end
+    class FieldHandler < YARD::Handlers::Ruby::ClassHandler
+
+      handles method_call(:field)
+      namespace_only
+
+      def process
+        register(getter)
+        register(setter)
+        namespace.attributes[:instance][field_name] = { :read => getter, :write => setter }
+      end
+      
+      def getter
+        @getter ||= YARD::CodeObjects::MethodObject.new(namespace, field_name, :instance).tap do |method|
+          method.docstring = getter_docstring
+        end
+      end
+
+      def setter
+        @setter ||= YARD::CodeObjects::MethodObject.new(namespace, field_name + '=', :instance).tap do |method|
+          method.docstring = setter_docstring
+        end
+      end
+
+      def getter_docstring
+        doc = "@return [#{field_type}]"
+        doc += " #{field_doc}"                if field_doc
+        doc += " [Default: #{field_default}]" if field_default
+        doc
+      end
+
+      def setter_docstring
+        doc = "@return [#{field_type}]"
+        doc += " #{field_doc}"                if field_doc
+        doc += " [Default: #{field_default}]" if field_default
+        doc
+      end
+
+      def field_name
+        statement.parameters.first.jump(:tstring_content, :ident).source
+      end
+
+      def field_type
+        statement.parameters[1].jump(:string_content, :ident).source
+      end
+      
+      def field_options
+        return @field_options if @field_options
+        @field_options = {}
+        field_options_obj = statement.parameters[2]
+        if field_options_obj
+          keys_and_values = field_options_obj.jump(:assoc)
+          until keys_and_values.empty?
+            obj = keys_and_values.shift
+            if obj.type == :symbol_literal
+              key   = obj.source.to_s.gsub(/^:/,'').to_sym
+              value_obj = keys_and_values.shift
+              if value_obj
+                value = case key
+                        when :doc then value_obj.source.to_s.gsub(/^"/,'').gsub(/"$/,'')
+                        else
+                          value_obj.source
+                        end
+                @field_options[key] = value
+              end
+            end
+          end
+        end
+        @field_options
+      end
+
+      def field_doc
+        field_options[:doc]
+      end
+
+      def field_default
+        field_options[:default]
+      end
+        
+    end
+  end
+end
+

data/lib/wukong/doc_helpers/processor_handler.rb

@@ -0,0 +1,29 @@
+module Wukong
+  module DocHelpers
+
+    # Handles the Wukong.processor syntax.
+    class ProcessorHandler < YARD::Handlers::Ruby::ClassHandler
+
+      handles method_call(:processor)
+
+      # :nodoc:
+      def base_processor_class
+        @base_processor_class ||= YARD::CodeObjects::ClassObject.new(namespace, "Wukong::Processor")
+      end
+
+      # :nodoc:
+      def process
+        processor_name  = statement.parameters.first.jump(:tstring_content, :ident).source
+        class_name      = Gorillib::Inflector.camelize(processor_name)
+        processor_class = create_class(class_name, base_processor_class)
+        processor_body  = statement.last.last
+        
+        push_state(:owner => processor_class, :scope => :class, :namespace => processor_class) do
+          parse_block(processor_body)
+        end
+      end
+      
+    end
+  end
+end
+

data/lib/wukong/driver.rb

@@ -1,12 +1,16 @@
 module Wukong
   module DriverMethods
 
+    attr_accessor :dataflow
+    
+    attr_accessor :settings
+
     def driver
       @driver ||= Driver.new(dataflow)
     end
 
     def lookup(label)
-      raise Wukong::Error.new("could not find definition for #{label}") unless Wukong.registry.registered?(label.to_sym)
+      raise Wukong::Error.new("could not find definition for <#{label}>") unless Wukong.registry.registered?(label.to_sym)
       Wukong.registry.retrieve(label.to_sym)
     end
     
@@ -29,6 +33,12 @@ module Wukong
       dataflow.each(&:setup)
     end
 
+    def finalize_dataflow
+      dataflow.each do |stage|
+        stage.finalize(&driver.advance(stage)) if stage.respond_to?(:finalize)
+      end
+    end
+
     def finalize_and_stop_dataflow
       dataflow.each do |stage|
         stage.finalize(&driver.advance(stage)) if stage.respond_to?(:finalize)

data/lib/wukong/local.rb

@@ -0,0 +1,40 @@
+module Wukong
+
+  # Provides methods for supporting the running of Wukong processors
+  # and dataflows entirely locally, without any frameworks like Hadoop
+  # or Storm.
+  #
+  # This module is actually a plugin for Wukong.
+  module Local
+    include Plugin
+
+    # Configures the given +settings+ object with all settings
+    # specific to Wukong::Local for the given program +name+.
+    #
+    # @param [Configliere::Param] settings the settings to configure
+    # @param [String] program the name of the currently executing program
+    def self.configure settings, program
+      case program
+      when 'wu-local'
+        settings.define :run,      description: "Name of the processor or dataflow to use. Defaults to basename of the given path.", flag: 'r'
+        settings.define :tcp_port, description: "Consume TCP requests on the given port instead of lines over STDIN", type: Integer, flag: 't'
+        
+        settings.define :from, description: "Parse input from given data format (json, tsv, &c.) before processing"
+        settings.define :to,   description: "Convert input to given data format (json, tsv, &c.) before emitting"
+
+        settings.define :consumes, description: "Parse input as instances of given model class before processing", type: Class
+      end
+    end
+
+    # Boots Wukong::Local using the given +settings+ at the given
+    # +root.
+    #
+    # @param [Configliere::Param] settings the settings to use to boot
+    # @param [String] root the root directory to boot in
+    def self.boot(settings, root)
+    end
+    
+  end
+end
+
+require_relative('local/runner')