ul-wukong 4.1.0

data/lib/wukong/model/flatpack_parser/parser.rb

@@ -0,0 +1,55 @@
+module Wukong
+  module FlatPack
+    class Parser
+      attr_accessor :re
+      attr_accessor :lang
+
+      def initialize(lang)
+        @lang = lang
+        @re = re_from_language @lang
+      end
+
+      # returns true if the supplied string is in the parser's language
+      def string_in_lang? str
+        return (not (str =~ @re).nil?)
+      end
+
+      # Creates a regular expression from the 
+      # supplied language
+      def re_from_language lang
+        regex = "^"
+        lang.each do |token|
+          regex += "(#{token.re})"
+        end
+        regex += "$"
+        return Regexp.new(regex)
+      end
+
+      def parse(str,trim=false)
+        return nil unless string_in_lang? str
+        result = []
+        str.match(@re)[1..-1].each_with_index do |val,index|
+          token = lang[index].translate(val)
+          if trim and token.is_a?(String)
+            token.strip!
+          end
+          result << token
+        end
+        return result - [:ignore]
+      end
+
+      def file_to_tsv(in_filename,out_filename,trim=true)
+        infile =  File.open(in_filename,'r')
+        outfile = File.open(out_filename,'a')
+        infile.each_line do |line|
+          outfile.write(line_to_tsv(line,trim))
+        end
+      end
+
+      def line_to_tsv(line,trim=true)
+        fields = parse(line,trim)
+        return fields.join("\t") + "\n"
+      end
+    end
+  end
+end

data/lib/wukong/model/flatpack_parser/tokens.rb

@@ -0,0 +1,130 @@
+module Wukong
+  module FlatPack
+    module Tokens
+      TOKEN_CLASSES = {}
+      
+      def self.token_for_indicator(indicator)
+        return TOKEN_CLASSES[indicator].new
+      end
+      
+      class Token
+        attr_accessor :position
+        attr_accessor :length
+        attr_accessor :indicator
+
+        def self.indicator= indicator
+          TOKEN_CLASSES[indicator] = self
+          @indicator = indicator
+        end
+      end
+
+      class FixedPointToken < Token
+        attr_accessor :power
+        attr_accessor :strict
+
+        self.indicator = 'D'
+
+        #TODO: Allow negative powers
+        def re
+          strict ? "(?:(?:\\+|-)\\d{#{@length-1}}|\\d{#{@length}})" : ".{#{@length}}"
+        end
+        
+        def translate str
+          return nil if str.strip == ""
+          base = str.to_f
+          return base / (10**@power)
+        end
+      end
+
+      class BasicToken < Token
+        attr_accessor :modifier
+     
+        def re token= '.'
+          if not @length.nil?
+            return "#{token}{#{@length}}"
+          elsif not @modifier.nil?
+            return "#{token}#{@modifier}"
+          else
+            return token
+          end
+        end
+
+      end
+      
+      class IntToken < BasicToken
+        self.indicator = 'i'
+        RE = '(?:\+|-)?\\d'
+
+        def re
+          if not @length.nil?
+            return "(?:(?:\\+|-)\\d{#{@length-1}}|\\d{#{@length}})"
+          elsif not @modifier.nil?
+            return "#{RE}#{@modifier}"
+          else
+            return RE
+          end
+        end
+
+        def translate str
+          return Integer(str)
+        rescue ArgumentError => err
+          return str.to_i
+        end
+      end
+
+      class StringToken < BasicToken
+        self.indicator = 's'
+        
+        def translate str
+          return str
+        end
+      end
+
+      class  FloatToken < BasicToken
+        self.indicator = 'f'
+        #TODO: Implement floats
+
+        def get_re
+          #TODO: Implement 
+        end
+
+        def translate
+          #TODO: Implement
+        end
+      end
+     
+      class BoolToken < BasicToken
+        self.indicator = 'b'
+        TRUE_TOKENS = ['t','y','1'] 
+        FALSE_TOKENS = ['f','n','0']
+        
+        #TODO: Add back multi-char options and think through allowing padding
+        #TODO: Allow users to override true and false
+
+        def re
+          return "(?:#{(TRUE_TOKENS + TRUE_TOKENS.map {|c| c.upcase} +
+                       FALSE_TOKENS + FALSE_TOKENS.map{|c| c.upcase}).join('|')})"
+        end
+
+        def translate str
+          if TRUE_TOKENS.include?(str.downcase)
+            return true
+          elsif FALSE_TOKENS.include?(str.downcase)
+            return false
+          else
+            return nil
+          end
+        end
+      end
+
+      class IgnoreToken < BasicToken
+        self.indicator = '_'
+
+        # ignore symbols are removed from the final output
+        def translate str
+          return :ignore
+        end
+      end
+    end
+  end
+end

data/lib/wukong/plugin.rb

@@ -0,0 +1,48 @@
+module Wukong
+
+  # An array of known plugins.
+  PLUGINS = []
+
+  # Asks each loaded plugin to configure the given +settings+ for the
+  # given +program_name+.
+  #
+  # @param [Configliere::Param] settings the settings to be configured by each plugin
+  # @param [String] program_name the name of the currently executing program
+  def self.configure_plugins(settings, program_name)
+    PLUGINS.each do |plugin|
+      plugin.configure(settings, program_name) if plugin.respond_to?(:configure)
+    end
+  end
+
+  # Asks each loaded plugin to boot itself from the given +settings+
+  # in the given +root+ directory.
+  #
+  # @param [Configliere::Param] settings the settings for each plugin to boot from
+  # @param [String] root the root directory the plugins are booting in
+  def self.boot_plugins(settings, root)
+    PLUGINS.each do |plugin|
+      plugin.boot(settings, root) if plugin.respond_to?(:boot)
+    end
+  end
+
+  # Include this module in your own class or module to have it
+  # register itself as a Wukong plugin.
+  #
+  # Your class or module must define the following methods:
+  #
+  # * `configure` called with a (pre-resolved) Configliere::Param argument and the basename of the running program
+  # * `boot` called with a (resolved) Configliere::Param argument and the current working directory of the running program, reacts to any settings as necessary
+  #
+  # Subclasses of Wukong::Runner will automatically load and boot each
+  # plugin.
+  module Plugin
+    # :nodoc:
+    def self.included mod
+      PLUGINS << mod unless PLUGINS.include?(mod)
+    end
+  end
+  
+end
+
+    
+    

data/lib/wukong/processor.rb

@@ -0,0 +1,110 @@
+module Wukong
+  class ProcessorBuilder < Hanuman::StageBuilder
+    def namespace(*args)
+      args.first.is_a?(Class) ? args.first : Wukong::Processor
+    end
+  end
+
+  # The Processor is the basic unit of computation in Wukong.  A
+  # processor can be thought of as an arbitrary function that takes
+  # certain inputs and produces certain (or no) outputs.
+  #
+  # A Processor can be written and tested purely in Ruby and on your
+  # local machine.  You can glue processors together
+  class Processor < Hanuman::Stage
+    include Logging
+    
+    field :action, Whatever, :doc => false
+
+    class << self
+
+      def description desc=nil
+        @description = desc if desc
+        @description
+      end
+            
+      def configure(settings)
+        settings.description = description if description
+        fields.each_pair do |name, field|
+          next if field.doc == false || field.doc.to_s == 'false'
+          next if [:log].include?(name)
+          field_props = {}.tap do |props|
+            props[:description] = field.doc unless field.doc == "#{name} field"
+            field_type = (field.type.respond_to?(:product) ? field.type.product : field.type)
+            configliere_type = case field_type
+            when String                then nil
+            when TrueClass, FalseClass then :boolean
+            else field_type
+            end
+            
+            props[:type]        = configliere_type if configliere_type
+            props[:default]     = field.default if field.default
+          end
+          existing_value = settings[name]
+          settings.define(name, field_props)
+          settings[name] = existing_value unless existing_value.nil?
+        end
+      end
+
+    end
+        
+    # When instantiated with a block, the block will replace this
+    # method.
+    #
+    # @param [Array<Object>] args
+    # @yield record a record that might be yielded by the block
+    # @yieldparam [Object] record the yielded record
+    def perform_action(*args)
+    end 
+    
+    # :nodoc:
+    #
+    # The action attribute is turned into the perform action method.
+    #
+    # @param [Proc] action
+    def receive_action(action)
+      self.define_singleton_method(:perform_action, &action)
+    end
+
+    # This method is called after the processor class has been
+    # instantiated but before any records are given to it to process.
+    #
+    # Override this method in your subclass.
+    def setup
+    end
+
+    # This method is called once per record.
+    #
+    # Override this method in your subclass.
+    #
+    # @param [Object] record
+    # @yield record the record you want to yield
+    # @yieldparam [Object] record the yielded record
+    def process(record, &emit)
+      yield record
+    end
+
+    # This method is called to signal the last record has been
+    # received but that further processing may still be done, events
+    # still be yielded, &c.
+    #
+    # This can be used within an aggregating processor (like a reducer
+    # in a map/reduce job) to start processing the final aggregate of
+    # records since the "last record" has already been received.
+    #
+    # Override this method in your subclass
+    #
+    # @yield record the record you want to yield
+    # @yieldparam [Object] record the yielded record
+    def finalize
+    end
+
+    # This method is called after all records have been passed.  It
+    # signals that processing should stop.
+    #
+    # Override this method in your subclass.
+    def stop
+    end
+
+  end
+end

data/lib/wukong/rake_helper.rb

@@ -0,0 +1,6 @@
+require 'rake'
+require 'wukong'
+
+task :environment => [] do
+  Wukong::Runner.run
+end

data/lib/wukong/runner.rb

@@ -0,0 +1,169 @@
+require_relative("runner/code_loader")
+require_relative("runner/deploy_pack_loader")
+require_relative("runner/boot_sequence")
+require_relative("runner/command_runner")
+
+module Wukong
+
+  # A base class which handles
+  #
+  # * requiring any necessary code like deploy packs or code from command-line arguments
+  # * having all plugins configure settings as necessary
+  # * resolving settings
+  # * having all plugins boot from now resolved settings
+  # * parsing command-line arguments
+  # * instantiating and handing over control to a driver which runs the actual code
+  class Runner
+    
+    include Logging
+    include CodeLoader
+    include DeployPackLoader
+    include BootSequence
+    include CommandRunner
+
+    # The settings object that will be configured and booted from.
+    # All plugins will configure this object.
+    attr_accessor :settings
+
+    # Create a new Runner with the given +settings+.
+    #
+    # Uses an empty Configliere::Param object if no +settings+ are
+    # given.
+    #
+    # @param [Configliere::Param] settings
+    def initialize settings=Configliere::Param.new
+      self.settings = settings
+    end
+    
+    # Instantiates a new Runner and boot it up.
+    #
+    # Will rescue any Wukong::Error with a logged error message and
+    # exit.
+    def self.run(settings=Configliere::Param.new)
+      begin
+        new(settings).boot!
+      rescue Wukong::Error => e
+        die(e.message, 127)
+      end
+    end
+
+    # The parsed command-line arguments.
+    #
+    # Will raise an error if +boot+ hasn't been called yet.
+    #
+    # @return [Array<String>]
+    def args
+      settings.rest
+    end
+
+    # The root directory we should consider ourselves to be running
+    # in.
+    #
+    # Defaults to the root directory of a deploy pack if we're running
+    # inside one, else just returns `Dir.pwd`.
+    #
+    # @return [String]
+    def root
+      in_deploy_pack? ? deploy_pack_dir : Dir.pwd
+    end
+
+    # Convenience method for setting the usage message of a Runner.
+    #
+    # @param [String, nil] msg set the usage message
+    # @return [String] the usage message
+    def self.usage msg=nil
+      return @usage unless msg
+      @usage = msg
+    end
+    
+    # Convenience method for setting the description message of a Runner.
+    #
+    # @param [String, nil] msg set the description message
+    # @return [String] the description message
+    def self.description msg=nil
+      return @description unless msg
+      @description = msg
+    end
+
+    # Kill this process with the given error `message` and exit
+    # `code`.
+    #
+    # @param [String] message
+    # @param [Integer] code.
+    def self.die(message=nil, code=127)
+      log.error(message) if message
+      exit(code)
+    end
+
+    # Return the name of the program this Runner is running.
+    #
+    # This is passed to plugins which can configure settings
+    # appropriately.  Defaults to the name of the currently running
+    # process.
+    #
+    # @return [String]
+    def program_name
+      @program_name || File.basename($0)
+    end
+
+    # Explicitly set the name of the program this Runner is running.
+    #
+    # This is useful for unit tests in which the name of the currently
+    # running process may be different from the runner command being
+    # tested (`rspec` vs. `wu-local`).
+    #
+    # @param [String] name
+    def program_name= name
+      @program_name = name
+    end
+
+    # Return the usage message for this runner.
+    #
+    # @return [String] the usage message
+    def usage
+      ["usage: #{program_name} [ --param=val | --param | -p val | -p ]", self.class.usage].compact.join(' ')
+    end
+
+    # Return the description text for this runner.
+    #
+    # @return [String] the description text
+    def description
+      self.class.description
+    end
+
+    # Is there a processor or dataflow registered with the given
+    # `name`?
+    #
+    # @param [String] name
+    # @return [true, false]
+    def registered? name
+      name && Wukong.registry.registered?(name.to_sym)
+    end
+
+    # Retrieve the dataflow registered under a given `name`.
+    #
+    # @param [String,Symbol] name
+    # @return [Wukong::Processor, Wukong::Dataflow, nil]
+    def dataflow_class_for(name)
+      builder = (Wukong.registry.retrieve(name.to_sym) or return)
+      builder.for_class
+    end
+
+    # Is the given `name` a registered as a processor?
+    #
+    # @param [String,Symbol] name
+    # @return [true, false]
+    def processor?(name)
+      registered?(name) && dataflow_class_for(name).ancestors.include?(Wukong::Processor)
+    end
+
+    # Is the given `name` a registered as a dataflow?
+    #
+    # @param [String,Symbol] name
+    # @return [true, false]
+    def dataflow?(name)
+      registered?(name) && dataflow_class_for(name).ancestors.include?(Wukong::Dataflow)
+    end
+
+  end    
+end