wukong 3.0.0.pre → 3.0.0.pre2

data/lib/wukong/model/flatpack_parser/flat.rb

@@ -0,0 +1,60 @@
+class String
+  def match_all regex
+    self.to_enum(:scan, regex).map {Regexp.last_match}
+  end
+end 
+
+module Wukong
+  module FlatPack
+
+    # Creates a 'simple' token from the supplied string
+    # and position.
+    def self.simple_token_from_string(str, position)
+      token_pieces = str.match(Language::NAMED_SIMPLE_TYPE_RE)
+      t = Flat::Tokens.token_for_indicator(token_pieces[:type])
+      t.position = position
+      t.length = token_pieces[:length].nil? ? nil : token_pieces[:length].to_i
+      t.modifier = token_pieces[:modifier]
+      return t
+    end
+
+    # Creates a fixed point token. Strict input formatting is
+    # enforced if the strict param is true.
+    def self.fixed_point_token_from_string(str, position, strict=true)
+      float_pieces = str.match(Language::NAMED_FIXED_POINT_RE)
+      t = Flat::Tokens::FixedPointToken.new
+      t.position = position
+      t.strict = strict
+      t.power = float_pieces[:power].nil? ? nil : float_pieces[:power].to_i
+      t.length = float_pieces[:length].to_i
+      return t
+    end
+
+    # Validates the supplied format string
+    # and creates a parser from it.
+    def self.create_parser(str, delimiter_width=0, strict_fixed_point=true)
+      return nil unless Language.string_in_lang str
+      lang = []
+      str.match_all(Language::CAPTURE_TOKEN_RE).each do |match|
+        token_str = match[0]
+        case token_str
+        when Language::TOTAL_SIMPLE_TYPE_RE
+          lang << simple_token_from_string(token_str, match.begin(0))
+        when Language::TOTAL_FIXED_POINT_RE
+          lang << fixed_point_token_from_string(token_str, match.begin(0), strict_fixed_point)
+        when Language::TOTAL_DATE_RE
+          date_match = token_str.match(Language::NAMED_DATE_RE)
+          #TODO: Implement
+        end
+        if delimiter_width != 0
+          t = Flat::Tokens::IgnoreToken.new
+          t.position = -1
+          t.length = delimiter_width
+          lang << t
+        end
+      end
+      lang = lang[0..-2] if delimiter_width != 0 #pop off the delimiter on the end
+      return Flat::Parser.new(lang)
+    end
+  end
+end

data/lib/wukong/model/flatpack_parser/flatpack.rb

@@ -0,0 +1,4 @@
+require 'flat'
+require 'lang'
+require 'parser'
+require 'tokens'

data/lib/wukong/model/flatpack_parser/lang.rb

@@ -0,0 +1,46 @@
+module Wukong
+  module FlatPack
+    module Language
+      
+      #language definition
+      SIMPLE_TYPES = %w{i f s b _}
+      SIMPLE_TYPE_RE = "[#{SIMPLE_TYPES.join}]"
+
+      MODIFIERS = %w{+ *}
+      MODIFIER_RE = "[#{MODIFIERS.join}]"
+
+      SIMPLE_TOKEN_RE = "#{SIMPLE_TYPE_RE}(?:#{MODIFIER_RE}|[0-9]+)?"
+
+      DATE_TYPES = %w{a A b B c d H I j m M p S U w W x X Y Z}
+      DATE_TYPES_RE = "[#{DATE_TYPES.join} ]" # the extra space is supposed to be there
+      DATE_TOKEN_RE = "%#{DATE_TYPES_RE}*%" 
+
+      FIXED_POINT_TYPE = 'D'
+      FIXED_POINT_SEP = 'e'
+      FIXED_POINT_TOKEN_RE = "#{FIXED_POINT_TYPE}\\d+(?:#{FIXED_POINT_SEP}\\d+)?"
+
+      TOKENS = [SIMPLE_TOKEN_RE, DATE_TOKEN_RE, FIXED_POINT_TOKEN_RE]
+      TOKEN_RE =  "#{TOKENS.join('|')}"
+      CAPTURE_TOKEN_RE =  /(#{TOKENS.join('|')})/
+
+      LANGUAGE_RE = /^(?:(#{TOKEN_RE}) *)+$/
+
+      #total regexes, i.e. regexes that must match the whole string
+      TOTAL_SIMPLE_TYPE_RE = /^#{SIMPLE_TOKEN_RE}$/
+      TOTAL_FIXED_POINT_RE = /^#{FIXED_POINT_TOKEN_RE}$/
+      TOTAL_DATE_RE = /^#{DATE_TOKEN_RE}$/
+
+      #named regexes used for parsing tokens
+      NAMED_SIMPLE_TYPE_RE = /(?<type>#{SIMPLE_TYPE_RE})(?:(?<length>[0-9]+)|(?<modifier>#{MODIFIER_RE}))?/
+      NAMED_FIXED_POINT_RE = /#{FIXED_POINT_TYPE}(?<length>\d+)(?:#{FIXED_POINT_SEP}(?<power>\d+))?/
+      NAMED_DATE_RE = /%(?<format>#{DATE_TYPES_RE})%/
+
+      # Returns true if the supplied string is in
+      # Flat's formatting language, as determined
+      # by the LANGUAGE_RE regex.
+      def self.string_in_lang(str)
+        return (not (str =~ LANGUAGE_RE).nil?)
+      end
+    end
+  end
+end

data/lib/wukong/model/flatpack_parser/parser.rb

@@ -0,0 +1,55 @@
+module Wukong
+  module FlatPack
+    class Parser
+      attr_accessor :re
+      attr_accessor :lang
+
+      def initialize(lang)
+        @lang = lang
+        @re = re_from_language @lang
+      end
+
+      # returns true if the supplied string is in the parser's language
+      def string_in_lang? str
+        return (not (str =~ @re).nil?)
+      end
+
+      # Creates a regular expression from the 
+      # supplied language
+      def re_from_language lang
+        regex = "^"
+        lang.each do |token|
+          regex += "(#{token.re})"
+        end
+        regex += "$"
+        return Regexp.new(regex)
+      end
+
+      def parse(str,trim=false)
+        return nil unless string_in_lang? str
+        result = []
+        str.match(@re)[1..-1].each_with_index do |val,index|
+          token = lang[index].translate(val)
+          if trim and token.is_a?(String)
+            token.strip!
+          end
+          result << token
+        end
+        return result - [:ignore]
+      end
+
+      def file_to_tsv(in_filename,out_filename,trim=true)
+        infile =  File.open(in_filename,'r')
+        outfile = File.open(out_filename,'a')
+        infile.each_line do |line|
+          outfile.write(line_to_tsv(line,trim))
+        end
+      end
+
+      def line_to_tsv(line,trim=true)
+        fields = parse(line,trim)
+        return fields.join("\t") + "\n"
+      end
+    end
+  end
+end

data/lib/wukong/model/flatpack_parser/tokens.rb

@@ -0,0 +1,130 @@
+module Wukong
+  module FlatPack
+    module Tokens
+      TOKEN_CLASSES = {}
+      
+      def self.token_for_indicator(indicator)
+        return TOKEN_CLASSES[indicator].new
+      end
+      
+      class Token
+        attr_accessor :position
+        attr_accessor :length
+        attr_accessor :indicator
+
+        def self.indicator= indicator
+          TOKEN_CLASSES[indicator] = self
+          @indicator = indicator
+        end
+      end
+
+      class FixedPointToken < Token
+        attr_accessor :power
+        attr_accessor :strict
+
+        self.indicator = 'D'
+
+        #TODO: Allow negative powers
+        def re
+          strict ? "(?:(?:\\+|-)\\d{#{@length-1}}|\\d{#{@length}})" : ".{#{@length}}"
+        end
+        
+        def translate str
+          return nil if str.strip == ""
+          base = str.to_f
+          return base / (10**@power)
+        end
+      end
+
+      class BasicToken < Token
+        attr_accessor :modifier
+     
+        def re token= '.'
+          if not @length.nil?
+            return "#{token}{#{@length}}"
+          elsif not @modifier.nil?
+            return "#{token}#{@modifier}"
+          else
+            return token
+          end
+        end
+
+      end
+      
+      class IntToken < BasicToken
+        self.indicator = 'i'
+        RE = '(?:\+|-)?\\d'
+
+        def re
+          if not @length.nil?
+            return "(?:(?:\\+|-)\\d{#{@length-1}}|\\d{#{@length}})"
+          elsif not @modifier.nil?
+            return "#{RE}#{@modifier}"
+          else
+            return RE
+          end
+        end
+
+        def translate str
+          return Integer(str)
+        rescue ArgumentError => err
+          return str.to_i
+        end
+      end
+
+      class StringToken < BasicToken
+        self.indicator = 's'
+        
+        def translate str
+          return str
+        end
+      end
+
+      class  FloatToken < BasicToken
+        self.indicator = 'f'
+        #TODO: Implement floats
+
+        def get_re
+          #TODO: Implement 
+        end
+
+        def translate
+          #TODO: Implement
+        end
+      end
+     
+      class BoolToken < BasicToken
+        self.indicator = 'b'
+        TRUE_TOKENS = ['t','y','1'] 
+        FALSE_TOKENS = ['f','n','0']
+        
+        #TODO: Add back multi-char options and think through allowing padding
+        #TODO: Allow users to override true and false
+
+        def re
+          return "(?:#{(TRUE_TOKENS + TRUE_TOKENS.map {|c| c.upcase} +
+                       FALSE_TOKENS + FALSE_TOKENS.map{|c| c.upcase}).join('|')})"
+        end
+
+        def translate str
+          if TRUE_TOKENS.include?(str.downcase)
+            return true
+          elsif FALSE_TOKENS.include?(str.downcase)
+            return false
+          else
+            return nil
+          end
+        end
+      end
+
+      class IgnoreToken < BasicToken
+        self.indicator = '_'
+
+        # ignore symbols are removed from the final output
+        def translate str
+          return :ignore
+        end
+      end
+    end
+  end
+end

data/lib/wukong/processor.rb

@@ -1,142 +1,88 @@
-require 'vayacondios-client'
-
-Settings.define :monitor_interval, :default => 50_000, :type => Integer
+require 'log4r'
 
 module Wukong
-  class ProcessorError < StandardError ; end
-
-  class Processor < Hanuman::Action
-    include Hanuman::IsOwnInputSlot
-    include Hanuman::IsOwnOutputSlot
+  class ProcessorBuilder < Hanuman::StageBuilder
+    def namespace(*args)
+      args.first.is_a?(Class) ? args.first : Wukong::Processor
+    end
+  end
 
-    field :name, Symbol, :default => ->{ self.class.handle }
-    field :count, Integer, doc: 'Number of records seen this run', default: 0
+  # The Processor is the basic unit of computation in Wukong.  A
+  # processor can be thought of as an arbitrary function that takes
+  # certain inputs and produces certain (or no) outputs.
+  #
+  # A Processor can be written and tested purely in Ruby and on your
+  # local machine.  You can glue processors together
+  class Processor < Hanuman::Stage
+    
+    field :action,   Whatever
+    field :log,      Whatever, :default => -> { log = Log4r::Logger.new(self.class.to_s) ; log.outputters = Log4r::StdoutOutputter.new('stdout', formatter: Log4r::PatternFormatter.new(pattern: "%d [%l] %c: %m")) ; log }
+    field :notifier, Vayacondios::NotifierFactory, :default => Vayacondios.default_notifier
 
-    # override this in your subclass
-    def process(record)
+    def self.describe desc
+      @description = desc
     end
 
-    # passes a record on down the line
-    def emit(record)
-      self.count += 1
-      if (count % Settings.monitor_interval.to_i == 0)
-        log.info "emit\t%-23s\t%-47s\t%s" % [self.class, self.inspect, record.inspect]
-      end
-      output.process(record)
-    rescue Wukong::ProcessorError
-      raise
-    rescue StandardError => err
-      next_block = output.name rescue "(bad stage)"
-      log.warn "#{self}: error emitting #{next_block}: #{err.message}"
-      raise Wukong::ProcessorError, err.message, err.backtrace
+    def self.description
+      @description
     end
 
-    def bad_record(*args)
-      BadRecord.make(*args)
+    def self.consumes label
     end
 
-    def self.register_processor(name=nil, &block)
-      register_action(name, &block)
+    def self.produces label
     end
+
+    # This is a placeholder method intended to be overridden
+    def perform_action(*args) ; end 
     
-    include Vayacondios::Notifications
-    
-    class_attribute :log
-    self.log = Log
-    
-    config :error_handler, Vayacondios::NotifierFactory, :default => ->{ Vayacondios::NotifierFactory.receive(type: 'log', log: self.log) }
-    
-    def bad_record(record, options = {})
-      error_handler.notify(record, options.merge(level: 'error'))
+    # The action attribute is turned into the perform action method
+    def receive_action(action)
+      self.define_singleton_method(:perform_action, &action)
     end
-  end
 
-  class AsIs < Processor
-    # accepts records, emits as-is
-    def process(*args)
-      emit(*args)
+    # Valid notifier types are currently :http or :log
+    # This processor's log is passed to vayacondios
+    def receive_notifier(type)
+      if type.is_a?(Hash)
+        @notifier = Vayacondios::NotifierFactory.receive({type: 'log'}.merge(type))
+      else
+        @notifier = Vayacondios::NotifierFactory.receive(type: type, log: log)
+      end
     end
-    register_processor
-  end
-
-  class Null < Processor
-    self.register_processor
 
-    # accepts records, emits none
-    def process(*)
-      # ze goggles... zey do nussing!
+    # Send information to Vayacondios; data goes in, the right thing happens
+    def notify(topic, cargo)
+      notifier.notify(topic, cargo)
     end
-  end
 
-  #
-  # Foreach calls a block on every record, and depends on the block to call
-  # emit. You can emit one record, many records, or no records, and with any
-  # contents. If you'll always emit exactly one record out per record in,
-  # you may prefer Wukong::Widget::Map.
-  #
-  # @example regenerate a wordbag with counts matching the original
-  #   foreach{|rec| rec.count.times{ emit(rec.word) } }
-  #
-  # @see Project
-  # @see Map
-  class Foreach < Processor
-    self.register_processor
-
-    # @param [Proc] proc used for body of process method
-    # @yield ... or supply it as a &block arg.
-    def initialize(prc=nil, &block)
-      prc ||= block or raise "Please supply a proc or a block to #{self.class}.new"
-      define_singleton_method(:process, prc)
-    end
-
-    def self.make(workflow, *args, &block)
-      obj = new(*args, &block)
-      workflow.add_stage obj
-      obj
+    # This method is called after the processor class has been instantiated
+    # but before any records are given to it to process
+    def setup
     end
-  end
-
-  #
-  # Evaluates the block and emits the result if non-nil
-  #
-  # @example turn a record into a tuple
-  #   map{|rec| rec.attributes.values }
-  #
-  # @example pass along first matching term, drop on the floor otherwise
-  #   map{|str| str[/\b(love|hate|happy|sad)\b/] }
-  #
-  class Map < Processor
-    self.register_processor
-    attr_reader :blk
 
-    # @param [Proc] proc to delegate for call
-    # @yield if proc is omitted, block must be supplied
-    def initialize(blk=nil, &block)
-      @blk = blk || block or raise "Please supply a proc or a block to #{self.class}.new"
+    # This method is called once per record
+    # Override this in your subclass
+    def process(record, &emit)
+      yield record
     end
 
-    def process(*args)
-      result = blk.call(*args)
-      emit result unless result.nil?
+    # This method is called to signal the last record has been
+    # received but that further processing may still be done, events
+    # still be yielded, &c.
+    #
+    # This can be used within an aggregating processor (like a reducer
+    # in a map/reduce job) to start processing the final aggregate of
+    # records since the "last record" has already been received.
+    def finalize
     end
 
-    def self.make(workflow, *args, &block)
-      obj = new(*args, &block)
-      workflow.add_stage obj
-      obj
+    # This method is called after all records have been passed.  It
+    # signals that processing should stop.
+      
+    # This method is called after all records have been processed
+    def stop
     end
-  end
 
-  #
-  # Flatten emits each item in an enumerable as its own record
-  #
-  # @example turn a document into all its words
-  #   input > map{|line| line.split(/\W+/) } > flatten > output
-  class Flatten < Processor
-    self.register_processor
-
-    def process(iter)
-      iter.each{|*args| emit(*args) }
-    end
   end
 end