wukong 3.0.0.pre → 3.0.0.pre2

data/old/wukong.rb

@@ -1,18 +0,0 @@
-require 'wukong/extensions'
-require 'configliere'; Settings.use :define
-require 'wukong/datatypes'
-require 'wukong/periodic_monitor'
-require 'wukong/logger'
-autoload :BadRecord,   'wukong/bad_record'
-autoload :TypedStruct, 'wukong/typed_struct'
-module Wukong
-  autoload :Script,          'wukong/script'
-  autoload :Streamer,        'wukong/streamer'
-  autoload :Store,           'wukong/store'
-  autoload :FilenamePattern, 'wukong/filename_pattern'
-  autoload :Decorator,       'wukong/decorator'
-
-  def self.run mapper, reducer=nil, options={}
-    Wukong::Script.new(mapper, reducer, options).run
-  end
-end

data/old/wukong/and_pig.rb

@@ -1,38 +0,0 @@
-module Enumerable
-  #
-  # Convert an array of values to a string representing it as a pig tuple
-  #
-  def to_pig_tuple
-    '(' + self.join(',') + ')'
-  end
-
-  #
-  # Convert an array of values to a string pig format
-  # see also to_pig_bag
-  #
-  def to_pig *args
-    to_pig_tuple *args
-  end
-
-  #
-  # Convert an array of values to a string representing it as a pig bag
-  #
-  def to_pig_bag
-    '{' + self.map{|*vals| vals.to_pig_tuple}.join(",") + '}'
-  end
-
-  #
-  # Convert a string representing a pig bag into a nested array
-  #
-  def from_pig_bag
-    self.split("),(").map{|t| t.gsub(/[\{\}]/, '').from_pig_tuple} rescue []
-  end
-
-  #
-  # Convert a string representing a pig tuple into an array
-  #
-  def from_pig_tuple
-    self.gsub(/[\(\)]/, '').split(',')
-  end
-
-end

data/old/wukong/bad_record.rb

@@ -1,18 +0,0 @@
-#
-# Easily serialize bad records in-band, for later analysis or to discard if
-# neglectable.
-#
-# You can instantiate this as
-#  success = do_stuff_to record
-#  if ! success
-#    return BadRecord.new("do_stuff_to-failed", record)
-#  end
-#
-class BadRecord < Struct.new(
-    :errors,
-    :record
-    )
-  def initialize errors='', *record_fields
-    super errors, record_fields
-  end
-end

data/old/wukong/datatypes.rb

@@ -1,24 +0,0 @@
-module Wukong
-  RESOURCE_CLASS_MAP = { }
-
-  #
-  # Find the class from its underscored name. Note the klass is non-modularized.
-  # You can also pre-seed RESOURCE_CLASS_MAP
-  #
-  def self.class_from_resource rsrc
-    # This method has been profiled, so don't go making it more elegant unless you're doing same.
-    klass_name = rsrc.to_s
-    return RESOURCE_CLASS_MAP[klass_name] if RESOURCE_CLASS_MAP.include?(klass_name)
-    # kill off all but the non-modularized class name and camelize
-    klass_name.gsub!(/(?:^|_)(.)/){ $1.upcase }
-    begin
-      # convert it to class name
-      klass = klass_name.constantize
-    rescue Exception => e
-      warn "Bogus class name '#{klass_name}'? #{e}"
-      klass = nil
-    end
-    RESOURCE_CLASS_MAP[klass_name] = klass
-  end
-
-end

data/old/wukong/datatypes/enum.rb

@@ -1,127 +0,0 @@
-module Wukong
-  module Datatypes
-    #
-    # Infinity is bigger than any number
-    #
-    #
-    Infinity = 1.0/0
-
-
-    #
-    # A simple enumerated class
-    #
-    #   class MyEnum < Enum
-    #     enumerates :firefox, :safari, :ie, :chrome, :other
-    #   end
-    #   MyEnum[1].to_s # => "safari"
-    #
-    #
-    class Enum
-      attr_accessor            :val
-      class_inheritable_accessor :names
-      def initialize val
-        self.val = val
-      end
-      # MyEnum[val] is sugar for MyEnum.new(val)
-      def self.[] *args
-        new *args
-      end
-
-      # returns the value corresponding to that string representation
-      def index *args
-        # delegate
-        self.class.names.index *args
-      end
-
-      # Representations:
-      def to_i
-        val
-      end
-      def to_s
-        return nil if val.nil?
-        self.class.names[val]
-      end
-
-      def inspect
-        "<#{self.class.to_s} #{to_i} (#{to_s})>"
-      end
-
-      def to_flat
-        to_s #to_i
-      end
-
-      def self.to_sql_str
-        "ENUM('#{names.join("', '")}')"
-      end
-
-      def self.to_pig
-        'chararray'
-      end
-
-      #
-      # Use enumerates to set the class' names
-      #
-      #   class MyEnum < Enum
-      #     enumerates :firefox, :safari, :ie, :chrome, :other
-      #   end
-      #   MyEnum[1].to_s # => "safari"
-      #
-      #
-      def self.enumerates *names
-        self.names = names.map(&:to_s)
-      end
-    end
-
-    #
-    # Note that bin 0 is
-    #
-    class Binned < Enum
-      class_inheritable_accessor :bins, :empty_bin_name
-      @@empty_bin_name = '(none)'
-
-      def bins
-        self.class.bins
-      end
-
-      # FIXME -- doesn't respect a lower bound.
-      def initialize val
-        return super(val) if val.nil?
-        last_top = bins.first
-        bins.each_with_index do |bin_top, idx|
-          return super(idx) if val <= bin_top
-        end
-        return super(bins.length)
-      end
-
-      def self.enumerates *bins
-        options = bins.extract_options!
-        self.bins = bins
-        last_top = bins.shift
-        # bins.unshift bins.first if last_top == -Infinity
-        names = bins.map do |bin_top|
-          name = bin_name last_top, bin_top, options
-          last_top = (last_top.is_a?(Integer) ? bin_top + 1 : bin_top)
-          name
-        end
-        super(*names)
-      end
-
-      #
-      # Bins
-      #
-      def self.bin_name lo_val, hi_val, options = { }
-        # case lo_val
-        # when Integer then lo_val = [lo_val+1, hi_val].compact.min
-        # end
-        case
-        when lo_val == -Infinity then "< #{hi_val}"
-        when hi_val ==  Infinity then "#{lo_val}+"
-        when (lo_val == hi_val)  then    lo_val
-        else                         "#{lo_val} - #{hi_val}"
-        end
-      end
-
-    end
-  end
-end
-

data/old/wukong/datatypes/fake_types.rb

@@ -1,17 +0,0 @@
-module Wukong
-  module Datatypes
-    class Text       < String  ; end unless defined?(Text)
-    class Blob       < String  ; end unless defined?(Blob)
-    class Boolean    < Integer ; end unless defined?(Boolean)
-    class BigDecimal < Float   ; end unless defined?(BigDecimal)
-    class EpochTime  < Integer ; end unless defined?(EpochTime) 
-    class FilePath   < String  ; end unless defined?(FilePath)  
-    class Flag       < String  ; end unless defined?(Flag)      
-    class IPAddress  < String  ; end unless defined?(IPAddress) 
-    class URI        < String  ; end unless defined?(URI)       
-    class Csv        < String  ; end unless defined?(Csv)       
-    class Yaml       < String  ; end unless defined?(Yaml)      
-    class Json       < String  ; end unless defined?(Json)      
-    class Regex      < Regexp  ; end unless defined?(Regex)    
-  end 
-end

data/old/wukong/decorator.rb

@@ -1,28 +0,0 @@
-require 'java'
-
-java_import 'com.cloudera.flume.core.Event'
-java_import 'com.cloudera.flume.core.EventImpl'
-java_import 'com.cloudera.flume.core.EventSinkDecorator'
-
-module Wukong 
-  class Decorator < EventSinkDecorator
-
-    def initialize(mapper, reducer=nil, options={})
-      super(nil)
-      @mapper = mapper.new
-    end
-
-    def append(e)
-      line   = String.from_java_bytes(e.getBody)
-      record = @mapper.recordize(line.chomp)
-      @mapper.process(*record) do |output|
-        processed = output.to_flat.join("\t")
-        event     = EventImpl.new(processed.to_java_bytes, e.getTimestamp, e.getPriority, e.getNanos, e.getHost, e.getAttrs)
-        super event
-      end
-    end
-
-    def run() self ; end
-
-  end
-end

data/old/wukong/encoding/asciize.rb

@@ -1,108 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# http://www.jroller.com/obie/tags/unicode
-# http://www.unicode.org/faq/casemap_charprop.html
-# http://unicode.org/reports/tr10/#Conformance
-# http://intertwingly.net/stories/2009/11/30/asciize.rb
-# http://blog.stevenlevithan.com/archives/javascript-regex-and-unicode
-#
-# http://xregexp.com/tests/unicode.html
-
-class String
-  #
-  # Taken from http://intertwingly.net/stories/2009/11/30/asciize.rb
-  #
-  def asciize(name)
-    if name =~ /[^\x00-\x7F]/
-      # digraphs.  May be culturally sensitive
-      name.gsub! /\xc3\x9f/, 'ss'
-      name.gsub! /\xc3\xa4|a\xcc\x88/, 'ae'
-      name.gsub! /\xc3\xa5|a\xcc\x8a/, 'aa'
-      name.gsub! /\xc3\xa6/, 'ae'
-      name.gsub! /\xc3\xb1|n\xcc\x83/, 'ny'
-      name.gsub! /\xc3\xb6|o\xcc\x88/, 'oe'
-      name.gsub! /\xc3\xbc|u\xcc\x88/, 'ue'
-
-      # latin 1
-      name.gsub! /\xc3[\xa0-\xa5]/, 'a'
-      name.gsub! /\xc3\xa7/, 'c'
-      name.gsub! /\xc3[\xa8-\xab]/, 'e'
-      name.gsub! /\xc3[\xac-\xaf]/, 'i'
-      name.gsub! /\xc3[\xb2-\xb6]|\xc3\xb8/, 'o'
-      name.gsub! /\xc3[\xb9-\xbc]/, 'u'
-      name.gsub! /\xc3[\xbd\xbf]/, 'y'
-
-      # Latin Extended-A
-      name.gsub! /\xc4[\x80-\x85]/, 'a'
-      name.gsub! /\xc4[\x86-\x8d]/, 'c'
-      name.gsub! /\xc4[\x8e-\x91]/, 'd'
-      name.gsub! /\xc4[\x92-\x9b]/, 'e'
-      name.gsub! /\xc4[\x9c-\xa3]/, 'g'
-      name.gsub! /\xc4[\xa4-\xa7]/, 'h'
-      name.gsub! /\xc4[\xa8-\xb1]/, 'i'
-      name.gsub! /\xc4[\xb2-\xb3]/, 'ij'
-      name.gsub! /\xc4[\xb4-\xb5]/, 'j'
-      name.gsub! /\xc4[\xb6-\xb8]/, 'k'
-      name.gsub! /\xc4[\xb9-\xff]|\xc5[\x80-\x82]/, 'l'
-      name.gsub! /\xc5[\x83-\x8b]/, 'n'
-      name.gsub! /\xc5[\x8c-\x91]/, 'o'
-      name.gsub! /\xc5[\x92-\x93]/, 'oe'
-      name.gsub! /\xc5[\x94-\x99]/, 'r'
-      name.gsub! /\xc5[\x9a-\xa2]/, 's'
-      name.gsub! /\xc5[\xa2-\xa7]/, 't'
-      name.gsub! /\xc5[\xa8-\xb3]/, 'u'
-      name.gsub! /\xc5[\xb4-\xb5]/, 'w'
-      name.gsub! /\xc5[\xb6-\xb8]/, 'y'
-      name.gsub! /\xc5[\xb9-\xbe]/, 'z'
-
-      # denormalized diacritics
-      name.gsub! /\xcc[\x80-\xff]|\xcd[\x80-\xaf]/, ''
-    end
-
-    name.gsub /[^\w]+/, '-'
-  end
-
-end
-
-if __FILE__ == $PROGRAM_NAME
-  i18n = "I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xc3\xa0liz\xc3\xa6ti\xc3\xb8n"
-  puts "#{i18n} => #{i18n.asciize}"
-end
-
-# http://www.jroller.com/obie/tags/unicode
-#
-# require 'iconv'
-# require 'unicode'
-# 
-# class String
-#   
-#   def to_ascii
-#     # split in muti-byte aware fashion and translate characters over 127
-#     # and dropping characters not in the translation hash
-#     self.chars.split('').collect { |c| (c[0] <= 127) ? c : translation_hash[c[0]] }.join
-#   end
-#     
-#   def to_url_format
-#     url_format = self.to_ascii
-#     url_format = url_format.gsub(/[^A-Za-z0-9]/, '') # all non-word
-#     url_format.downcase!
-#     url_format
-#   end
-#   
-#   protected
-#   
-#     def translation_hash
-#       @@translation_hash ||= setup_translation_hash      
-#     end
-#     
-#     def setup_translation_hash
-#       accented_chars   = "ÀÁÂÃÄÅÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝàáâãäåçèéêëìíîïñòóôõöøùúûüý"
-#       unaccented_chars = "AAAAAACEEEEIIIIDNOOOOOxOUUUUYaaaaaaceeeeiiiinoooooouuuuy"
-#   
-#       translation_hash = Hash.zip(accented_chars.chars, unaccented_chars.chars)
-#       translation_hash["Æ".chars[0]] = 'AE'
-#       translation_hash["æ".chars[0]] = 'ae'
-#       translation_hash
-#     end
-#     
-# end

data/old/wukong/extensions.rb

@@ -1,16 +0,0 @@
-#
-# These pull in the minimal functionality of the extlib|activesupport family of
-# gems.
-#
-require 'extlib/blank'
-require 'extlib/class'
-require 'wukong/extensions/enumerable'
-require 'wukong/extensions/symbol'
-require 'wukong/extensions/hash'
-require 'wukong/extensions/hash_like'
-require 'wukong/extensions/array'
-require 'wukong/extensions/struct'
-require 'wukong/extensions/module'
-require 'wukong/extensions/string'
-require 'wukong/extensions/date_time'
-require 'wukong/extensions/emittable'

data/old/wukong/extensions/array.rb

@@ -1,18 +0,0 @@
-class Array
-  #
-  # The following is taken in whole from the extlib gem. Thanks y'all.
-  #
-
-  # Extracts options from a set of arguments. Removes and returns the last
-  # element in the array if it's a hash, otherwise returns a blank hash.
-  #
-  #   def options(*args)
-  #     args.extract_options!
-  #   end
-  #
-  #   options(1, 2)           # => {}
-  #   options(1, 2, :a => :b) # => {:a=>:b}
-  def extract_options!
-    last.is_a?(::Hash) ? pop : {}
-  end
-end

data/old/wukong/extensions/blank.rb

@@ -1,93 +0,0 @@
-#
-# This is taken in whole from the extlib gem. Thanks y'all.
-#
-
-class Object
-  ##
-  # Returns true if the object is nil or empty (if applicable)
-  #
-  #   [].blank?         #=>  true
-  #   [1].blank?        #=>  false
-  #   [nil].blank?      #=>  false
-  #
-  # @return [TrueClass, FalseClass]
-  #
-  # @api public
-  def blank?
-    nil? || (respond_to?(:empty?) && empty?)
-  end unless method_defined?(:blank?)
-end # class Object
-
-class Numeric
-  ##
-  # Numerics are never blank
-  #
-  #   0.blank?          #=>  false
-  #   1.blank?          #=>  false
-  #   6.54321.blank?    #=>  false
-  #
-  # @return [FalseClass]
-  #
-  # @api public
-  def blank?
-    false
-  end unless method_defined?(:blank?)
-end # class Numeric
-
-class NilClass
-  ##
-  # Nil is always blank
-  #
-  #   nil.blank?        #=>  true
-  #
-  # @return [TrueClass]
-  #
-  # @api public
-  def blank?
-    true
-  end unless method_defined?(:blank?)
-end # class NilClass
-
-class TrueClass
-  ##
-  # True is never blank.
-  #
-  #   true.blank?       #=>  false
-  #
-  # @return [FalseClass]
-  #
-  # @api public
-  def blank?
-    false
-  end unless method_defined?(:blank?)
-end # class TrueClass
-
-class FalseClass
-  ##
-  # False is always blank.
-  #
-  #   false.blank?      #=>  true
-  #
-  # @return [TrueClass]
-  #
-  # @api public
-  def blank?
-    true
-  end unless method_defined?(:blank?)
-end # class FalseClass
-
-class String
-  ##
-  # Strips out whitespace then tests if the string is empty.
-  #
-  #   "".blank?         #=>  true
-  #   "     ".blank?    #=>  true
-  #   " hey ho ".blank? #=>  false
-  #
-  # @return [TrueClass, FalseClass]
-  #
-  # @api public
-  def blank?
-    strip.empty?
-  end unless method_defined?(:blank?)
-end # class String