mrflip-wukong 0.1.0

data/lib/wukong/datatypes.rb

@@ -0,0 +1,24 @@
+module Wukong
+  RESOURCE_CLASS_MAP = { }
+
+  #
+  # Find the class from its underscored name. Note the klass is non-modularized.
+  # You can also pre-seed RESOURCE_CLASS_MAP
+  #
+  def self.class_from_resource rsrc
+    # This method has been profiled, so don't go making it more elegant unless you're doing same.
+    klass_name = rsrc.to_s
+    return RESOURCE_CLASS_MAP[klass_name] if RESOURCE_CLASS_MAP.include?(klass_name)
+    # kill off all but the non-modularized class name and camelize
+    klass_name.gsub!(/(?:^|_)(.)/){ $1.upcase }
+    begin
+      # convert it to class name
+      klass = klass_name.constantize
+    rescue Exception => e
+      warn "Bogus class name '#{klass_name}'? #{e}"
+      klass = nil
+    end
+    RESOURCE_CLASS_MAP[klass_name] = klass
+  end
+
+end

data/lib/wukong/datatypes/enum.rb

@@ -0,0 +1,123 @@
+module Wukong
+  module Datatypes
+    #
+    # Infinity is bigger than any number
+    #
+    #
+    Infinity = 1.0/0
+
+
+    #
+    # A simple enumerated class
+    #
+    #   class MyEnum < Enum
+    #     enumerates :firefox, :safari, :ie, :chrome, :other
+    #   end
+    #   MyEnum[1].to_s # => "safari"
+    #
+    #
+    class Enum
+      attr_accessor            :val
+      class_inheritable_accessor :names
+      def initialize val
+        self.val = val
+      end
+      # MyEnum[val] is sugar for MyEnum.new(val)
+      def self.[] *args
+        new *args
+      end
+      def to_i
+        val
+      end
+      def to_s
+        return nil if val.nil?
+        self.class.names[val]
+      end
+      def inspect
+        "<#{self.class.to_s} #{to_i} (#{to_s})>"
+      end
+      # returns the value corresponding to that string representation
+      def index *args
+        # delegate
+        self.class.names.index *args
+      end
+      def to_flat
+        to_s #to_i
+      end
+
+      #
+      # Use enumerates to set the class' names
+      #
+      #   class MyEnum < Enum
+      #     enumerates :firefox, :safari, :ie, :chrome, :other
+      #   end
+      #   MyEnum[1].to_s # => "safari"
+      #
+      #
+      def self.enumerates *names
+        self.names = names.map(&:to_s)
+      end
+
+      def self.to_sql_str
+        "ENUM('#{names.join("', '")}')"
+      end
+
+      def self.typify
+        'chararray'
+      end
+    end
+
+
+    #
+    # Note that bin 0 is
+    #
+    class Binned < Enum
+      class_inheritable_reader :bins, :empty_bin_name
+      @@empty_bin_name = '(none)'
+
+      def bins
+        self.class.bins
+      end
+
+      # FIXME -- doesn't respect a lower bound.
+      def initialize val
+        return super(val) if val.nil?
+        last_top = bins.first
+        bins.each_with_index do |bin_top, idx|
+          return super(idx) if val <= bin_top
+        end
+        return super(bins.length)
+      end
+
+      def self.enumerates *bins
+        options = bins.extract_options!
+        write_inheritable_attribute :bins, bins
+        last_top = bins.shift
+        # bins.unshift bins.first if last_top == -Infinity
+        names = bins.map do |bin_top|
+          name = bin_name last_top, bin_top, options
+          last_top = (last_top.is_a?(Integer) ? bin_top + 1 : bin_top)
+          name
+        end
+        super(*names)
+      end
+
+      #
+      # Bins
+      #
+      def self.bin_name lo_val, hi_val, options = { }
+        # case lo_val
+        # when Integer then lo_val = [lo_val+1, hi_val].compact.min
+        # end
+        case
+        when lo_val == -Infinity then "< #{hi_val}"
+        when hi_val ==  Infinity then "#{lo_val}+"
+        when (lo_val == hi_val)  then    lo_val
+        else                         "#{lo_val} - #{hi_val}"
+        end
+      end
+
+    end
+  end
+end
+

data/lib/wukong/dfs.rb

@@ -0,0 +1,80 @@
+require 'time' # ain't it always that way
+module Wukong
+  module Dfs
+    def self.list_files dfs_path
+      Wukong.logger.info{ "DFS: listing #{dfs_path}" }
+      listing = `hadoop dfs -ls #{dfs_path}`.split("\n").reject{|ls_line| ls_line =~ /Found \d+ items/i}
+      listing.map{|ls_line| HFile.new_from_ls(ls_line)}
+    end
+
+    #
+    # FIXME -- this will fail if multiple files in a listing have the
+    # same basename. Sorry.
+    #
+    def self.compare_listings src_files, dest_files, &block
+      src_files.sort.each do |src_file|
+        dest_file = dest_files.find{|df| File.basename(src_file) == df.basename }
+        case
+        when (! dest_file)                       then yield :missing, src_file, nil
+        when (! dest_file.kinda_equal(src_file)) then yield :differ,  src_file, dest_file
+        else                                          yield :same,    src_file, dest_file
+        end
+      end
+    end
+
+    class HFile < TypedStruct.new(
+        [:mode_str,     String],
+        [:i_count,      String],
+        [:owner,        String],
+        [:group,        String],
+        [:size,         Integer],
+        [:date,         Bignum],
+        [:path,     String]
+        )
+      def self.new_from_ls ls_line
+        mode, ic, o, g, sz, dt, tm, path = ls_line.chomp.split(/\s+/)
+        date = Time.parse("#{dt} #{tm}").utc.to_flat
+        new mode, ic.to_i, o, g, sz.to_i, date, path
+      end
+      def dirname
+        @dirname ||= File.dirname(path)
+      end
+      def basename
+        @basename ||= File.basename(path)
+      end
+      #
+      # Two files are kinda_equal if they match in size and if
+      # the hdfs version is later than the filesystem version.
+      #
+      def kinda_equal file
+        (self.size == File.size(file)) # && (self.date >= File.mtime(file).utc.to_flat)
+      end
+      def to_s
+        to_a.join("\t")
+      end
+
+      #
+      # These will be very slow.
+      # If some kind soul will integrate JRuby callouts the bards shall
+      # celebrate your name evermore.
+      #
+
+      # rename the file on the HDFS
+      def mv new_filename
+        self.class.run_dfs_command :mv, path, new_filename
+      end
+
+      def self.mkdir dirname
+        run_dfs_command :mkdir, dirname
+      end
+      def self.mkdir_p(*args) self.mkdir *args ; end # HDFS is always -p
+
+      def self.run_dfs_command *args
+        cmd = 'hadoop dfs -'+ args.flatten.compact.join(" ")
+        Wukong.logger.debug{ "DFS: Running #{cmd}" }
+        Wukong.logger.info{ `#{cmd} 2>&1`.gsub(/[\r\n\t]+/, " ") }
+      end
+
+    end
+  end
+end

data/lib/wukong/encoding.rb

@@ -0,0 +1,111 @@
+require 'htmlentities'
+require 'addressable/uri'
+
+module Wukong
+  #
+  # By default (or explicitly with the :xml strategy), convert string to
+  # * XML-encoded ASCII,
+  #
+  # * with a guarantee that the characters " quote, ' apos \\ backslash,
+  #   carriage-return \r newline \n and tab \t (as well as all other control
+  #   characters) are encoded.
+  #
+  # * Any XML-encoding in the original text is encoded with no introspection:
+  #     encode_str("<a href=\"foo\">")
+  #     # => "<a href="foo">"
+  #
+  # * Useful: http://rishida.net/scripts/uniview/conversion.php
+  #
+  # With the :url strategy,
+  # * URL-encode the string
+  # * This is as strict as possible: encodes all but alphanumeric and _ underscore.
+  #   The resulting string is thus XML- and URL-safe.
+  #   http://addressable.rubyforge.org/api/classes/Addressable/URI.html#M000010
+  #
+  # Wukong.decode_str(Wukong.encode_str(str)) returns the original str
+  #
+  #
+  #
+  def self.encode_str str, strategy=:xml
+    begin
+      case strategy
+      when :xml        then self.html_encoder.encode(str, :basic, :named, :decimal).gsub(/\\/, '\')
+      when :url        then Addressable::URI.encode_component(str, /[^\w]/)
+      else raise "Don't know how to encode with strategy #{strategy}"
+      end
+    rescue ArgumentError => e
+      str.gsub!(/[^\w\s\.\-@#%]+/, '')
+      '!!bad_encoding!! ' + str
+    end
+  end
+  # HTMLEntities encoder instance
+  def self.html_encoder
+    @html_encoder ||= HTMLEntities.new
+  end
+
+  #
+  # Decode string from its encode_str representation.  This can include
+  # dangerous things such as tabs, newlines, backslashes and cryptofascist
+  # propaganda.
+  #
+  def self.decode_str str, strategy=:xml
+    case strategy
+    when :xml        then HTMLEntities.decode_entities(str)
+    when :url        then Addressable::URI.unencode_component(str)
+    else raise "Don't know how to decode with strategy #{strategy}"
+    end
+  end
+
+  #
+  # Replace each given field in the hash with its
+  # encoded value
+  #
+  def self.encode_components hsh, *fields
+    fields.each do |field|
+      hsh[field] = hsh[field].to_s.wukong_encode if hsh[field]
+    end
+  end
+end
+
+String.class_eval do
+
+  #
+  # Strip control characters that might harsh our buzz, TSV-wise
+  # See Wukong.encode_str
+  #
+  def wukong_encode!
+    replace self.wukong_encode
+  end
+
+  def wukong_encode
+    Wukong.encode_str(self)
+  end
+
+  #
+  # Decode string into original (and possibly unsafe) form
+  # See Wukong.encode_str and Wukong.decode_str
+  #
+  def wukong_decode!
+    replace self.wukong_decode
+  end
+
+  def wukong_decode
+    Wukong.decode_str(self)
+  end
+
+  #
+  # Takes an XML-encoded or plaintext string and forces it into canonical encoding
+  #
+  def wukong_recode!
+    replace self.wukong_decode.wukong_encode
+  end
+  def wukong_recode
+    Wukong.encode_str(Wukong.decode_str(self))
+  end
+end
+
+Struct.class_eval do
+  def recode!
+    each_pair{|k,v| v.wukong_recode! if (v && v.respond_to?(:wukong_recode!)) }
+  end
+end

data/lib/wukong/extensions.rb

@@ -0,0 +1,15 @@
+#
+# These pull in the minimal functionality of the extlib|activesupport family of
+# gems.
+#
+require 'wukong/extensions/blank'
+require 'wukong/extensions/class'
+require 'wukong/extensions/symbol'
+require 'wukong/extensions/hash'
+require 'wukong/extensions/hash_like'
+require 'wukong/extensions/array'
+require 'wukong/extensions/struct'
+require 'wukong/extensions/module'
+require 'wukong/extensions/string'
+require 'wukong/extensions/date_time'
+require 'wukong/extensions/emittable'

data/lib/wukong/extensions/array.rb

@@ -0,0 +1,18 @@
+class Array
+  #
+  # The following is taken in whole from the extlib gem. Thanks y'all.
+  #
+
+  # Extracts options from a set of arguments. Removes and returns the last
+  # element in the array if it's a hash, otherwise returns a blank hash.
+  #
+  #   def options(*args)
+  #     args.extract_options!
+  #   end
+  #
+  #   options(1, 2)           # => {}
+  #   options(1, 2, :a => :b) # => {:a=>:b}
+  def extract_options!
+    last.is_a?(::Hash) ? pop : {}
+  end
+end

data/lib/wukong/extensions/blank.rb

@@ -0,0 +1,93 @@
+#
+# This is taken in whole from the extlib gem. Thanks y'all.
+#
+
+class Object
+  ##
+  # Returns true if the object is nil or empty (if applicable)
+  #
+  #   [].blank?         #=>  true
+  #   [1].blank?        #=>  false
+  #   [nil].blank?      #=>  false
+  #
+  # @return [TrueClass, FalseClass]
+  #
+  # @api public
+  def blank?
+    nil? || (respond_to?(:empty?) && empty?)
+  end
+end # class Object
+
+class Numeric
+  ##
+  # Numerics are never blank
+  #
+  #   0.blank?          #=>  false
+  #   1.blank?          #=>  false
+  #   6.54321.blank?    #=>  false
+  #
+  # @return [FalseClass]
+  #
+  # @api public
+  def blank?
+    false
+  end
+end # class Numeric
+
+class NilClass
+  ##
+  # Nil is always blank
+  #
+  #   nil.blank?        #=>  true
+  #
+  # @return [TrueClass]
+  #
+  # @api public
+  def blank?
+    true
+  end
+end # class NilClass
+
+class TrueClass
+  ##
+  # True is never blank.
+  #
+  #   true.blank?       #=>  false
+  #
+  # @return [FalseClass]
+  #
+  # @api public
+  def blank?
+    false
+  end
+end # class TrueClass
+
+class FalseClass
+  ##
+  # False is always blank.
+  #
+  #   false.blank?      #=>  true
+  #
+  # @return [TrueClass]
+  #
+  # @api public
+  def blank?
+    true
+  end
+end # class FalseClass
+
+class String
+  ##
+  # Strips out whitespace then tests if the string is empty.
+  #
+  #   "".blank?         #=>  true
+  #   "     ".blank?    #=>  true
+  #   " hey ho ".blank? #=>  false
+  #
+  # @return [TrueClass, FalseClass]
+  #
+  # @api public
+  def blank?
+    strip.empty?
+  end
+end # class String