wukong 0.1.1

data/lib/wukong/extensions/hashlike_class.rb

@@ -0,0 +1,47 @@
+require 'wukong/extensions/class'
+module Wukong
+
+  module HashlikeClass
+    module ClassMethods
+      def has_members *members
+        self.members ||= []
+        self.members = members.map(&:to_s) + self.members
+        self.members.each do |member|
+          attr_accessor member.to_sym
+        end
+      end
+      alias_method :has_member, :has_members
+      def keys
+        members
+      end
+    end
+
+    def [](key)
+      self.send(key)
+    end
+
+    def []=(key, val)
+      self.send("#{key}=", val)
+    end
+
+    def to_a
+      values_of(*members)
+    end
+
+    def to_flat
+      to_a.map(&:to_flat).flatten
+    end
+
+    def self.included base
+      base.class_eval do
+        extend ClassMethods
+        include HashLike
+        class_inheritable_accessor :members
+
+        def to_hash *args
+          super(*args).merge 'type' => self.class.to_s
+        end
+      end
+    end
+  end
+end

data/lib/wukong/extensions/module.rb

@@ -0,0 +1,2 @@
+require 'extlib/object'
+require 'extlib/module'

data/lib/wukong/extensions/pathname.rb

@@ -0,0 +1,27 @@
+require 'pathname'
+class Pathname
+  # Append path segments and expand to absolute path
+  #
+  #   file = Pathname(Dir.pwd) / "subdir1" / :subdir2 / "filename.ext"
+  #
+  # @param [Pathname, String, #to_s] path path segment to concatenate with receiver
+  #
+  # @return [Pathname]
+  #   receiver with _path_ appended and expanded to an absolute path
+  #
+  # @api public
+  def /(path)
+    (self + path).expand_path
+  end
+
+  def self.[](*vals)
+    new( File.join(vals) )
+  end
+end
+
+class Subdir < Pathname
+  def self.[](*vals)
+    dir = File.dirname(vals.shift)
+    new(File.join(dir, *vals))
+  end
+end

data/lib/wukong/extensions/string.rb

@@ -0,0 +1,65 @@
+#
+# String Monkeypatched for processing with wukong: see wukong/extensions/string
+#
+String.class_eval do
+  # By default, +camelize+ converts strings to UpperCamelCase. If the argument to +camelize+
+  # is set to <tt>:lower</tt> then +camelize+ produces lowerCamelCase.
+  #
+  # +camelize+ will also convert '/' to '::' which is useful for converting paths to namespaces.
+  #
+  # Examples:
+  #   "active_record".camelize                # => "ActiveRecord"
+  #   "active_record".camelize(:lower)        # => "activeRecord"
+  #   "active_record/errors".camelize         # => "ActiveRecord::Errors"
+  #   "active_record/errors".camelize(:lower) # => "activeRecord::Errors"
+  def camelize(first_letter_in_uppercase = true)
+    if first_letter_in_uppercase
+      self.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    else
+      self.first + camelize(self)[1..-1]
+    end
+  end
+
+  #
+  # The reverse of +camelize+. Makes an underscored, lowercase form from the expression in the string.
+  #
+  # Changes '::' to '/' to convert namespaces to paths.
+  #
+  # Examples:
+  #   "ActiveRecord".underscore         # => "active_record"
+  #   "ActiveRecord::Errors".underscore # => active_record/errors
+  #
+  # Stolen from active_support
+  #
+  def underscore
+    gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase
+  end
+
+  # Tries to find a constant with the name specified in the argument string:
+  #
+  #   "Module".constantize     # => Module
+  #   "Test::Unit".constantize # => Test::Unit
+  #
+  # The name is assumed to be the one of a top-level constant, no matter whether
+  # it starts with "::" or not. No lexical context is taken into account:
+  #
+  #   C = 'outside'
+  #   module M
+  #     C = 'inside'
+  #     C               # => 'inside'
+  #     "C".constantize # => 'outside', same as ::C
+  #   end
+  #
+  # NameError is raised when the name is not in CamelCase or the constant is
+  # unknown.
+  def constantize
+    unless /\A(?:::)?([A-Z]\w*(?:::[A-Z]\w*)*)\z/ =~ self
+      raise NameError, "#{self.inspect} is not a valid constant name!"
+    end
+    Object.module_eval("::#{$1}", __FILE__, __LINE__)
+  end
+end

data/lib/wukong/extensions/struct.rb

@@ -0,0 +1,17 @@
+require 'wukong/extensions/hash'
+require 'wukong/extensions/hash_like'
+require 'wukong/extensions/symbol'
+
+#
+# extensions/struct
+#
+# Add several methods to make a struct duck-type much more like a Hash
+#
+Struct.class_eval do
+  include Wukong::HashLike
+  def self.keys
+    members
+  end
+end
+
+

data/lib/wukong/extensions/symbol.rb

@@ -0,0 +1,11 @@
+#
+# h2. extensions/symbol.rb -- extensions to symbol class
+#
+class Symbol
+  #
+  # Turn the symbol into a simple proc (stolen from
+  # <tt>ActiveSupport::CoreExtensions::Symbol</tt>).
+  def to_proc
+    Proc.new { |*args| args.shift.__send__(self, *args) }
+  end
+end

data/lib/wukong/logger.rb

@@ -0,0 +1,53 @@
+module Wukong
+  # Common logger
+  #
+  # Set your own at any time with
+  #   Wukong.logger = YourAwesomeLogger.new(...)
+  # If you have log4r installed you can use
+  #   Wukong.logger = Wukong.default_log4r_logger
+  #
+  # If Wukong.logger is too much typing for you,
+  # use the Log constant
+  #
+  # Default format:
+  #     I, [2009-07-26T19:58:46-05:00 #12332]: Up to 2000 char message
+  #
+  def self.logger
+    @logger ||= default_ruby_logger
+  end
+
+  #
+  # Log4r logger, set up to produce tab-delimited (and thus, wukong|hadoop
+  # friendly) output lines
+  #
+  def self.default_log4r_logger logger_handle='wukong'
+    require 'log4r'
+    lgr       = Log4r::Logger.new logger_handle
+    outputter = Log4r::Outputter.stderr
+    # Define timestamp formatter method
+    ::Time.class_eval do def utc_iso8601() utc.iso8601 ; end ; end
+    # 2009-07-25T00:12:05Z INFO PID\t
+    outputter.formatter  = Log4r::PatternFormatter.new(
+      :pattern     => "%d %.4l #{Process.pid}\t%.2000m",
+      :date_method => :utc_iso8601
+      )
+    lgr.outputters = outputter
+    lgr
+  end
+
+  def self.default_ruby_logger
+    require 'logger'
+    Logger.new STDERR
+  end
+
+  def self.logger= logger
+    @logger = logger
+  end
+end
+
+#
+# A convenient logger.
+#
+# Define NO_WUKONG_LOG (or define Log yourself) to prevent its creation
+#
+Log = Wukong.logger unless (defined?(Log) || defined?(NO_WUKONG_LOG))

data/lib/wukong/models/graph.rb

@@ -0,0 +1,27 @@
+
+module Wukong
+  module Models
+    class Edge < TypedStruct.new(
+        [:src,              Integer],
+        [:dest,             Integer]
+        )
+    end
+
+    class MultiEdge < TypedStruct.new(
+        [:src,              Integer],
+        [:dest,             Integer],
+        [:a_follows_b,      Integer],
+        [:b_follows_a,      Integer],
+        [:a_replies_b,      Integer],
+        [:b_replies_a,      Integer],
+        [:a_atsigns_b,      Integer],
+        [:b_atsigns_a,      Integer],
+        [:a_retweets_b,     Integer],
+        [:b_retweets_a,     Integer],
+        [:a_favorites_b,    Integer],
+        [:b_favorites_a,    Integer]
+        )
+    end
+
+  end
+end

data/lib/wukong/rdf.rb

@@ -0,0 +1,104 @@
+module Wukong
+  #
+  # Dump wukong object as RDF triples:
+  #
+  #   <key     attr    val     module Wukong
+  #
+  # Dump wukong object as RDF triples:
+  #
+  #   <key>     <attr>    <val> # <extra>
+  #
+  # Each element of the triple is XML encoded such that it contains no tab,
+  # newline or carriage returns, and the three are tab-separated. Any extra
+  # fields -- reification info, for instance -- are appended as a comment.
+  #
+  # This makes the result not only a valid RDF triple file but perfectly
+  # palatable to Wukong for further processing.
+  #
+  module Rdf
+
+    #
+    # RDF-formatted date
+    #
+    def self.encode_datetime dt
+      DateTime.parse_safely(dt).xmlschema
+    end
+
+    #
+    # Emit a component (subject or object) with the right semantic encoding
+    #
+    # Use :boolskip if a false property should just be left out.
+    #
+    def rdf_component val, type
+      case type
+      when :tweet         then %Q{<http://twitter.com/statuses/show/#{val}.xml>}
+      when :user          then %Q{<http://twitter.com/users/show/#{val}.xml>}
+      when :bool          then ((!val) || (val==0) || (val=="0")) ? '"false"^^<xsd:boolean>' : '"true"^^<xsd:boolean>'
+      when :boolskip      then ((!val) || (val==0) || (val=="0")) ? nil                      : '"true"^^<xsd:boolean>'
+      when :int           then %Q{"#{val.to_i}"^^<xsd:integer>}
+      when :date          then %Q{"#{TwitterRdf.encode_datetime(val)}"^^<xsd:dateTime>}
+      when :str           then %Q{"#{val}"}
+      else raise "Don't know how to encode #{type}"
+      end
+    end
+
+    #
+    # Express relationship (predicate) in RDF
+    #
+    def rdf_pred pred
+      case pred
+      when :created_at  then %Q{<http://twitter.com/##{pred}>}
+      else                   %Q{<http://twitter.com/##{pred}>}
+      end
+    end
+
+    #
+    # RDF Triple string for the given (subject, object, predicate)
+    #   http://www.w3.org/TR/rdf-testcases/#ntriples
+    #
+    def self.rdf_triple subj, pred, obj, comment=nil
+      comment = "\t# " + comment.to_s unless comment.blank?
+      %Q{%-55s\t%-39s\t%-23s\t.%s} % [subj, pred, obj, comment]
+    end
+
+    def mutable?(attr)
+      false
+    end
+
+    #
+    # Extract [subject, predicate, object, (extra)] tuples.
+    #
+    # (extra) is set to +scraped at+ for #mutable? attributes, blank otherwise.
+    #
+    def to_rdf3_tuples
+      members_with_types.map do |attr, type|
+        next if self[attr].blank?
+        subj    = rdf_resource
+        pred    = rdf_pred(attr)
+        obj     = rdf_component(self[attr], type) or next
+        comment = scraped_at if mutable?(attr)
+        [subj, pred, obj, comment]
+      end.compact
+    end
+
+    #
+    # Convert an object to an rdf triple.
+    #
+    # Appends scraped at to #mutable? attributes
+    #
+    def to_rdf3
+      to_rdf3_tuples.map do |tuple|
+        self.class.rdf_triple tuple
+      end.join("\n")
+    end
+
+  end
+end
+>
+  #
+  #
+  module Rdf
+    def to_rdf
+    end
+  end
+end

data/lib/wukong/schema.rb

@@ -0,0 +1,37 @@
+module Wukong
+  #
+  # Export model's structure for other data frameworks:
+  # SQL and Pig
+  #
+  module Schema
+    def to_sql
+    end
+
+
+    # Export schema as Pig
+    def to_pig
+      members.zip(mtypes).map do |member, type|
+        member.to_s + ': ' + type.to_pig
+      end.join(', ')
+    end
+
+    def pig_klass
+      self.to_s.gsub(/.*::/, '')
+    end
+
+    def pig_load filename=nil
+      cmd = [
+        "%-23s" % pig_klass,
+        "= LOAD", filename || pig_klass.underscore.pluralize,
+        "AS ( rsrc:chararray,",     self.to_pig, ')',
+      ].join(" ")
+    end
+  end
+end
+
+class << Integer ; def to_pig() 'int'           end ; end
+class << Bignum  ; def to_pig() 'long'          end ; end
+class << Float   ; def to_pig() 'float'         end ; end
+class << String  ; def to_pig() 'chararray'     end ; end
+class << Symbol  ; def to_pig() self            end ; end
+class << Date    ; def to_pig() 'long'          end ; end

data/lib/wukong/script.rb

@@ -0,0 +1,265 @@
+require 'pathname'
+require 'wukong/script/hadoop_command'
+require 'wukong/script/local_command'
+require 'rbconfig'
+module Wukong
+
+  # == How to run a Wukong script
+  #
+  #   your/script.rb --run path/to/input_files path/to/output_dir
+  #
+  # All of the file paths are HDFS paths ; your script path, of course, is on the local filesystem.
+  #
+  # == Command-line options
+  #
+  # If you'd like to listen for any command-line options, specify them at the
+  # command line:
+  #
+  #   your/script.rb --my_bool_opt --my_val_taking_opt=val \
+  #     --run path/to/input_files path/to/output_dir
+  #
+  # In this case the options hash for both Mapper and Reducer will contain
+  #
+  #   :my_bool_opt       => true,
+  #   :my_val_taking_opt => 'val'
+  #
+  # == Complicated input paths
+  #
+  # To use more than one file as input, you can use normal * ? [] wildcards or
+  # give a comma-separated list -- see the hadoop documentation for syntax.
+  #
+  # == Run locally (--run=local)
+  #
+  # To run your script locally, use --run=local
+  #
+  #   your/script.rb --run=local path/to/input_files path/to/output_dir
+  #
+  # This will pipe the contents of path/to/input_files through first your
+  # mapper, then sort, then the reducer, storing the results in the given output
+  # directory.
+  #
+  # All paths refer to the /local/ filesystem -- hadoop is never involved and in
+  # fact doesn't even have to be installed.
+  #
+  # == How to test your scripts
+  #
+  # You can supply the --map argument in place of --run to run the mapper on its
+  # own (and similarly, --reduce to run the reducer standalone):
+  #
+  #   cat ./local/test/input.tsv | ./examples/word_count.rb --map | more
+  #
+  # or, if your test data lies on the HDFS,
+  #
+  #   hdp-cat test/input.tsv | ./examples/word_count.rb --map | more
+  #
+  #
+  class Script
+    include Wukong::HadoopCommand
+    include Wukong::LocalCommand
+    attr_accessor :mapper_klass, :reducer_klass, :options
+
+    #
+    # Instantiate the Script with the Mapper and the Reducer class (each a
+    # Wukong::Streamer) it should call back.
+    #
+    #
+    # == Identity or External program as map or reduce
+    #
+    # To use the identity reducer ('cat'), instantiate your Script class with
+    # +nil+ as the reducer class. (And similarly to use an identity mapper,
+    # supply +nil+ for the mapper class.)
+    #
+    # To use an external program as your reducer (mapper), subclass the
+    # reduce_command (map_command) method to return the full command line
+    # expression to call.
+    #
+    #   class MyMapper < Wukong::Streamer::Base
+    #     # ... awesome stuff ...
+    #   end
+    #
+    #   class MyScript < Wukong::Script
+    #     # prefix each unique line with the count of its occurrences.
+    #     def reduce_command
+    #       '/usr/bin/uniq -c'
+    #     end
+    #   end
+    #   MyScript.new(MyMapper, nil).run
+    #
+    def initialize mapper_klass, reducer_klass, extra_options={}
+      self.options = default_options.merge(extra_options)
+      process_argv!
+      self.mapper_klass  = mapper_klass
+      self.reducer_klass = reducer_klass
+      # If no reducer_klass and no reduce_command, then skip the reduce phase
+      options[:reduce_tasks] = 0 if (! reducer_klass) && (! options[:reduce_command]) && (! options[:reduce_tasks])
+    end
+
+    #
+    # Gives default options.  Command line parameters take precedence
+    #
+    # MAKE SURE YOU CALL SUPER: write your script according to the patter
+    #
+    #   super.merge :my_option => :val
+    #
+    def default_options
+      Wukong::CONFIG[:runner_defaults] || {}
+    end
+
+    # Options that don't need to go in the :all_args hash
+    def std_options
+      @std_options ||= [:run, :map, :reduce, ] + HADOOP_OPTIONS_MAP.keys
+    end
+
+    #
+    # Parse the command-line args into the options hash.
+    #
+    # I should not reinvent the wheel.
+    # Yet: here we are.
+    #
+    # '--foo=foo_val'  produces :foo => 'foo_val' in the options hash.
+    # '--'             After seeing a non-'--' flag, or a '--' on its own, no further flags are parsed
+    #
+    # options[:all_args] contains all arguments that are not in std_options
+    # options[:rest]     contains all arguments following the first non-flag (or the '--')
+    #
+    def process_argv!
+      options[:all_args] = []
+      args = ARGV.dup
+      while args do
+        arg = args.shift
+        case
+        when arg == '--'
+          break
+        when arg =~ /\A--(\w+)(?:=(.+))?\z/
+          opt, val = [$1, $2]
+          opt = opt.to_sym
+          val ||= true
+          self.options[opt] = val
+          options[:all_args] << arg unless std_options.include?(opt)
+        else
+          args.unshift(arg) ; break
+        end
+      end
+      options[:all_args] = options[:all_args].join(" ")
+      options[:rest]     = args
+    end
+
+    def this_script_filename
+      Pathname.new($0).realpath
+    end
+
+    def ruby_interpreter_path
+      Pathname.new(
+                   File.join(Config::CONFIG["bindir"],
+                             Config::CONFIG["RUBY_INSTALL_NAME"]+
+                             Config::CONFIG["EXEEXT"])
+                   ).realpath
+    end
+
+    #
+    # by default, call this script in --map mode
+    #
+    def map_command
+      case
+      when mapper_klass
+        "#{ruby_interpreter_path} #{this_script_filename} --map " + options[:all_args]
+      else options[:map_command] || Wukong::CONFIG[:default_mapper] end
+    end
+
+    #
+    # Shell command for reduce phase
+    # by default, call this script in --reduce mode
+    #
+    def reduce_command
+      case
+      when reducer_klass
+        "#{ruby_interpreter_path} #{this_script_filename} --reduce " + options[:all_args]
+      else options[:reduce_command] || Wukong::CONFIG[:default_reducer] end
+    end
+
+    #
+    # Shell command to re-run in mapreduce mode using --map and --reduce
+    #
+    def runner_command input_path, output_path
+      # run as either local or hadoop
+      case run_mode
+      when 'local'
+        $stderr.puts "  Reading STDIN / Writing STDOUT"
+        command = local_command input_path, output_path
+      when 'hadoop', 'mapred'
+        $stderr.puts "  Launching hadoop as"
+        command = hadoop_command input_path, output_path
+      else
+        raise "Need to use --run=local or --run=hadoop; or to use the :default_run_mode in config.yaml just say --run "
+      end
+    end
+
+    def run_mode
+      # if only --run is given, assume default run mode
+      options[:run] = Wukong::CONFIG[:default_run_mode] if (options[:run] == true)
+      options[:run].to_s
+    end
+
+    def input_output_paths
+      # input / output paths
+      input_path, output_path = options[:rest][0..1]
+      raise "You need to specify a parsed input directory and a directory for output. Got #{ARGV.inspect}" if (! options[:fake]) && (input_path.blank? || output_path.blank?)
+      [input_path, output_path]
+    end
+
+    def maybe_overwrite_output_paths! output_path
+      if (options[:overwrite] || options[:rm]) && (run_mode != 'local')
+        $stderr.puts "Removing output file #{output_path}"
+        `hdp-rm -r '#{output_path}'`
+      end
+    end
+
+    #
+    # Execute the runner phase
+    #
+    def exec_hadoop_streaming
+      $stderr.puts "Streaming on self"
+      input_path, output_path = input_output_paths
+      maybe_overwrite_output_paths! output_path
+      command = runner_command(input_path, output_path)
+      $stderr.puts command
+      if ! options[:fake]
+        $stdout.puts `#{command}`
+      end
+    end
+
+    #
+    # If --map or --reduce, dispatch to the mapper or reducer.
+    # Otherwise,
+    #
+    def run
+      case
+      when options[:map]
+        mapper_klass.new(self.options).stream
+      when options[:reduce]
+        reducer_klass.new(self.options).stream
+      when options[:run]
+        exec_hadoop_streaming
+      else
+        self.help # Normant Vincent Peale is proud of you
+      end
+    end
+
+    #
+    # Command line usage
+    #
+    def help
+      $stderr.puts "#{self.class} script"
+      $stderr.puts %Q{
+        #{$0} --run=hadoop input_hdfs_path output_hdfs_dir    # run the script with hadoop streaming
+        #{$0} --run=local  input_hdfs_path output_hdfs_dir    # run the script on local filesystem using unix pipes
+        #{$0} --run        input_hdfs_path output_hdfs_dir    # run the script with the mode given in config/wukong*.yaml
+        #{$0} --map
+        #{$0} --reduce                                 # dispatch to the mapper or reducer
+
+      You can specify as well arbitrary script-specific command line flags; they are added to your options[] hash.
+      }
+    end
+  end
+
+end