wukong 1.4.0 → 1.4.1

data/docpages/INSTALL.textile

@@ -27,8 +27,6 @@ pre. $ sudo gem install {{ site.gemname }} --source=http://gemcutter.org
 
 You can instead download this project in either "zip":http://github.com/mrflip/{{ site.gemname }}/zipball/master or "tar":http://github.com/mrflip/{{ site.gemname }}/tarball/master formats.
 
-<notextile></div><div class="toggle"></notextile>
-
 h3. Get the Dependencies
 
 * Hadoop

data/docpages/index.textile

@@ -71,7 +71,7 @@ h2. Documentation index
 
 h2. Credits
 
-Wukong was written by "Philip (flip) Kromer":http://mrflip.com (flip@infochimps.org) for the "infochimps project":http://infochimps.org
+Wukong was written by "Philip (flip) Kromer":http://mrflip.com (flip@infochimps.org / "@mrflip":http://twitter.com/mrflip) for the "infochimps project":http://infochimps.org
 
 Patches submitted by:
 * gemified by Ben Woosley (ben.woosley@gmail.com)
@@ -81,10 +81,12 @@ Thanks to:
 * "Brad Heintz":http://www.bradheintz.com/no1thing/talks/ for his early feedback
 * "Phil Ripperger":http://blog.pdatasolutions.com for his "wukong in the Amazon AWS cloud":http://blog.pdatasolutions.com/post/191978092/ruby-on-hadoop-quickstart tutorial.
 
-<notextile><div class="toggle"></notextile>
+<notextile></div><div class="toggle"></notextile>
 
 h2. Help!
 
 Send Wukong questions to the "Infinite Monkeywrench mailing list":http://groups.google.com/group/infochimps-code
 
 <notextile></div></notextile>
+
+{% include news.html %}

data/examples/apache_log_parser.rb

@@ -5,21 +5,21 @@ require 'wukong'
 module ApacheLogParser
   class Mapper < Wukong::Streamer::LineStreamer
 
+    # regular expression for apache-style log lines
+    # note that we strip out the google analytics listener.
+    LOG_RE = %r{\A
+           (\d+\.\d+\.\d+\.\d+)                         # IP addr
+         \s([^\s]+)\s                                   # -
+         \s([^\s]+)                                     # -
+         \s\[(\d\d/\w+/\d+):(\d\d:\d\d:\d\d)([^\]]*)\]  # [07/Jun/2008:20:37:11 +0000]
+         \s(\d+)                                        # 400
+         \s"([^\"]*(?:\" \+ gaJsHost \+ \"[^\"]*)?)"    # "GET /faq" + gaJsHost + "google-analytics.com/ga.js HTTP/1.1"
+         \s(\d+)                                        # 173
+         \s"([^\"]*)" "([^\"]*)" "([^\"]*)"             #  "-" "-" "-"
+      \z}x
 
-    def parse_request req
-      m = %r{\A(\w+) (.*) (\w+/[\w\.]+)\z}.match(req)
-      if m
-        [''] + m.captures
-      else
-        [req, '', '', '']
-      end
-    end
-
-
-    # regular expression to match on apache-style log lines
-    #             IP addr              -         -        [07/Jun/2008:20:37:11 +0000]               400   "GET /faq" + gaJsHost + "google-analytics.com/ga.js HTTP/1.1" 173 "-" "-" "-"
-    LOG_RE = %r{\A(\d+\.\d+\.\d+\.\d+) ([^\s]+) ([^\s]+) \[(\d\d/\w+/\d+):(\d\d:\d\d:\d\d)([^\]]*)\] (\d+) "([^\"]*(?:\" \+ gaJsHost \+ \"[^\"]*)?)" (\d+) "([^\"]*)" "([^\"]*)" "([^\"]*)"\z}
-
+    # Use the regex to break line into fields
+    # Emit each record as flat line
     def process line
       line.chomp
       m = LOG_RE.match(line)
@@ -32,8 +32,20 @@ module ApacheLogParser
         yield [:unparseable, line]
       end
     end
+
+
+    def parse_request req
+      m = %r{\A(\w+) (.*) (\w+/[\w\.]+)\z}.match(req)
+      if m
+        [''] + m.captures
+      else
+        [req, '', '', '']
+      end
+    end
+
   end
 
+
   class Reducer < Wukong::Streamer::LineStreamer
   end
 

data/examples/graph/gen_symmetric_links.rb

@@ -17,6 +17,16 @@ module Wukong::Streamer
   end
 end
 
+#
+# Find symmetric links
+#
+# Takes adjacency list for a directed graph and emits only edges where
+#   A->B and B->A
+#
+# The output will list each such symmetric edge exactly once as
+#    a_symmetric_b   node1    node2
+# where node1 is lexicographically less than node2.
+#
 module FindSymmetricLinks
 
   class Mapper < Wukong::Streamer::EdgeStreamer

data/examples/sample_records.rb

@@ -8,7 +8,7 @@ require 'wukong'
 # Set the sampling fraction at the command line using the
 #   --sampling_fraction=
 # option: for example, to take a random 1/1000th of the lines in huge_files,
-#  ./examples/sample_records.rb --sampling_fraction=0.001 --go huge_files sampled_files
+#  ./examples/sample_records.rb --sampling_fraction=0.001 --run huge_files sampled_files
 #
 class Mapper < Wukong::Streamer::LineStreamer
   include Wukong::Streamer::Filter
@@ -32,13 +32,11 @@ class Mapper < Wukong::Streamer::LineStreamer
   end
 end
 
-class Script < Wukong::Script
-  def default_options
-    super.merge :reduce_tasks => 0
-  end
-end
-
 #
 # Executes the script
 #
-Script.new( Mapper, nil ).run
+Wukong::Script.new( Mapper,
+  nil,
+  :reduce_tasks => 0,
+  :reuse_jvms   => true
+  ).run

data/lib/wukong/datatypes/enum.rb

@@ -76,7 +76,7 @@ module Wukong
     # Note that bin 0 is
     #
     class Binned < Enum
-      class_inheritable_reader :bins, :empty_bin_name
+      class_inheritable_accessor :bins, :empty_bin_name
       @@empty_bin_name = '(none)'
 
       def bins
@@ -95,7 +95,7 @@ module Wukong
 
       def self.enumerates *bins
         options = bins.extract_options!
-        write_inheritable_attribute :bins, bins
+        self.bins = bins
         last_top = bins.shift
         # bins.unshift bins.first if last_top == -Infinity
         names = bins.map do |bin_top|

data/lib/wukong/dfs.rb

@@ -22,15 +22,16 @@ module Wukong
       end
     end
 
-    class HFile < TypedStruct.new(
-        [:mode_str,     String],
-        [:i_count,      String],
-        [:owner,        String],
-        [:group,        String],
-        [:size,         Integer],
-        [:date,         Bignum],
-        [:path,     String]
-        )
+    HFile = TypedStruct.new(
+      [:mode_str,     String],
+      [:i_count,      String],
+      [:owner,        String],
+      [:group,        String],
+      [:size,         Integer],
+      [:date,         Bignum],
+      [:path,     String]
+      )
+    HFile.class_eval do
       def self.new_from_ls ls_line
         mode, ic, o, g, sz, dt, tm, path = ls_line.chomp.split(/\s+/)
         date = Time.parse("#{dt} #{tm}").utc.to_flat

data/lib/wukong/encoding.rb

@@ -1,6 +1,23 @@
 require 'htmlentities'
 require 'addressable/uri'
 
+# Fix a bug (?) in the HTMLEntities encoder class with $KCODE='NONE'
+HTMLEntities::Encoder.class_eval do
+private
+  def extended_entity_regexp
+    @extended_entity_regexp ||= (
+      if encoding_aware?
+        regexp = '[^\u{20}-\u{7E}]'
+      else
+        # regexp = '[^\x20-\x7E]'
+        regexp = '[\x00-\x1f]|[\xc0-\xfd][\x80-\xbf]+'
+      end
+      regexp += "|'" if @flavor == 'html4'
+      Regexp.new(regexp)
+      )
+  end
+end
+
 module Wukong
   #
   # By default (or explicitly with the :xml strategy), convert string to
@@ -24,7 +41,9 @@ module Wukong
   #
   # Wukong.decode_str(Wukong.encode_str(str)) returns the original str
   #
-  #
+  # If you're seeing bad_encoding errors, try
+  #   $KCODE='u' unless "1.9".respond_to?(:encoding)
+  # at the start of your script.
   #
   def self.encode_str str, strategy=:xml
     begin
@@ -34,8 +53,7 @@ module Wukong
       else raise "Don't know how to encode with strategy #{strategy}"
       end
     rescue ArgumentError => e
-      str.gsub!(/[^\w\s\.\-@#%]+/, '')
-      '!!bad_encoding!! ' + str
+      '!bad_encoding!! ' + str.gsub(/[^\w\s\.\-@#%]+/, '')
     end
   end
   # HTMLEntities encoder instance
@@ -89,7 +107,7 @@ String.class_eval do
     replace self.wukong_decode(*args)
   end
 
-  def wukong_decode
+  def wukong_decode(*args)
     Wukong.decode_str(self, *args)
   end
 

data/lib/wukong/extensions/emittable.rb

@@ -18,7 +18,7 @@ Struct.class_eval do
   #
   # Flatten for packing as resource name followed by all fields
   #
-  def to_flat include_key=true
+  def to_flat include_key=false
     if include_key.is_a? Proc
       sort_key = include_key.call(self)
     elsif (! include_key.blank?) && respond_to?(:key)

data/lib/wukong/extensions/hash_keys.rb

@@ -0,0 +1,16 @@
+class Hash
+
+  # Return a new hash with all keys converted to symbols.
+  def symbolize_keys
+    inject({}) do |options, (key, value)|
+      options[(key.to_sym rescue key) || key] = value
+      options
+    end
+  end
+
+  # Destructively convert all keys to symbols.
+  def symbolize_keys!
+    self.replace(self.symbolize_keys)
+  end
+
+end

data/lib/wukong/extensions/hash_like.rb

@@ -114,6 +114,23 @@ module Wukong
         extend ClassMethods
       end
     end
+
+    def coerce_attr attr, coerce_blank_to_nil=false, &block
+      orig_val = self.send(attr)
+      new_val = (coerce_blank_to_nil && orig_val.blank?) ? nil : block.call(orig_val)
+      self.send("#{attr}=", new_val)
+    end
+
+    def coerce_to_int! attr, *args
+      coerce_attr(attr, *args) do |val|
+        val.to_i
+      end
+    end
+
+    def coerce_to_date! attr, *args
+      coerce_attr(attr, *args){|val| val.is_a?(DateTime) ? val : DateTime.parse(val) rescue nil }
+    end
+
   end
 
 end

data/lib/wukong/models/graph.rb

@@ -1,27 +1,25 @@
 
 module Wukong
   module Models
-    class Edge < TypedStruct.new(
-        [:src,              Integer],
-        [:dest,             Integer]
-        )
-    end
+    Edge = TypedStruct.new(
+      [:src,              Integer],
+      [:dest,             Integer]
+      )
 
-    class MultiEdge < TypedStruct.new(
-        [:src,              Integer],
-        [:dest,             Integer],
-        [:a_follows_b,      Integer],
-        [:b_follows_a,      Integer],
-        [:a_replies_b,      Integer],
-        [:b_replies_a,      Integer],
-        [:a_atsigns_b,      Integer],
-        [:b_atsigns_a,      Integer],
-        [:a_retweets_b,     Integer],
-        [:b_retweets_a,     Integer],
-        [:a_favorites_b,    Integer],
-        [:b_favorites_a,    Integer]
-        )
-    end
+    MultiEdge = TypedStruct.new(
+      [:src,              Integer],
+      [:dest,             Integer],
+      [:a_follows_b,      Integer],
+      [:b_follows_a,      Integer],
+      [:a_replies_b,      Integer],
+      [:b_replies_a,      Integer],
+      [:a_atsigns_b,      Integer],
+      [:b_atsigns_a,      Integer],
+      [:a_retweets_b,     Integer],
+      [:b_retweets_a,     Integer],
+      [:a_favorites_b,    Integer],
+      [:b_favorites_a,    Integer]
+      )
 
   end
 end

data/lib/wukong/schema.rb

@@ -106,11 +106,11 @@ module Wukong
       # should, follow with an immediate GENERATE to ditch that field.)
       #
       def pig_load filename=nil
-        filename ||= table_name+'.tsv'
+        filename ||= resource_name.to_s+'.tsv'
         cmd = [
-          "%-23s" % resource_name,
-          "= LOAD", filename,
-          "AS ( rsrc:chararray,", self.to_pig, ')',
+          "%-23s" % self.to_s.gsub(/^.*\W/, ""),
+          "= LOAD '#{filename}'",
+          "AS ( rsrc:chararray,", self.to_pig, ') ;',
         ].join(" ")
       end
 
@@ -125,7 +125,7 @@ module Wukong
         sql_str = []
         members.zip(mtypes).each do |attr, type|
           type_str = type.respond_to?(:to_sql) ? type.to_sql : type.to_s.upcase
-          sql_str << "  %-21s\t%s" %["`#{attr}`", type_str]
+          sql_str << "  %-29s\t%s" %["`#{attr}`", type_str]
         end
         sql_str.join(",\n")
       end
@@ -184,13 +184,15 @@ module Wukong
       # different objects jumbled together, you can just dump in the whole file,
       # landing each object in its correct table.
       #
-      def sql_load_mysql
+      def sql_load_mysql(filename=nil)
+        filename ||= ":resource_name.tsv"
+        filename.gsub!(/:resource_name/, self.table_name)
         str = []
         # disable indexing during bulk load
-        str << %Q{ALTER TABLE            `#{self.resource_name}` DISABLE KEYS; }
+        str << %Q{ALTER TABLE            `#{self.table_name}` DISABLE KEYS; }
         # Bulk load the tab-separated-values file.
-        str << %Q{LOAD DATA LOCAL INFILE '#{self.resource_name}.tsv'}
-        str << %Q{  REPLACE INTO TABLE   `#{self.resource_name}`    }
+        str << %Q{LOAD DATA LOCAL INFILE '#{filename}'}
+        str << %Q{  REPLACE INTO TABLE   `#{self.table_name}`    }
         str << %Q{  COLUMNS                                         }
         str << %Q{    TERMINATED BY           '\\t'                 }
         str << %Q{    OPTIONALLY ENCLOSED BY  ''                    }
@@ -200,9 +202,9 @@ module Wukong
         str << '    '+self.sql_members
         str << %Q{\n  ); }
         # Re-enable indexing
-        str << %Q{ALTER TABLE `#{self.resource_name}` ENABLE KEYS ; }
+        str << %Q{ALTER TABLE `#{self.table_name}` ENABLE KEYS ; }
         # Show it loaded correctly
-        str << %Q{SELECT '#{self.resource_name}', NOW(), COUNT(*) FROM `#{self.resource_name}`; }
+        str << %Q{SELECT NOW(), COUNT(*), '#{self.table_name}' FROM `#{self.table_name}`; }
         str.join("\n")
       end
 

data/lib/wukong/script.rb

@@ -124,12 +124,13 @@ module Wukong
     #
     def process_argv!
       options[:all_args] = []
-      args = ARGV.dup
-      while args do
+      options[:rest]     = []
+      args      = ARGV.dup
+      while (! args.blank?) do
         arg = args.shift
         case
         when arg == '--'
-          break
+          options[:rest] += args
         when arg =~ /\A--(\w+)(?:=(.+))?\z/
           opt, val = [$1, $2]
           opt = opt.to_sym
@@ -137,11 +138,12 @@ module Wukong
           self.options[opt] = val
           options[:all_args] << arg unless std_options.include?(opt)
         else
-          args.unshift(arg) ; break
+          options[:all_args]  << arg
+          options[:rest]      << arg
         end
+        # p [options, arg, args]
       end
       options[:all_args] = options[:all_args].join(" ")
-      options[:rest]     = args
     end
 
     def this_script_filename
@@ -203,7 +205,7 @@ module Wukong
     def input_output_paths
       # input / output paths
       input_path, output_path = options[:rest][0..1]
-      raise "You need to specify a parsed input directory and a directory for output. Got #{ARGV.inspect}" if (! options[:fake]) && (input_path.blank? || output_path.blank?)
+      raise "You need to specify a parsed input directory and a directory for output. Got #{ARGV.inspect}" if (! options[:dry_run]) && (input_path.blank? || output_path.blank?)
       [input_path, output_path]
     end
 
@@ -223,7 +225,7 @@ module Wukong
       maybe_overwrite_output_paths! output_path
       command = runner_command(input_path, output_path)
       $stderr.puts command
-      if ! options[:fake]
+      unless options[:dry_run]
         $stdout.puts `#{command}`
       end
     end
@@ -255,8 +257,24 @@ module Wukong
         #{$0} --run=local  input_hdfs_path output_hdfs_dir    # run the script on local filesystem using unix pipes
         #{$0} --run        input_hdfs_path output_hdfs_dir    # run the script with the mode given in config/wukong*.yaml
         #{$0} --map
-        #{$0} --reduce                                 # dispatch to the mapper or reducer
+        #{$0} --reduce                                        # dispatch to the mapper or reducer
 
+      All flags must precede the input and output paths.
+      Additional flags:
+        --dry_run
+      Hadoop Options (see hadoop documentation)
+        --max_node_map_tasks     => 'mapred.tasktracker.map.tasks.maximum',
+        --max_node_reduce_tasks  => 'mapred.tasktracker.reduce.tasks.maximum',
+        --map_tasks              => 'mapred.map.tasks',
+        --reduce_tasks           => 'mapred.reduce.tasks',
+        --sort_fields            => 'stream.num.map.output.key.fields',
+        --key_field_separator    => 'map.output.key.field.separator',
+        --partition_fields       => 'num.key.fields.for.partition',
+        --output_field_separator => 'stream.map.output.field.separator',
+        --map_speculative        => 'mapred.map.tasks.speculative.execution',
+        --timeout                => 'mapred.task.timeout',
+        --reuse_jvms             => 'mapred.job.reuse.jvm.num.tasks',
+        --ignore_exit_status     => 'stream.non.zero.exit.status.is.failure',
       You can specify as well arbitrary script-specific command line flags; they are added to your options[] hash.
       }
     end