mrflip-wukong 0.1.0

data/examples/size.rb

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+
+module Size
+  #
+  # Feed the entire dataset through wc and sum the results
+  #
+  class Script < Wukong::Script
+    #
+    # Don't implement a wukong script to do something if there's a unix command
+    # that does it faster: just override map_command or reduce_command in your
+    # subclass of Wukong::Script to return the complete command line
+    #
+    def map_command
+      '/usr/bin/wc'
+    end
+
+    # Make all records go to one reducer
+    def default_options
+      super.merge :reduce_tasks => 1
+    end
+  end
+
+  #
+  # Sums the numeric value of each column in its input
+  #
+  class Reducer < Wukong::Streamer::Base
+    attr_accessor :sums
+
+    #
+    # The unix +wc+ command uses whitespace, not tabs, so we'll recordize
+    # accordingly.
+    #
+    def recordize line
+      line.strip.split(/\s+/)
+    end
+
+    #
+    # add each corresponding column in the input
+    #
+    def process *vals
+      self.sums = vals.zip( sums || [] ).map{|val,sum| val.to_i + sum.to_i }
+    end
+
+    #
+    # run through the whole reduction input and then output the total
+    #
+    def stream *args
+      super *args
+      emit sums
+    end
+  end
+end
+
+# Execute the script
+Size::Script.new(
+  nil,
+  Size::Reducer
+  ).run

data/examples/word_count.rb

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+
+module WordCount
+  class Mapper < Wukong::Streamer::LineStreamer
+    #
+    # Split a string into its constituent words.
+    #
+    # This is pretty simpleminded:
+    # * downcase the word
+    # * Split at any non-alphanumeric boundary, including '_'
+    # * However, preserve the special cases of 's or 't at the end of a
+    #   word.
+    #
+    #   tokenize("Jim's dawg won't hunt: dawg_hunt error #3007a4")
+    #   # => ["jim's", "dawd", "won't", "hunt", "dawg", "hunt", "error", "3007a4"]
+    #
+    def tokenize str
+      return [] unless str
+      str = str.downcase;
+      # kill off all punctuation except [stuff]'s or [stuff]'t
+      # this includes hyphens (words are split)
+      str = str.
+        gsub(/[^a-zA-Z0-9\']+/, ' ').
+        gsub(/(\w)\'([st])\b/, '\1!\2').gsub(/\'/, ' ').gsub(/!/, "'")
+      # Busticate at whitespace
+      words = str.strip.split(/\s+/)
+      words.reject!{|w| w.blank? }
+      words
+    end
+
+    #
+    # Emit each word in each line.
+    #
+    def process line
+      tokenize(line).each{|word| yield [word, 1] }
+    end
+  end
+
+  #
+  # Accumulate the sum record-by-record:
+  #
+  class Reducer0 < Wukong::Streamer::Base
+    attr_accessor :key_count
+    def process word, count
+      @last_word ||= word
+      if (@last_word == word)
+        self.key_count += 1
+      else
+        yield [ @last_word, key_count ]
+        @last_word = word
+      end
+    end
+    def stream
+      emit @last_word, key_count
+    end
+  end
+
+  #
+  # You can stack up all the values in a list then sum them at once:
+  #
+  require 'active_support/core_ext/enumerable'
+  class Reducer1 < Wukong::Streamer::ListReducer
+    def finalize
+      yield [ key, values.map(&:last).map(&:to_i).sum ]
+    end
+  end
+
+  #
+  # A bit kinder to your memory manager: accumulate the sum record-by-record:
+  #
+  class Reducer2 < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :key_count
+    def start!(*args)      self.key_count =  0 end
+    def accumulate(*args)  self.key_count += 1 end
+    def finalize
+      yield [ key, key_count ]
+    end
+  end
+
+  #
+  # ... easiest of all, though: this is common enough that it's already included
+  #
+  require 'wukong/streamer/count_keys'
+  class Reducer3 < Wukong::Streamer::CountKeys
+  end
+
+end
+
+# Execute the script
+Wukong::Script.new(
+  WordCount::Mapper,
+  WordCount::Reducer1
+  ).run

data/lib/wukong.rb

@@ -0,0 +1,11 @@
+require 'wukong/boot'
+require 'wukong/extensions'
+require 'wukong/datatypes'
+require 'wukong/logger'
+require 'wukong/bad_record'
+autoload :TypedStruct, 'wukong/typed_struct'
+module Wukong
+  autoload :Dfs,         'wukong/dfs'
+  autoload :Script,      'wukong/script'
+  autoload :Streamer,    'wukong/streamer'
+end

data/lib/wukong/and_pig.rb

@@ -0,0 +1,62 @@
+require 'wukong/and_pig/pig_var'
+require 'wukong/and_pig/as'
+require 'wukong/and_pig/functions'
+require 'wukong/and_pig/operators'
+require 'wukong/and_pig/data_types'
+require 'wukong/and_pig/pig_struct'
+require 'wukong/and_pig/generate'
+require 'wukong/and_pig/symbol'
+require 'wukong/and_pig/utils'
+
+module Wukong
+  #
+  # Wukong::AndPig lets you generate and run pig[http://hadoop.apache.org/pig]
+  # code from within ruby (and interactively, from the +irb+ console).
+  #
+  # It uses the same typed structures you've defined for Wukong to create
+  # pig-types aware commands. For example, the Wukong class
+  #
+  #   class Customer < TypedStruct.new( [:id, Integer],
+  #     [:name, String], [:postal_code, Integer], [:balance, Float] )
+  #   end
+  #
+  # will generate a LOAD command for pig as
+  #
+  #   Customer1.pig_load('q4_reports/customers.tsv').set!
+  #   # => Q4ReportsCustomers2 = LOAD 'q4_reports/customers.tsv'
+  #        AS (id: int, name: chararray, postal_code: int, balance: float) ;
+  #
+  # You can write anonymous chains
+  #
+  #   q1 = Customer1.
+  #     pig_load('q4_reports/customers.tsv').set!.
+  #     distinct.set! ;
+  #   q1.
+  #     group(:by => :postal_code).set!.
+  #     generate([:group, :postal_code], ["COUNT(#{q1.relation})", :customers_per_zip]).set!.
+  #     store!
+  #
+  #   Q4ReportsCustomers35    = LOAD    'q4_reports/customers.tsv' AS (id: int,name: chararray,postal_code: int,balance: float) ;
+  #   Q4ReportsCustomers36    = DISTINCT Q4ReportsCustomers35 ;
+  #   Q4ReportsCustomers37    = GROUP    Q4ReportsCustomers36 BY postal_code ;
+  #   Q4ReportsCustomers38    = FOREACH  Q4ReportsCustomers37 GENERATE
+  #       group AS postal_code,
+  #       COUNT(Q4ReportsCustomers36) AS customers_per_zip ;
+  #
+  # ---------------------------------------------------------------------------
+  #
+  # Note on pig:
+  #
+  # 1) Reverse the order of your tables in your join statement. Pig always
+  #    streams the keys of the last input, (materializing in memory the keys of
+  #    the first), so if one of your inputs has less instances of of a given key
+  #    this may help.
+  #
+  # 2) Reduce the number of maps and reducers per machine and give it all the
+  #    memory you can.
+  #
+  #
+  module AndPig
+  end
+end
+

data/lib/wukong/and_pig/README.textile

@@ -0,0 +1,12 @@
+Wukong::AndPig is a small library to more easily generate code for the
+"Pig":http://hadoop.apache.org/pig data analysis language.  
+
+Wukong::AndPig lets you use the structs from your Wukong scripts to
+generate Pig instructions that know their types and structure -- even through
+multiple pig commands. For example, if you use +FOREACH ... GENERATE+ to select
+only a few of those fields, Wukong::AndPig will know that the result has only
+those fields.
+
+We're still trying to figure out if this is a stupid and crazy idea, or just a
+crazy idea: Yeah, we're using a functional/OO scripting language to generate code for an
+imperative query language that generates Java code for ad-hoc map-reduce operations.

data/lib/wukong/and_pig/as.rb

@@ -0,0 +1,37 @@
+class AS
+  attr_accessor :expr, :name, :type, :ref, :options
+  def initialize expr, name=nil, type=nil, ref=nil, *option_flags
+    case expr
+    when AS
+      self.expr = expr.expr
+      self.name = expr.name
+      self.type = expr.type
+      self.ref  = expr.ref
+      self.options = expr.options
+    end
+    self.expr ||= expr
+    self.name = name if name
+    self.type = type if type
+    self.ref  = ref  if ref
+    self.options ||= { }
+    option_flags.each{|option| self.options[option] = true }
+  end
+
+  def to_s
+    clause  = "%-30s \t" % [ref, expr].compact.join('::')
+    if name
+      clause << "AS #{name}"      unless options[:skip_name]
+      clause << ":#{type.typify}" unless ((!type) || options[:skip_type])
+    end
+    clause
+  end
+
+  def self.[] *args
+    self.new *args
+  end
+
+  # Useful for feeding back into TypedStruct
+  def name_type
+    [name, type]
+  end
+end

data/lib/wukong/and_pig/data_types.rb

@@ -0,0 +1,30 @@
+# == SimpleDataTypes ==
+# int
+# long
+# double
+# arrays
+# chararray
+# bytearray
+#
+# == ComplexDataTypes ==
+# tuple
+# bag
+# map
+
+module Wukong
+  module AndPig
+    class PigVar
+
+    end
+  end
+end
+
+# class ScalarInteger  < TypedStruct.new [
+#     [:count,    Integer  ],
+#   ]
+#   include Wukong::AndPig::PigEmitter
+#   def self.load_scalar path
+#     var = super path
+#     var.to_i
+#   end
+# end

data/lib/wukong/and_pig/functions.rb

@@ -0,0 +1,50 @@
+
+# == Built-in Functions
+# EvalFunctions
+# AVG
+# CONCAT
+# COUNT
+# DIFF
+# MIN
+# MAX
+# SIZE
+# SUM
+# TOKENIZE
+
+# == NullOperators
+# isnull
+# isnotnull
+#
+# == BooleanOperators
+# and
+# or
+# not
+#
+# == DereferenceOperators
+# tupledereference.
+# mapdereference#
+#
+# == SignOperators
+# positive+
+# negative-
+#
+# == CastOperators
+# (type)$0
+# (type)alias
+#
+# == ArithmeticOperators
+# addition+
+# subtraction-
+# multiplication*
+# division/
+# modulo%
+# bincond?
+#
+# == ComparisonOperators
+# Equal==
+# notequal!=
+# lessthan<
+# greaterthan>
+# lessthanorequalto<=
+# greaterthanorequalto>=
+# patternmatchingmatches

data/lib/wukong/and_pig/generate.rb

@@ -0,0 +1,85 @@
+require 'wukong/and_pig/generate/variable_inflections'
+
+module Wukong
+  module AndPig
+
+    mattr_accessor :comments
+    self.comments = true
+    # send output to stdout or to captured pig instance
+    mattr_accessor :emit_dest
+    # full pathname to the pig executable
+    PIG_EXECUTABLE = '/usr/local/bin/pig'
+
+    def self.finish
+      PigVar.pig_in_poke.close if PigVar.pig_in_poke.respond_to?(:close)
+    end
+
+    #
+    # All the embarrassing magick to pretend ruby symbols are pig relations
+    #
+    class PigVar
+
+      # Output a command
+      def self.emit cmd, semicolon=true
+        cmd = cmd + ' ;' if semicolon
+        case Wukong::AndPig.emit_dest
+        when :captured
+          pig_in_poke.puts(cmd)
+          pig_in_poke.flush
+          puts pig_in_poke.gets
+        else
+          puts(cmd)
+        end
+      end
+
+      # generate the code
+      def self.emit_setter relation, rval
+        emit "%-23s\t= %s" % [relation, rval.cmd]
+        rval
+      end
+
+      # generate the code
+      def self.emit_imperative imperative, *rest
+        cmd_part = "%-14s \t" % imperative
+        arg_part = rest.map{|s| "%14s" % s.to_s }.join(" \t")
+        emit cmd_part+arg_part
+        rest.first
+      end
+
+      def self.pig_in_poke
+        return @pig_in_poke if @pig_in_poke
+        case Wukong::AndPig.emit_dest
+        when :captured
+          @pig_in_poke = IO.popen(PIG_EXECUTABLE, "w+")
+          @pig_in_poke.sync = true
+          @pig_in_poke
+        else @pig_in_poke = $stdout
+        end
+      end
+
+      #
+      # Reset the captured pig instance
+      #
+      def self.reset_pig_in_poke!
+        begin pig_in_poke.close ; rescue nil ; end
+        @pig_in_poke = nil
+      end
+
+      def set!
+        self.class.emit_setter(relation, self)
+      end
+
+      #
+      # Emit a comment
+      # skips if Wukong::AndPig.comments is false
+      #
+      def self.rem comment
+        return unless Wukong::AndPig.comments
+        PigVar.emit comment.gsub(/(^|\n)(#([\t ]|$))?/, "\n--  "), false
+      end
+    end
+
+  end
+end
+
+