mrflip-wukong 0.1.0

data/lib/wukong/and_pig/operators/load_store.rb

@@ -0,0 +1,65 @@
+# == Load/StoreFunctions ==
+# BinaryDeserializer
+# BinarySerializer
+# BinStorage
+# PigStorage
+# PigDump
+# TextLoader
+
+module Wukong
+  module AndPig
+    class PigVar
+      #===========================================================================
+      #
+      # The "LOAD" pig expression:
+      #   MyRelation = LOAD 'my_relation.tsv' AS (attr_a: int, attr_b: chararray) ;
+      #
+      # The AS type spec is generated from klass
+      #
+      def self.pig_load rel, klass, options={ }
+        filename = options[:filename] || default_filename(rel)
+        self.set rel, self.new(klass, rel, "LOAD    '#{filename}' AS #{klass.typify(options[:has_rsrc])}")
+        if options[:has_rsrc]
+          lval = self[rel]
+          lval.generate lval, *lval.fields
+        end
+        rel
+      end
+
+      #===========================================================================
+      #
+      #
+      # The "STORE" pig imperative:
+      #   STORE Relation INTO 'filename'
+      # If no filename is given, the relation's name is used
+      #
+      def store filename=nil
+        filename ||= default_filename
+        self.class.emit "STORE %-19s INTO    '%s'" % [relation, filename]
+        self
+      end
+
+      # Store the relation, removing the existing file
+      def store! filename=nil
+        filename ||= default_filename
+        rmf!  filename
+        mkdir File.dirname(filename)
+        store filename
+      end
+
+      # Force a store to disk, then load (so all calculations proceed from there)
+      def checkpoint! options={}
+        options = options.reverse_merge :filename => default_filename
+        store!   options[:filename]
+        self.class.pig_load(self.name, klass, options)
+      end
+
+      def default_filename
+        self.class.default_filename self.name
+      end
+      def self.default_filename name
+        File.join(working_dir, name.to_s)
+      end
+    end
+  end
+end

data/lib/wukong/and_pig/operators/meta.rb

@@ -0,0 +1,42 @@
+# == DiagnosticOperators
+# describe
+# dump
+# explain
+# illustrate
+# == UDFStatements
+# define
+# register
+
+module Wukong
+  module AndPig
+    class PigVar
+      # DESCRIBE pig imperative
+      def describe
+        self.class.describe self
+      end
+      def self.describe rel
+        emit %Q{ -- PREDICTED: #{rel.klass.typify} }
+        simple_declaration :describe, rel.relationize
+        rel
+      end
+
+      # DUMP pig imperative
+      def dump()       simple_operation :dump        end
+
+      # EXPLAIN pig imperative
+      def explain()    simple_operation :explain     end
+
+      # ILLUSTRATE pig imperative
+      def illustrate() simple_operation :illustrate  end
+
+
+      def self.define pig_alias, *args
+        emit_imperative :DEFINE, pig_alias, args
+      end
+
+      def self.register path_to_jar
+        emit_imperative :REGISTER, path_to_jar
+      end
+    end
+  end
+end

data/lib/wukong/and_pig/operators/relational.rb

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+# == RelationalOperators
+#
+# GROUP, COGROUP, JOIN see groupies.rb
+# CROSS see
+
+# distinct
+# filter
+# limit
+# order
+# split
+# union
+
+#
+# stream
+# load
+# store
+#
+module Wukong
+  module AndPig
+    class PigVar
+
+      # ===========================================================================
+      #
+      # Options
+      #
+      def self.parallelize! str, options
+        str << " PARALLEL #{options[:parallel]}" if options[:parallel]
+      end
+
+      # ===========================================================================
+      #
+      # DISTINCT
+      #
+      def distinct lval, options={}
+        self.class.distinct lval, self, options
+      end
+
+      def self.distinct lval, rel, options={ }
+        cmd_str = rel.relationize
+        parallelize! cmd_str, options
+        simple_operation lval, rel, :distinct, cmd_str
+      end
+
+      # ===========================================================================
+      #
+      # FILTER
+      #
+      def filter by_str
+        new_in_chain klass, "FILTER   #{relation} BY #{by_str}"
+      end
+      def self.filter lval, rel, by_str
+        simple_operation    lval, rel, "FILTER", "#{rel.relation} BY #{by_str}"
+      end
+
+      # ===========================================================================
+      #
+      # LIMIT
+      #
+      def limit n
+        new_in_chain klass, "LIMIT   #{relation} #{n}"
+      end
+
+      # ===========================================================================
+      #
+      # ORDER
+      #
+      # alias = ORDER alias BY { * [ASC|DESC] |
+      #           field_alias [ASC|DESC] [, field_alias [ASC|DESC] …]
+      #           } [PARALLEL n];
+      #
+      def order cmd_str, options={}
+        result = new_in_chain klass, "ORDER    #{relation} BY #{cmd_str}"
+        parallelize! result.cmd, options
+        result
+      end
+
+      # ===========================================================================
+      #
+      # SPLIT
+      #
+      # SPLIT alias INTO alias IF expression, alias IF expression [, alias IF expression …];
+      #
+      #
+      def split relation_tests={}
+        split_str = relation_tests.map do |out_rel, test|
+          "#{out_rel} IF #{test}"
+        end.join(", ")
+        new_in_chain klass, "SPLIT    #{relation} INTO #{split_str}"
+      end
+
+      # ===========================================================================
+      #
+      # CROSS
+      #
+      def cross *relations
+        options = relations.extract_options!
+        raise CrossArgumentError unless relations.length >= 1
+        relations_str = [self, *relations].map(&:relation).join(", ")
+        result = new_in_chain relations.first.klass, "CROSS    #{relations_str}"
+        parallelize! result.cmd, options
+        result
+      end
+
+      # ===========================================================================
+      #
+      # UNION
+      #
+      # def self.union *relations
+      #   raise UnionArgumentError unless relations.length >= 2
+      #   new_in_chain relations.first.klass, "UNION #{relations}"
+      # end
+
+      # UNION as method
+      def union lval, *relations
+        self.class.union lval, [self]+relations
+      end
+
+      def self.union lval, *relations
+        raise UnionArgumentError unless relations.length >= 2
+        relations_str = relations.map(&:relation).join(", ")
+        simple_operation lval, relations.first, :union, relations_str
+      end
+
+    end
+    CrossArgumentError = ArgumentError.new("CROSS requires at least two relations. Heh heh: relations.")
+    UnionArgumentError = ArgumentError.new("UNION requires at least two relations. Heh heh: relations.")
+  end
+end

data/lib/wukong/and_pig/pig_struct.rb

@@ -0,0 +1,48 @@
+module Wukong
+  module PigStructMethods
+    module ClassMethods
+      #
+      # Pig type string --
+      # the pig type strings for each sub-element.
+      #
+      def typify has_rsrc=nil
+        vars_str = members.zip(mtypes).map do |attr, mtype|
+          "%s: %s" % [attr, mtype.typify]
+        end
+        vars_str = ["rsrc: chararray"] + vars_str if has_rsrc
+        "(#{vars_str.join(', ')})"
+      end
+
+      #
+      #
+      #
+      def pig_load rel, *args
+        Wukong::AndPig::PigVar.pig_load rel, self, *args
+      end
+
+      #
+      # Returns type for a fieldspec
+      #
+      def field_type field
+        case field
+        when Symbol             then members_types[field]
+        # when Array
+        #   if field.length > 1   then members_types[field.first].field_type(field[1..-1])
+        #   else                       field_type field.first
+        #   end
+        end
+      end
+
+    end
+    def self.included base
+      base.extend ClassMethods
+    end
+  end
+end
+
+Struct.class_eval do
+  include Wukong::PigStructMethods
+  def self.mtypes
+    members
+  end
+end

data/lib/wukong/and_pig/pig_var.rb

@@ -0,0 +1,95 @@
+module Wukong
+  module AndPig
+
+    #
+    # Make a PigVar understand the struct it describes
+    #
+    class PigVar
+      attr_accessor :klass, :name, :cmd
+      cattr_accessor :working_dir ; self.working_dir = '.'
+      def initialize klass, name, cmd
+        self.klass    = klass
+        self.name     = name
+        self.cmd      = cmd
+      end
+
+      # Sugar for PigVar.new_relation
+      def self.[]= name, *args
+        set name, *args
+      end
+      # Sugar for PigVar.new_relation
+      def self.[] name
+        PIG_SYMBOLS[name]
+      end
+
+      # extract a field from an alias
+      def _ field
+        as_name = [name, field].join("_").to_sym
+        AS["#{relationize}.(#{field})", as_name, Bag.new([field, field_type(field)]), nil, :skip_type]
+      end
+
+
+      def self.set name, rval
+        PIG_SYMBOLS[name] = rval
+        rval.name = name
+        emit_setter rval.relation, rval
+      end
+
+      def relation
+        name.relationize
+      end
+      alias_method :relationize, :relation
+
+      #
+      # Create a name for a new anonymous relation
+      #
+      def self.anon slug
+        idx = (Wukong::AndPig.anon_var_idx += 1)
+        "anon_#{slug}_#{idx}_".to_sym
+      end
+      # Create a name building off this one
+      def anon
+        slug = name.to_s.gsub(/^anon_/,'').gsub(/_\d+_$/,'')
+        self.class.anon slug
+      end
+
+      #
+      def new_in_chain lval, l_klass, l_cmd
+        rval = self.class.new l_klass, lval, l_cmd
+        self.class.set lval, rval
+      end
+
+      # Delegate to klass
+      def field_type *args
+        self.klass.field_type *args
+      end
+
+      # Fields in this relation
+      def fields
+        klass.members.map(&:to_sym)
+      end
+
+      #
+      # Side-effect free operation
+      #
+      def simple_operation op
+        self.class.emit  "#{op.to_s.upcase} #{relation}"
+        self
+      end
+
+      def self.simple_operation lval, rel, op, r_str
+        cmd  = "%-8s %s" % [op.to_s.upcase, r_str]
+        rval = new(rel.klass, lval, cmd)
+        set lval, rval
+      end
+
+      def self.simple_declaration op, r_str
+        cmd  = "%-8s %s" % [op.to_s.upcase, r_str]
+        emit cmd
+      end
+
+    end
+  end
+end
+
+

data/lib/wukong/and_pig/symbol.rb

@@ -0,0 +1,29 @@
+module Wukong
+  module AndPig
+    PIG_SYMBOLS = { }
+    mattr_accessor :anon_var_idx
+    self.anon_var_idx = 0
+  end
+end
+
+
+Symbol.class_eval do
+  def << relation
+    case
+    when relation.is_a?(Wukong::AndPig::PigVar)
+      Wukong::AndPig::PigVar.new_relation(self, relation)
+    when relation.is_a?(Symbol) && (pig_var = Wukong::AndPig::PIG_SYMBOLS[relation])
+      Wukong::AndPig::PigVar.new_relation(self, pig_var)
+    else raise "Don't know how to pigify RHS #{relation.inspect}"
+    end
+  end
+
+  def method_missing method, *args
+    pig_var = Wukong::AndPig::PIG_SYMBOLS[self]
+    if pig_var && pig_var.respond_to?(method)
+      pig_var.send(method, *args)
+    else
+      super method, *args
+    end
+  end
+end

data/lib/wukong/bad_record.rb

@@ -0,0 +1,18 @@
+#
+# Easily serialize bad records in-band, for later analysis or to discard if
+# neglectable.
+#
+# You can instantiate this as
+#  success = do_stuff_to record
+#  if ! success
+#    return BadRecord.new("do_stuff_to-failed", record)
+#  end
+#
+class BadRecord < Struct.new(
+    :errors,
+    :record
+    )
+  def initialize errors='', *record_fields
+    super errors, record_fields
+  end
+end

data/lib/wukong/boot.rb

@@ -0,0 +1,47 @@
+module Wukong
+
+  # ---------------------------------------------------------------------------
+  #
+  # Default options for Wukong
+  #   http://github.com/infochimps/wukong
+  #
+  # If you set an environment variable WUKONG_CONFIG, *or* if the file
+  # $HOME/.wukong.rb exists, that file will be +require+'d as well.
+  #
+  # Important values to set:
+  #
+  # * Wukong::CONFIG[:hadoop_home] --
+  #   Path to root of hadoop install. If your hadoop runner is
+  #     /usr/local/share/hadoop/bin/hadoop
+  #   then your hadoop_home is
+  #     /usr/local/share/hadoop.
+  #   You can also set a
+  #
+  # * Wukong::CONFIG[:default_run_mode] -- Whether to run using hadoop (and
+  #   thus, requiring a working hadoop install), or to run in local mode
+  #   (script --map | sort | script --reduce)
+  #
+  CONFIG = {
+    # Run as local or as hadoop?
+    :default_run_mode => 'hadoop',
+
+    # The command to run when a nil mapper or reducer is given.
+    :default_mapper   => '/bin/cat',
+    :default_reducer  => '/bin/cat',
+
+    # Anything in HADOOP_OPTIONS_MAP (see lib/wukong/script/hadoop_command.rb)
+    :runner_defaults => {
+    },
+  }
+
+  def self.config_options
+    # # override with site-specific options
+    site_config_filename = ENV['WUKONG_CONFIG'] || (ENV['HOME'].to_s+'/.wukong.rb')
+    require site_config_filename.gsub(/\.rb$/,'') if File.exists?(site_config_filename)
+
+    # try to guess a hadoop_home if none given
+    Wukong::CONFIG[:hadoop_home] ||= ENV['HADOOP_HOME'] || '/usr/lib/hadoop'
+  end
+  self.config_options
+end
+