wukong 0.1.1

data/lib/wukong/and_pig/generate.rb

@@ -0,0 +1,85 @@
+require 'wukong/and_pig/generate/variable_inflections'
+
+module Wukong
+  module AndPig
+
+    mattr_accessor :comments
+    self.comments = true
+    # send output to stdout or to captured pig instance
+    mattr_accessor :emit_dest
+    # full pathname to the pig executable
+    PIG_EXECUTABLE = '/usr/local/bin/pig'
+
+    def self.finish
+      PigVar.pig_in_poke.close if PigVar.pig_in_poke.respond_to?(:close)
+    end
+
+    #
+    # All the embarrassing magick to pretend ruby symbols are pig relations
+    #
+    class PigVar
+
+      # Output a command
+      def self.emit cmd, semicolon=true
+        cmd = cmd + ' ;' if semicolon
+        case Wukong::AndPig.emit_dest
+        when :captured
+          pig_in_poke.puts(cmd)
+          pig_in_poke.flush
+          puts pig_in_poke.gets
+        else
+          puts(cmd)
+        end
+      end
+
+      # generate the code
+      def self.emit_setter relation, rval
+        emit "%-23s\t= %s" % [relation, rval.cmd]
+        rval
+      end
+
+      # generate the code
+      def self.emit_imperative imperative, *rest
+        cmd_part = "%-14s \t" % imperative
+        arg_part = rest.map{|s| "%14s" % s.to_s }.join(" \t")
+        emit cmd_part+arg_part
+        rest.first
+      end
+
+      def self.pig_in_poke
+        return @pig_in_poke if @pig_in_poke
+        case Wukong::AndPig.emit_dest
+        when :captured
+          @pig_in_poke = IO.popen(PIG_EXECUTABLE, "w+")
+          @pig_in_poke.sync = true
+          @pig_in_poke
+        else @pig_in_poke = $stdout
+        end
+      end
+
+      #
+      # Reset the captured pig instance
+      #
+      def self.reset_pig_in_poke!
+        begin pig_in_poke.close ; rescue nil ; end
+        @pig_in_poke = nil
+      end
+
+      def set!
+        self.class.emit_setter(relation, self)
+      end
+
+      #
+      # Emit a comment
+      # skips if Wukong::AndPig.comments is false
+      #
+      def self.rem comment
+        return unless Wukong::AndPig.comments
+        PigVar.emit comment.gsub(/(^|\n)(#([\t ]|$))?/, "\n--  "), false
+      end
+    end
+
+  end
+end
+
+

data/lib/wukong/and_pig/generate/variable_inflections.rb

@@ -0,0 +1,82 @@
+String.class_eval do
+  #
+  # Generate relation name from a handle
+  #
+  def relationize() camelize end
+end
+Symbol.class_eval do
+  #
+  # Generate relation name from a handle
+  #
+  def relationize
+    to_s.relationize
+  end
+end
+
+Object.class_eval do
+  def typify() self.class ; end
+
+  def symbolize
+    self.to_s.underscore.gsub(%r{.*/}, '').to_sym
+  end
+end
+
+class << Integer ; def typify() 'int'           end ; end
+class << Bignum  ; def typify() 'long'          end ; end
+class << Float   ; def typify() 'float'         end ; end
+class << String  ; def typify() 'chararray'     end ; end
+class << Symbol  ; def typify() self            end ; end
+class << Date    ; def typify() 'long'          end ; end
+
+# Array.class_eval do
+#   def typify()
+#     "{ #{ map{|f,t| "#{f}: #{t.typify}"} } }"
+#   end
+# end
+# class Tuple
+#   attr_accessor :contents
+#   def initialize *args
+#     self.contents = args
+#   end
+#   def typify
+#     "bag { #{ contents.map{|f,t| "#{f}: #{t.typify}"} } }"
+#   end
+#   #
+#   # Sugar for creating a new bag. The following are equivalent:
+#   #
+#   #   Bag[:foo]
+#   #   Bag.new :foo
+#   #
+#   def self.[] *args
+#     new *args
+#   end
+# end
+
+module BagMethods
+  module ClassMethods
+    #
+    # Pig type string --
+    # the pig type strings for each sub-element.
+    #
+    def typify
+      vars_str = members.zip(mtypes).map do |attr, mtype|
+        "%s: %s" % [attr, mtype.typify]
+      end
+      "{ #{vars_str.join(', ')} }"
+    end
+  end
+  def self.included base
+    base.extend ClassMethods
+  end
+end
+
+class Bag < TypedStruct
+  def self.new *args
+    bag = super *args
+    bag.class_eval{ include BagMethods }
+  end
+  def self.[] *args
+    new *args
+  end
+end
+

data/lib/wukong/and_pig/junk.rb

@@ -0,0 +1,51 @@
+
+
+module Wukong
+  module AndPig
+
+    #
+    # Load the main class definitions
+    #
+    def self.init_load
+      puts File.open(PIG_DEFS_DIR+"/init_load.pig").read
+    end
+
+
+
+
+    #
+    # OK we're going to cheat here:
+    # just cat the file in, and treat it as a scalar
+    #
+    def load_scalar path
+      # var = `hadoop dfs -cat '#{path}/part-*' | head -n1 `.chomp
+      var = "636"
+    end
+
+
+
+    def count_distinct dest_rel, attr, group_by
+      distincted =
+        generate(temp_rel(dest_rel), attr).
+        distinct(temp_rel(dest_rel), :parallel => 10)
+      distincted.
+        group(   temp_rel(dest_rel), group_by).
+        foreach( dest_rel,  "GENERATE COUNT(#{distincted.relation}.#{attr}) AS n_#{attr}")
+    end
+
+    #
+    # Group a relation into bins, and return the counts for each bin
+    # * dest_rel - Relation to store
+    #   {bin,
+    #
+    def histogram dest_rel, bin_attr, bin_expr=nil
+      bin_expr ||= bin_attr
+      bin_name   = "#{bin_attr}_bin"
+      binned     = foreach(temp_rel(dest_rel), "GENERATE #{bin_expr} AS #{bin_name}")
+      binned.      group(  temp_rel(dest_rel), :by => bin_name).
+        foreach(         dest_rel,  "GENERATE group AS #{bin_name}, COUNT(#{binned.relation}) AS #{bin_attr}_count")
+    end
+
+
+  end
+end

data/lib/wukong/and_pig/operators.rb

@@ -0,0 +1,8 @@
+require 'wukong/and_pig/operators/evaluators'
+require 'wukong/and_pig/operators/foreach'
+require 'wukong/and_pig/operators/groupies'
+require 'wukong/and_pig/operators/load_store'
+require 'wukong/and_pig/operators/meta'
+require 'wukong/and_pig/operators/relational'
+require 'wukong/and_pig/operators/file_methods'
+require 'wukong/and_pig/operators/compound'

data/lib/wukong/and_pig/operators/compound.rb

@@ -0,0 +1,29 @@
+#
+# The FOREACH relational operator
+#
+module Wukong
+  module AndPig
+    class PigVar
+      #
+      # Select all elements in the source relation that match on the selecting relation,
+      # creating a relation with the same type as the source relation.
+      #
+      # For example, 
+      #
+      #   PV.isolate :isolated_cvals, :my_ids, :id, :my_complicated_values, :id
+      #
+      # returns a relation IsolatedCvals, whose type is identical to
+      # MyComplicatedValues' type, with only the elements having an id also
+      # presend in MyIds.
+      #
+      #
+      def self.isolate lval, on, on_field, from, from_field, options={ }
+        joined   = join anon(lval), on => on_field, from => from_field, :parallel => options.delete(:parallel)
+        isolated = joined.generate lval, { "'#{from}'" => :rsrc}, *PV[from].fields.map{|field| [from, field]}
+        isolated.klass = from.klass
+        isolated
+      end
+
+    end
+  end
+end

data/lib/wukong/and_pig/operators/evaluators.rb

@@ -0,0 +1,7 @@
+module Wukong
+  module AndPig
+    class PigVar
+
+    end
+  end
+end

data/lib/wukong/and_pig/operators/execution.rb

@@ -0,0 +1,15 @@
+module Wukong
+  module AndPig
+    class PigVar
+
+      # ===========================================================================
+      #
+      # STREAM
+      #
+      def stream options={}
+        new_in_chain klass, "STREAM #{relation}"
+      end
+    end
+  end
+end
+

data/lib/wukong/and_pig/operators/file_methods.rb

@@ -0,0 +1,29 @@
+module Wukong
+  module AndPig
+    class PigVar
+      # ===========================================================================
+      #
+      # Pig expressions
+      #
+
+      #
+      def dfs cmd, filename
+        # note == no '' on path
+        self.class.emit "%-23s\t           %s" % [cmd, filename]
+      end
+      #
+      # remove the stored file
+      #
+      def rmf! filename
+        dfs :rmf, filename
+      end
+
+      #
+      #
+      #
+      def mkdir filename
+        dfs :mkdir, filename
+      end
+    end
+  end
+end

data/lib/wukong/and_pig/operators/foreach.rb

@@ -0,0 +1,98 @@
+#
+# The FOREACH relational operator
+#
+module Wukong
+  module AndPig
+    class PigVar
+
+      # ===========================================================================
+      #
+      # FOREACH
+      #
+      def generate lval,  *field_specs
+        gen_clauses = field_specs.map{|field_spec| parse_gen_clause(field_spec)}.flatten
+        l_klass     = TypedStruct.new(* gen_clauses.map(&:name_type))
+        l_cmd       = "FOREACH  #{self.relation} GENERATE\n  #{gen_clauses.join(",\n  ")}"
+        new_in_chain(lval, l_klass, l_cmd)
+      end
+
+      #
+      # for a list of GENERATE args, we need
+      #
+      # * gen_clauses, the clause to stuff into the GENERATE line
+      #     gen_expr AS gen_field_name: gen_field_type
+      #
+      # * new_types, the resulting types for each
+      #
+      # gen_expr common cases include
+      #
+      #   field
+      #   Rel::field
+      #   Rel.(field)
+      #   "ComplicatedExpression"
+      #
+      #
+      # field_attrs
+      #
+      #
+      def parse_gen_clause field_spec
+        case field_spec
+        when AS
+          field_spec
+        when Symbol
+          AS[field_spec, field_spec, field_type(field_spec)];
+        when Array
+          alias_in, field_in, name, type = field_spec
+          name      ||= field_in
+          type        = alias_in.field_type(field_in)
+          AS[field_in, name, type, alias_in.relationize]
+        when Hash
+          field_spec.map do |field_in, field_out|
+            AS[field_in, field_out, field_type(field_in)]
+          end
+        else raise "Don't know how to specify type for #{field_specs.inspect}"
+        end
+      end
+    end
+  end
+end
+
+
+
+
+
+
+
+          # # when Array
+          # #   unless [2,3].include?(field_spec.length) then raise "Complex fields must be (field_spec, as_name) or (field_spec, as_name, as_type)" end
+          # #   field_expr, field_attr, field_type = field_spec
+          # #   field_as   = field_attr.is_a?(Array) ? "(#{field_attr.join(", ")})" : field_attr
+          # #   gen_clauses << "#{field_expr} AS #{field_as}"
+          # #   field_attrs << [field_attr, field_type || klass.members_types[field_expr]]
+
+      # def prelimify *field_specs
+      #   gen_clauses = []
+      #   field_attrs = []
+      #   field_specs.map do |field_spec|
+      #     unless field_spec.length == 2 then raise "Complex fields must be a pair (field_spec, as_name)" end
+      #     field_expr, field_attr = field_spec
+      #     gen_clauses << "#{field_expr}"
+      #     field_attrs += [field_attr].flatten
+      #   end
+      #   [ gen_clauses, field_attrs ]
+      # end
+      #
+      # # def generate *args
+      # #   gen_clauses, field_attrs = self.class.fieldify *args
+      # #   l_klass = TypedStruct.new(*field_attrs)
+      # #   new_in_chain l_klass, "FOREACH  #{relation} GENERATE\n    #{gen_clauses.join(",\n    ")}"
+      # # end
+      #
+      # def foreach *args
+      #   generate_clause = args.pop
+      #   prelim_exprs, prelim_attrs = prelimify *args
+      #   prelims = prelim_exprs.zip(prelim_attrs).map{|e,a| "#{a} = #{e}" }.join(";\n    ")+";"
+      #   gen_clauses, field_attrs   = fieldify *generate_clause
+      #   l_klass = TypedStruct.new(*field_attrs)
+      #   new_in_chain l_klass, %Q{FOREACH  #{relation} {\n    #{prelims}\n  GENERATE\n    #{gen_clauses.join(",\n    ")} ; } }
+      # end

data/lib/wukong/and_pig/operators/groupies.rb

@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+#
+# The FOREACH relational operator
+#
+module Wukong
+  module AndPig
+    class PigVar
+
+      #===========================================================================
+      #
+      # GROUP and COGROUP
+      #
+
+      #
+      # COGROUP - Groups the data in two or more relations.
+      #
+      # == Syntax
+      #
+      #   alias  = COGROUP alias1 BY field_alias [INNER | OUTER],
+      #                    aliasN BY field_alias [INNER | OUTER] [PARALLEL n] ;
+      #
+      # == Structure
+      #
+      #   { group, <structure of alias1>, <structure of alias2>, ... }
+      #
+      # == Terms
+      #
+      # * alias         The name a relation.
+      #
+      # * field_alias The name of one or more fields in a relation.  If multiple
+      #                 fields are specified, separate with commas and enclose
+      #                 in parentheses. For example, X = COGROUP A BY (f1, f2);
+      #
+      #                 The number of fields specified in each BY clause must
+      #                 match. For example, X = COGROUP A BY (a1,a2,a3), B BY
+      #                 (b1,b2,b3);
+      #
+      # * BY            Keyword.
+      #
+      # * INNER         Eliminate NULLs on that grouping
+      # * OUTER         Do not eliminate NULLs on that grouping (default)
+      #
+      # * PARALLEL n -- Increase the parallelism of a job by specifying the
+      #                 number of reduce tasks, n. The optimal number of
+      #                 parallel tasks depends on the amount of memory on each
+      #                 node and the memory required by each of the tasks. To
+      #                 determine n, use the following as a general guideline:
+      #
+      #                     n = (nr_nodes - 1) * 0.45 * nr_GB
+      #
+      #                 where nr_nodes is the number of nodes used and nr_GB is
+      #                 the amount of physical memory on each node.
+      #
+      #                 Note the following:
+      #                 - Parallel only affects the number of reduce tasks. Map
+      #                   parallelism is determined by the input file, one map
+      #                   for each HDFS block.
+      #                 - If you don’t specify parallel, you still get the same
+      #                   map parallelism but only one reduce task.
+      #
+      # == Usage
+      #
+      # The COGOUP operator groups the data in two or more relations based on
+      # the common field values.
+      #
+      # Note: The COGROUP and JOIN operators perform similar functions. COGROUP
+      # creates a nested set of output tuples while JOIN creates a flat set of
+      # output tuples with NULLs eliminated.
+      #
+      # == Examples
+      #
+      # Suppose we have two relations, A and B.
+      #
+      # A: (owner:chararray, pet:chararray)
+      # ---------------
+      # (Alice, cat)
+      # (Alice, goldfish)
+      # (Alice, turtle)
+      # (Bob,   cat)
+      # (Bob,   dog)
+      #
+      # B: (friend1:chararray, friend2:charrarray)
+      # ---------------------
+      # (Cindy, Alice)
+      # (Mark, Alice)
+      # (Paul, Bob)
+      # (Paul, Jane)
+      #
+      # In this example tuples are co-grouped using field “owner” from relation
+      # A and field “friend2” from relation B as the key fields. The DESCRIBE
+      # operator shows the schema for relation X, which has two fields, "group"
+      # and "A" (for an explanation, see GROUP).
+      #
+      #   X = COGROUP A BY owner, B BY friend2;
+      #   DESCRIBE X;
+      #
+      #    X: {group: chararray,
+      #        A: {owner:   chararray,pet:     chararray},
+      #        B: {friend1: chararray,friend2: chararray}}
+      #
+      # Relation X looks like this. A tuple is created for each unique key
+      # field. The tuple includes the key field and two bags. The first bag is
+      # the tuples from the first relation with the matching key field. The
+      # second bag is the tuples from the second relation with the matching key
+      # field. If no tuples match the key field, the bag is empty.
+      #
+      #   (Alice, {(Alice, turtle), (Alice, goldfish), (Alice, cat)},
+      #           {(Cindy, Alice), (Mark, Alice)})
+      #   (Bob,   {(Bob, dog), (Bob, cat)},
+      #           {(Paul, Bob)})
+      #   (Jane,  {},
+      #           {(Paul, Jane)})
+      #
+      # In this example tuples are co-grouped and the INNER keyword is used to
+      # ensure that only bags with at least one tuple are returned.
+      #
+      #   X = COGROUP A BY owner INNER, B BY friend2 INNER;
+      #
+      # Relation X looks like this.
+      #
+      #   (Alice, {(Alice, turtle), (Alice, goldfish), (Alice, cat)},
+      #           {(Cindy, Alice), (Mark, Alice)})
+      #   (Bob,   {(Bob, dog), (Bob, cat)},
+      #           {(Paul, Bob)})
+      #
+      # In this example tuples are co-grouped and the INNER keyword is used
+      # asymmetrically on only one of the relations.
+      #
+      #   X = COGROUP A BY owner, B BY friend2 INNER;
+      #
+      # Relation X looks like this.
+      #
+      #   (Alice, {(Alice, turtle), (Alice, goldfish), (Alice, cat)},
+      #           {(Cindy, Alice), (Mark, Alice)})
+      #   (Bob,   {(Bob, dog), (Bob, cat)},
+      #           {(Paul, Bob)})
+      #   (Jane,  {},
+      #           {(Paul, Jane)})
+      #
+      #
+      def group group_by
+        l_klass   = l_klass_for_group group_by
+        by_clause = self.class.make_by_clause(group_by)
+        new_in_chain anon, l_klass, "GROUP    #{relation} #{by_clause}"
+      end
+
+      def self.make_by_clause by_spec
+        case by_spec
+        when Array      then 'BY ' + by_spec.join(", ")
+        when :all       then 'ALL'
+        when Symbol     then "BY #{by_spec}"
+        when String     then by_spec
+        when Hash       then make_by_clause(by_spec[:by])
+        else raise "Don't know how to group on #{by_spec.inspect}"
+        end
+      end
+      def types_for_fields field
+        klass.members_types[field]
+      end
+      def l_klass_for_group group_by
+        self.class.l_klass_for_group group_by, self
+      end
+      def self.l_klass_for_group group_by, *rels
+        TypedStruct.new(
+          [:group,       rels.first.types_for_fields(group_by)],
+          *rels.map{|rel| [rel.relation, rel.klass] }
+          )
+      end
+
+      #
+      # COGROUP pig expression:
+      #   UserPosts = COGROUP Posts BY user_id, Users BY user_id ;
+      #
+      def self.cogroup lval, *by
+        by_clause = by.map do |relation, group_by, as|
+          "%s %s" % [relation.relation, make_by_clause(group_by)]
+        end.join(", ")
+        l_klass  = l_klass_for_group by[0][1], *by.map(&:first)
+        rval = new l_klass, lval, "COGROUP    #{by_clause}"
+        set lval, rval
+      end
+
+      def cogroup *args
+        self.class.cogroup self, *args
+      end
+
+
+      # ===========================================================================
+      #
+      # JOIN
+      #
+      def self.klass_from_join by
+        klasses = by.map(&:first)
+        TypedStruct.new(*klasses.zip(klasses.map(&:klass)))
+      end
+
+      def self.join_by_clause by
+        by.map{|rel, field| "#{rel.relationize} BY #{field}" }.join(", ")
+      end
+
+      def self.join lval, by
+        parallel = by.delete(:parallel)
+        cmd  = "JOIN " + join_by_clause(by)
+        parallelize! cmd, :parallel => parallel
+        l_klass = klass_from_join(by)
+        rval = new(l_klass, lval, cmd)
+        set lval, rval
+      end
+
+    end
+  end
+end