wukong 0.1.4 → 1.4.0

data/lib/wukong/and_pig.rb

@@ -1,62 +0,0 @@
-require 'wukong/and_pig/pig_var'
-require 'wukong/and_pig/as'
-require 'wukong/and_pig/functions'
-require 'wukong/and_pig/operators'
-require 'wukong/and_pig/data_types'
-require 'wukong/and_pig/pig_struct'
-require 'wukong/and_pig/generate'
-require 'wukong/and_pig/symbol'
-require 'wukong/and_pig/utils'
-
-module Wukong
-  #
-  # Wukong::AndPig lets you generate and run pig[http://hadoop.apache.org/pig]
-  # code from within ruby (and interactively, from the +irb+ console).
-  #
-  # It uses the same typed structures you've defined for Wukong to create
-  # pig-types aware commands. For example, the Wukong class
-  #
-  #   class Customer < TypedStruct.new( [:id, Integer],
-  #     [:name, String], [:postal_code, Integer], [:balance, Float] )
-  #   end
-  #
-  # will generate a LOAD command for pig as
-  #
-  #   Customer1.pig_load('q4_reports/customers.tsv').set!
-  #   # => Q4ReportsCustomers2 = LOAD 'q4_reports/customers.tsv'
-  #        AS (id: int, name: chararray, postal_code: int, balance: float) ;
-  #
-  # You can write anonymous chains
-  #
-  #   q1 = Customer1.
-  #     pig_load('q4_reports/customers.tsv').set!.
-  #     distinct.set! ;
-  #   q1.
-  #     group(:by => :postal_code).set!.
-  #     generate([:group, :postal_code], ["COUNT(#{q1.relation})", :customers_per_zip]).set!.
-  #     store!
-  #
-  #   Q4ReportsCustomers35    = LOAD    'q4_reports/customers.tsv' AS (id: int,name: chararray,postal_code: int,balance: float) ;
-  #   Q4ReportsCustomers36    = DISTINCT Q4ReportsCustomers35 ;
-  #   Q4ReportsCustomers37    = GROUP    Q4ReportsCustomers36 BY postal_code ;
-  #   Q4ReportsCustomers38    = FOREACH  Q4ReportsCustomers37 GENERATE
-  #       group AS postal_code,
-  #       COUNT(Q4ReportsCustomers36) AS customers_per_zip ;
-  #
-  # ---------------------------------------------------------------------------
-  #
-  # Note on pig:
-  #
-  # 1) Reverse the order of your tables in your join statement. Pig always
-  #    streams the keys of the last input, (materializing in memory the keys of
-  #    the first), so if one of your inputs has less instances of of a given key
-  #    this may help.
-  #
-  # 2) Reduce the number of maps and reducers per machine and give it all the
-  #    memory you can.
-  #
-  #
-  module AndPig
-  end
-end
-

data/lib/wukong/and_pig/README.textile

@@ -1,12 +0,0 @@
-Wukong::AndPig is a small library to more easily generate code for the
-"Pig":http://hadoop.apache.org/pig data analysis language.  
-
-Wukong::AndPig lets you use the structs from your Wukong scripts to
-generate Pig instructions that know their types and structure -- even through
-multiple pig commands. For example, if you use +FOREACH ... GENERATE+ to select
-only a few of those fields, Wukong::AndPig will know that the result has only
-those fields.
-
-We're still trying to figure out if this is a stupid and crazy idea, or just a
-crazy idea: Yeah, we're using a functional/OO scripting language to generate code for an
-imperative query language that generates Java code for ad-hoc map-reduce operations.

data/lib/wukong/and_pig/as.rb

@@ -1,37 +0,0 @@
-class AS
-  attr_accessor :expr, :name, :type, :ref, :options
-  def initialize expr, name=nil, type=nil, ref=nil, *option_flags
-    case expr
-    when AS
-      self.expr = expr.expr
-      self.name = expr.name
-      self.type = expr.type
-      self.ref  = expr.ref
-      self.options = expr.options
-    end
-    self.expr ||= expr
-    self.name = name if name
-    self.type = type if type
-    self.ref  = ref  if ref
-    self.options ||= { }
-    option_flags.each{|option| self.options[option] = true }
-  end
-
-  def to_s
-    clause  = "%-30s \t" % [ref, expr].compact.join('::')
-    if name
-      clause << "AS #{name}"      unless options[:skip_name]
-      clause << ":#{type.typify}" unless ((!type) || options[:skip_type])
-    end
-    clause
-  end
-
-  def self.[] *args
-    self.new *args
-  end
-
-  # Useful for feeding back into TypedStruct
-  def name_type
-    [name, type]
-  end
-end

data/lib/wukong/and_pig/data_types.rb

@@ -1,30 +0,0 @@
-# == SimpleDataTypes ==
-# int
-# long
-# double
-# arrays
-# chararray
-# bytearray
-#
-# == ComplexDataTypes ==
-# tuple
-# bag
-# map
-
-module Wukong
-  module AndPig
-    class PigVar
-
-    end
-  end
-end
-
-# class ScalarInteger  < TypedStruct.new [
-#     [:count,    Integer  ],
-#   ]
-#   include Wukong::AndPig::PigEmitter
-#   def self.load_scalar path
-#     var = super path
-#     var.to_i
-#   end
-# end

data/lib/wukong/and_pig/functions.rb

@@ -1,50 +0,0 @@
-
-# == Built-in Functions
-# EvalFunctions
-# AVG
-# CONCAT
-# COUNT
-# DIFF
-# MIN
-# MAX
-# SIZE
-# SUM
-# TOKENIZE
-
-# == NullOperators
-# isnull
-# isnotnull
-#
-# == BooleanOperators
-# and
-# or
-# not
-#
-# == DereferenceOperators
-# tupledereference.
-# mapdereference#
-#
-# == SignOperators
-# positive+
-# negative-
-#
-# == CastOperators
-# (type)$0
-# (type)alias
-#
-# == ArithmeticOperators
-# addition+
-# subtraction-
-# multiplication*
-# division/
-# modulo%
-# bincond?
-#
-# == ComparisonOperators
-# Equal==
-# notequal!=
-# lessthan<
-# greaterthan>
-# lessthanorequalto<=
-# greaterthanorequalto>=
-# patternmatchingmatches

data/lib/wukong/and_pig/generate.rb

@@ -1,85 +0,0 @@
-require 'wukong/and_pig/generate/variable_inflections'
-
-module Wukong
-  module AndPig
-
-    mattr_accessor :comments
-    self.comments = true
-    # send output to stdout or to captured pig instance
-    mattr_accessor :emit_dest
-    # full pathname to the pig executable
-    PIG_EXECUTABLE = '/usr/local/bin/pig'
-
-    def self.finish
-      PigVar.pig_in_poke.close if PigVar.pig_in_poke.respond_to?(:close)
-    end
-
-    #
-    # All the embarrassing magick to pretend ruby symbols are pig relations
-    #
-    class PigVar
-
-      # Output a command
-      def self.emit cmd, semicolon=true
-        cmd = cmd + ' ;' if semicolon
-        case Wukong::AndPig.emit_dest
-        when :captured
-          pig_in_poke.puts(cmd)
-          pig_in_poke.flush
-          puts pig_in_poke.gets
-        else
-          puts(cmd)
-        end
-      end
-
-      # generate the code
-      def self.emit_setter relation, rval
-        emit "%-23s\t= %s" % [relation, rval.cmd]
-        rval
-      end
-
-      # generate the code
-      def self.emit_imperative imperative, *rest
-        cmd_part = "%-14s \t" % imperative
-        arg_part = rest.map{|s| "%14s" % s.to_s }.join(" \t")
-        emit cmd_part+arg_part
-        rest.first
-      end
-
-      def self.pig_in_poke
-        return @pig_in_poke if @pig_in_poke
-        case Wukong::AndPig.emit_dest
-        when :captured
-          @pig_in_poke = IO.popen(PIG_EXECUTABLE, "w+")
-          @pig_in_poke.sync = true
-          @pig_in_poke
-        else @pig_in_poke = $stdout
-        end
-      end
-
-      #
-      # Reset the captured pig instance
-      #
-      def self.reset_pig_in_poke!
-        begin pig_in_poke.close ; rescue nil ; end
-        @pig_in_poke = nil
-      end
-
-      def set!
-        self.class.emit_setter(relation, self)
-      end
-
-      #
-      # Emit a comment
-      # skips if Wukong::AndPig.comments is false
-      #
-      def self.rem comment
-        return unless Wukong::AndPig.comments
-        PigVar.emit comment.gsub(/(^|\n)(#([\t ]|$))?/, "\n--  "), false
-      end
-    end
-
-  end
-end
-
-

data/lib/wukong/and_pig/generate/variable_inflections.rb

@@ -1,82 +0,0 @@
-String.class_eval do
-  #
-  # Generate relation name from a handle
-  #
-  def relationize() camelize end
-end
-Symbol.class_eval do
-  #
-  # Generate relation name from a handle
-  #
-  def relationize
-    to_s.relationize
-  end
-end
-
-Object.class_eval do
-  def typify() self.class ; end
-
-  def symbolize
-    self.to_s.underscore.gsub(%r{.*/}, '').to_sym
-  end
-end
-
-class << Integer ; def typify() 'int'           end ; end
-class << Bignum  ; def typify() 'long'          end ; end
-class << Float   ; def typify() 'float'         end ; end
-class << String  ; def typify() 'chararray'     end ; end
-class << Symbol  ; def typify() self            end ; end
-class << Date    ; def typify() 'long'          end ; end
-
-# Array.class_eval do
-#   def typify()
-#     "{ #{ map{|f,t| "#{f}: #{t.typify}"} } }"
-#   end
-# end
-# class Tuple
-#   attr_accessor :contents
-#   def initialize *args
-#     self.contents = args
-#   end
-#   def typify
-#     "bag { #{ contents.map{|f,t| "#{f}: #{t.typify}"} } }"
-#   end
-#   #
-#   # Sugar for creating a new bag. The following are equivalent:
-#   #
-#   #   Bag[:foo]
-#   #   Bag.new :foo
-#   #
-#   def self.[] *args
-#     new *args
-#   end
-# end
-
-module BagMethods
-  module ClassMethods
-    #
-    # Pig type string --
-    # the pig type strings for each sub-element.
-    #
-    def typify
-      vars_str = members.zip(mtypes).map do |attr, mtype|
-        "%s: %s" % [attr, mtype.typify]
-      end
-      "{ #{vars_str.join(', ')} }"
-    end
-  end
-  def self.included base
-    base.extend ClassMethods
-  end
-end
-
-class Bag < TypedStruct
-  def self.new *args
-    bag = super *args
-    bag.class_eval{ include BagMethods }
-  end
-  def self.[] *args
-    new *args
-  end
-end
-

data/lib/wukong/and_pig/junk.rb

@@ -1,51 +0,0 @@
-
-
-module Wukong
-  module AndPig
-
-    #
-    # Load the main class definitions
-    #
-    def self.init_load
-      puts File.open(PIG_DEFS_DIR+"/init_load.pig").read
-    end
-
-
-
-
-    #
-    # OK we're going to cheat here:
-    # just cat the file in, and treat it as a scalar
-    #
-    def load_scalar path
-      # var = `hadoop dfs -cat '#{path}/part-*' | head -n1 `.chomp
-      var = "636"
-    end
-
-
-
-    def count_distinct dest_rel, attr, group_by
-      distincted =
-        generate(temp_rel(dest_rel), attr).
-        distinct(temp_rel(dest_rel), :parallel => 10)
-      distincted.
-        group(   temp_rel(dest_rel), group_by).
-        foreach( dest_rel,  "GENERATE COUNT(#{distincted.relation}.#{attr}) AS n_#{attr}")
-    end
-
-    #
-    # Group a relation into bins, and return the counts for each bin
-    # * dest_rel - Relation to store
-    #   {bin,
-    #
-    def histogram dest_rel, bin_attr, bin_expr=nil
-      bin_expr ||= bin_attr
-      bin_name   = "#{bin_attr}_bin"
-      binned     = foreach(temp_rel(dest_rel), "GENERATE #{bin_expr} AS #{bin_name}")
-      binned.      group(  temp_rel(dest_rel), :by => bin_name).
-        foreach(         dest_rel,  "GENERATE group AS #{bin_name}, COUNT(#{binned.relation}) AS #{bin_attr}_count")
-    end
-
-
-  end
-end

data/lib/wukong/and_pig/operators.rb

@@ -1,8 +0,0 @@
-require 'wukong/and_pig/operators/evaluators'
-require 'wukong/and_pig/operators/foreach'
-require 'wukong/and_pig/operators/groupies'
-require 'wukong/and_pig/operators/load_store'
-require 'wukong/and_pig/operators/meta'
-require 'wukong/and_pig/operators/relational'
-require 'wukong/and_pig/operators/file_methods'
-require 'wukong/and_pig/operators/compound'

data/lib/wukong/and_pig/operators/compound.rb

@@ -1,29 +0,0 @@
-#
-# The FOREACH relational operator
-#
-module Wukong
-  module AndPig
-    class PigVar
-      #
-      # Select all elements in the source relation that match on the selecting relation,
-      # creating a relation with the same type as the source relation.
-      #
-      # For example, 
-      #
-      #   PV.isolate :isolated_cvals, :my_ids, :id, :my_complicated_values, :id
-      #
-      # returns a relation IsolatedCvals, whose type is identical to
-      # MyComplicatedValues' type, with only the elements having an id also
-      # presend in MyIds.
-      #
-      #
-      def self.isolate lval, on, on_field, from, from_field, options={ }
-        joined   = join anon(lval), on => on_field, from => from_field, :parallel => options.delete(:parallel)
-        isolated = joined.generate lval, { "'#{from}'" => :rsrc}, *PV[from].fields.map{|field| [from, field]}
-        isolated.klass = from.klass
-        isolated
-      end
-
-    end
-  end
-end