piglet 0.1.1 → 0.1.2

data/README.rdoc

@@ -256,14 +256,34 @@ All the aggregate functions are supported:
 * +SUM+
 * +TOKENIZE+
   
-Piglet only supports a limited set of arithmetic, comparison and boolean operators, so in practice the support for +FILTER+ and <code>FOREACH … GENERATE</code> is limited. This will be remedied soon, although the <code>!=</code> operator may never be supported in Ruby 1.8, since the <code>!</code> operator cannot be overridden (so there will be some less good looking solution like <code>.not</code>).
+Piglet only supports most arithmetic and logic operators (see below) on fields -- but check the output and make sure that it's doing what you expect because some it's tricky to see where Piglet hijacks the operators and when it's Ruby that is running the show. I'm doing the best I can, but there are many things that can't be done, at least not in Ruby 1.8.
+
+Piglet does support these field operators:
+
+* <code>==</code> (equality)
+* <code>&gt;</code> (greater than)
+* <code>&lt;</code> (less than)
+* <code>&gt=</code> (greater or equal to)
+* <code>&lt;=</code> (less than or equal to)
+* <code>%</code> (modulo)
+* <code>+</code> (addition)
+* <code>-</code> (subtraction)
+* <code>*</code> (multiplication)
+* <code>/</code> (division)
+
+It also has these operators, see below for explanations:
+
+* <code>#not</code> (logical negation)
+* <code>#neg</code> (numerical negation)
+* <code>#ne</code> (not equals)
+* <code>#test</code> (binary conditionals)
 
 Piglet does not support:
 
-* <code>!=</code> (not equals, you have to use <code>==</code> and a <code>NOT</code>, e.g. <code>(a == b).not</code>, which will be translated as <code>NOT (a == b)</code> or you can use <code>#ne</code>, which will translate to !=, e.g. <code>a.ne(b)</code> will become <code>a != b</code>)
+* <code>!=</code> (not equals, you have to use <code>==</code> and a <code>NOT</code>, e.g. <code>(a == b).not</code>, which will be translated as <code>NOT (a == b)</code> or you can use <code>#ne</code>, which will translate to !=, e.g. <code>a.ne(b)</code> will become <code>a != b</code>. May be supported in the future, but only in Ruby 1.9)
 * <code>? :</code> (the ternary operator)
-* <code>-</code> (negation, but you can use <code>#neg</code> on a field expression to get the same result, e.g. <code>a.neg</code> will be translated as <code>-a</code>)
-* <code>key#value</code> (map dereferencing)
+* <code>-</code> (negation, but you can use <code>#neg</code> on a field expression to get the same result, e.g. <code>a.neg</code> will be translated as <code>-a</code>. May be supported in the future, but only in Ruby 1.9)
+* <code>key#value</code> (map dereferencing, may be supported in the future)
 
 === Why aren't the aliases in the Pig Latin the same as the variable names in the Piglet script?
 

data/lib/piglet/field/binary_conditional.rb

@@ -0,0 +1,15 @@
+module Piglet
+  module Field
+    include Operators
+    
+    class BinaryConditional
+      def initialize(test, if_true, if_false)
+        @test, @if_true, @if_false = test, if_true, if_false
+      end
+    
+      def to_s
+        "(#{@test} ? #{@if_true} : #{@if_false})"
+      end
+    end
+  end
+end

data/lib/piglet/field/call_expression.rb

@@ -0,0 +1,21 @@
+module Piglet
+  module Field
+    class CallExpression # :nodoc:
+      include Operators
+    
+      def initialize(name, inner_expression, options=nil)
+        options ||= {}
+        @name, @inner_expression = name, inner_expression
+        @new_name = options[:as]
+      end
+    
+      def simple?
+        false
+      end
+    
+      def to_s
+        "#{@name}(#{@inner_expression})"
+      end
+    end
+  end
+end

data/lib/piglet/field/infix_expression.rb

@@ -0,0 +1,19 @@
+module Piglet
+  module Field
+    class InfixExpression # :nodoc:
+      include Operators
+    
+      def initialize(operator, left_expression, right_expression)
+        @operator, @left_expression, @right_expression = operator, left_expression, right_expression
+      end
+    
+      def simple?
+        false
+      end
+    
+      def to_s
+        "#{parenthesise(@left_expression)} #{@operator} #{parenthesise(@right_expression)}"
+      end
+    end
+  end
+end

data/lib/piglet/field/literal.rb

@@ -0,0 +1,20 @@
+module Piglet
+  module Field
+    class Literal
+      include Operators
+    
+      def initialize(obj)
+        @obj = obj
+      end
+    
+      def to_s
+        case @obj
+        when Numeric
+          @obj.to_s
+        else
+          "'#{escape(@obj.to_s)}'"
+        end
+      end
+    end
+  end
+end

data/lib/piglet/field/operators.rb

@@ -0,0 +1,80 @@
+module Piglet
+  module Field
+    module Operators # :nodoc:
+      SYMBOLIC_OPERATORS = [:==, :>, :<, :>=, :<=, :%, :+, :-, :*, :/]
+      FUNCTIONS = [:avg, :count, :diff, :max, :min, :size, :sum, :tokenize]
+    
+      FUNCTIONS.each do |fun|
+        define_method(fun) do
+          CallExpression.new(fun.to_s.upcase, self)
+        end
+      end
+
+      def empty?
+        CallExpression.new('IsEmpty', self)
+      end
+    
+      def as(new_name)
+        Rename.new(new_name, self)
+      end
+    
+      def not
+        PrefixExpression.new('NOT', self)
+      end
+    
+      def null?
+        SuffixExpression.new('is null', self)
+      end
+    
+      def not_null?
+        SuffixExpression.new('is not null', self)
+      end
+    
+      def cast(type)
+        PrefixExpression.new("(#{type.to_s})", self)
+      end
+    
+      def matches(pattern)
+        regex_options_pattern = /^\(\?.+?:(.*)\)$/
+        pattern = pattern.to_s.sub(regex_options_pattern, '\1') if pattern.is_a?(Regexp) && pattern.to_s =~ regex_options_pattern
+        InfixExpression.new('matches', self, "'#{pattern.to_s}'")
+      end
+    
+      def neg
+        PrefixExpression.new('-', self, false)
+      end
+    
+      def ne(other)
+        InfixExpression.new('!=', self, other)
+      end
+    
+      def and(other)
+        InfixExpression.new('AND', self, other)
+      end
+    
+      def or(other)
+        InfixExpression.new('OR', self, other)
+      end
+    
+      SYMBOLIC_OPERATORS.each do |op|
+        define_method(op) do |other|
+          InfixExpression.new(op.to_s, self, other)
+        end
+      end
+    
+    protected
+  
+      def parenthesise(expr)
+        if expr.respond_to?(:simple?) && ! expr.simple?
+          "(#{expr})"
+        else
+          expr.to_s
+        end
+      end
+      
+      def escape(str)
+        str.gsub(/("|'|\\)/) { |m| "\\#{$1}" }
+      end
+    end
+  end
+end

data/lib/piglet/field/prefix_expression.rb

@@ -0,0 +1,23 @@
+module Piglet
+  module Field
+    class PrefixExpression # :nodoc:
+      include Operators
+    
+      def initialize(operator, expression, space_between=true)
+        @operator, @expression, @space_between = operator, expression, space_between
+      end
+    
+      def simple?
+        true
+      end
+    
+      def to_s
+        if @space_between
+          "#{@operator} #{parenthesise(@expression)}"
+        else
+          "#{@operator}#{parenthesise(@expression)}"
+        end
+      end
+    end
+  end
+end

data/lib/piglet/field/reference.rb

@@ -0,0 +1,41 @@
+module Piglet
+  module Field
+    class Reference # :nodoc:
+      include Operators
+    
+      def initialize(name, relation=nil, options=nil)
+        options ||= {}
+        @name, @parent = name, relation
+        @explicit_ancestry = options[:explicit_ancestry] || false
+      end
+    
+      def simple?
+        true
+      end
+    
+      def method_missing(name, *args)
+        if name.to_s =~ /^\w+$/ && args.empty?
+          Reference.new(name, self, :explicit_ancestry => true)
+        else
+          super
+        end
+      end
+    
+      def [](n)
+        Reference.new("\$#{n}", self, :explicit_ancestry => true)
+      end
+    
+      def to_s
+        if @explicit_ancestry
+          if @parent.respond_to?(:alias)
+            "#{@parent.alias}.#{@name.to_s}"
+          else
+            "#{@parent}.#{@name.to_s}"
+          end
+        else
+          @name.to_s
+        end
+      end
+    end
+  end
+end

data/lib/piglet/field/rename.rb

@@ -0,0 +1,13 @@
+module Piglet
+  module Field
+    class Rename # :nodoc:
+      def initialize(new_name, field_expression)
+        @new_name, @field_expression = new_name, field_expression
+      end
+    
+      def to_s
+        "#{@field_expression} AS #{@new_name}"
+      end
+    end
+  end
+end

data/lib/piglet/field/suffix_expression.rb

@@ -0,0 +1,19 @@
+module Piglet
+  module Field
+    class SuffixExpression # :nodoc:
+      include Operators
+    
+      def initialize(operator, expression)
+        @operator, @expression = operator, expression
+      end
+    
+      def simple?
+        false
+      end
+    
+      def to_s
+        "#{parenthesise(@expression)} #{@operator}"
+      end
+    end
+  end
+end

data/lib/piglet/inout/describe.rb

@@ -0,0 +1,7 @@
+module Piglet
+  module Inout
+    class Describe # :nodoc:
+      include Output
+    end
+  end
+end

data/lib/piglet/inout/dump.rb

@@ -0,0 +1,7 @@
+module Piglet
+  module Inout
+    class Dump # :nodoc:
+      include Output
+    end
+  end
+end

data/lib/piglet/inout/explain.rb

@@ -0,0 +1,15 @@
+module Piglet
+  module Inout
+    class Explain # :nodoc:
+      include Output
+    
+      def to_s
+        if relation.nil?
+          "EXPLAIN"
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/piglet/inout/illustrate.rb

@@ -0,0 +1,7 @@
+module Piglet
+  module Inout
+    class Illustrate # :nodoc:
+      include Output
+    end
+  end
+end

data/lib/piglet/inout/load.rb

@@ -0,0 +1,31 @@
+module Piglet
+  module Inout
+    class Load # :nodoc:
+      include StorageTypes
+    
+      def initialize(path, options={})
+        options ||= {}
+        @path, @using, @schema = path, options[:using], options[:schema]
+      end
+    
+      def to_s
+        str  = "LOAD '#{@path}'"
+        str << " USING #{resolve_load_store_function(@using)}" if @using
+        str << " AS (#{schema_string})" if @schema
+        str
+      end
+    
+    private
+        
+      def schema_string
+        @schema.map do |field|
+          if field.is_a?(Enumerable)
+            field.map { |f| f.to_s }.join(':')
+          else
+            field.to_s
+          end
+        end.join(', ')
+      end
+    end
+  end
+end

data/lib/piglet/inout/output.rb

@@ -0,0 +1,15 @@
+module Piglet
+  module Inout
+    module Output # :nodoc:
+      attr_reader :relation
+    
+      def initialize(relation)
+        @relation = relation
+      end
+    
+      def to_s
+        "#{self.class.name.split(/::/).last.upcase} #{@relation.alias}"
+      end
+    end
+  end
+end

data/lib/piglet/inout/storage_types.rb

@@ -0,0 +1,18 @@
+module Piglet  
+  module Inout
+    module StorageTypes # :nodoc:
+      LOAD_STORE_FUNCTIONS = {
+        :binary_serializer => 'BinarySerializer',
+        :binary_deserializer => 'BinaryDeserializer',
+        :bin_storage => 'BinStorage',
+        :pig_storage => 'PigStorage',
+        :pig_dump => 'PigDump',
+        :text_loader => 'TextLoader'
+      }
+    
+      def resolve_load_store_function(name)
+        LOAD_STORE_FUNCTIONS[name] || name.to_s
+      end
+    end
+  end
+end

data/lib/piglet/inout/store.rb

@@ -0,0 +1,19 @@
+module Piglet
+  module Inout
+    class Store # :nodoc:
+      include StorageTypes
+      include Output
+    
+      def initialize(relation, path, options={})
+        @relation, @path, @using = relation, path, options[:using]
+      end
+    
+      def to_s
+        str  = super
+        str << " INTO '#{@path}'"
+        str << " USING #{resolve_load_store_function(@using)}" if @using
+        str
+      end
+    end
+  end
+end

data/lib/piglet/interpreter.rb

@@ -52,7 +52,9 @@ module Piglet
     # NOTE: the syntax load('path', :schema => {:a => :chararray, :b => :int})
     # would be nice, but the order of the keys can't be guaranteed in Ruby 1.8.
     def load(path, options={})
-      Load.new(path, options)
+      load = Inout::Load.new(path, options)
+      load.extend Piglet::Relation::Relation
+      load
     end
   
     # STORE
@@ -61,28 +63,28 @@ module Piglet
     #   store(x, 'some/path', :using => 'Xyz') # => STORE x INTO 'some/path' USING Xyz
     #   store(x, 'some/path', :using => :pig_storage) # => STORE x INTO 'some/path' USING PigStorage
     def store(relation, path, options={})
-      @stores << Store.new(relation, path, options)
+      @stores << Inout::Store.new(relation, path, options)
     end
   
     # DUMP
     #
     #   dump(x) # => DUMP x
     def dump(relation)
-      @stores << Dump.new(relation)
+      @stores << Inout::Dump.new(relation)
     end
   
     # ILLUSTRATE
     #
     #   illustrate(x) # => ILLUSTRATE x
     def illustrate(relation)
-      @stores << Illustrate.new(relation)
+      @stores << Inout::Illustrate.new(relation)
     end
   
     # DESCRIBE
     #
     #   describe(x) # => DESCRIBE x
     def describe(relation)
-      @stores << Describe.new(relation)
+      @stores << Inout::Describe.new(relation)
     end
   
     # EXPLAIN
@@ -90,7 +92,23 @@ module Piglet
     #   explain    # => EXPLAIN
     #   explain(x) # => EXPLAIN(x)
     def explain(relation=nil)
-      @stores << Explain.new(relation)
+      @stores << Inout::Explain.new(relation)
+    end
+    
+    # Support for binary conditions, a.k.a. the ternary operator.
+    #
+    #   x.test(x.a > x.b, x.a, x.b) # => (a > b ? a : b)
+    # 
+    # Should only be used in the block given to #filter and #foreach
+    def test(test, if_true, if_false)
+      Field::BinaryConditional.new(test, if_true, if_false)
+    end
+    
+    # Support for literals in FOREACH … GENERATE blocks.
+    #
+    #   x.foreach { |r| [literal("hello").as(:hello)] } # => FOREACH x GENERATE 'hello' AS hello
+    def literal(obj)
+      Field::Literal.new(obj)
     end
   
   private
@@ -104,5 +122,19 @@ module Piglet
         [assignment]
       end
     end
-  end  
+  end
+  
+private
+
+  class Assignment # :nodoc:
+    attr_reader :target
+
+    def initialize(relation)
+      @target = relation
+    end
+
+    def to_s
+      "#{@target.alias} = #{@target.to_s}"
+    end
+  end
 end

data/lib/piglet/relation/cogroup.rb

@@ -0,0 +1,33 @@
+module Piglet
+  module Relation
+    class Cogroup # :nodoc:
+      include Relation
+    
+      def initialize(relation, description)
+        @join_fields = description.reject { |k, v| ! (k.is_a?(Relation)) }
+        @sources = @join_fields.keys
+        @parallel = description[:parallel]
+      end
+    
+      def to_s
+        joins = @sources.map do |s|
+          fields = @join_fields[s]
+          if fields.is_a?(Enumerable) && fields.size > 1 && (fields.last == :inner || fields.last == :outer)
+            inout = fields.last.to_s.upcase
+            fields = fields[0..-2]
+          end
+          if fields.is_a?(Enumerable) && fields.size > 1
+            str = "#{s.alias} BY (#{fields.join(', ')})"
+          else
+            str = "#{s.alias} BY #{fields}"
+          end
+          str << " #{inout}" if inout
+          str
+        end
+        str  = "COGROUP #{joins.join(', ')}"
+        str << " PARALLEL #{@parallel}" if @parallel
+        str
+      end
+    end
+  end
+end

data/lib/piglet/relation/cross.rb

@@ -0,0 +1,24 @@
+module Piglet
+  module Relation
+    class Cross # :nodoc:
+      include Relation
+    
+      def initialize(relations, options={})
+        options ||= {}
+        @sources, @parallel = relations, options[:parallel]
+      end
+    
+      def to_s
+        str  = "CROSS #{source_aliases.join(', ')}"
+        str << " PARALLEL #{@parallel}" if @parallel
+        str
+      end
+  
+    private
+    
+      def source_aliases
+        @sources.map { |s| s.alias }
+      end
+    end
+  end
+end

data/lib/piglet/relation/distinct.rb

@@ -0,0 +1,18 @@
+module Piglet
+  module Relation
+    class Distinct # :nodoc:
+      include Relation
+    
+      def initialize(relation, options={})
+        options ||= {}
+        @sources, @parallel = [relation], options[:parallel]
+      end
+    
+      def to_s
+        str  = "DISTINCT #{@sources.first.alias}"
+        str << " PARALLEL #{@parallel}" if @parallel
+        str
+      end
+    end
+  end
+end

data/lib/piglet/relation/filter.rb

@@ -0,0 +1,15 @@
+module Piglet
+  module Relation
+    class Filter # :nodoc:
+      include Relation
+    
+      def initialize(relation, expression)
+        @sources, @expression = [relation], expression
+      end
+    
+      def to_s
+        "FILTER #{@sources.first.alias} BY #{@expression}"
+      end
+    end
+  end
+end

data/lib/piglet/relation/foreach.rb

@@ -0,0 +1,21 @@
+module Piglet
+  module Relation
+    class Foreach # :nodoc:
+      include Relation
+    
+      def initialize(relation, field_expressions)
+        @sources, @field_expressions = [relation], [field_expressions].flatten
+      end
+    
+      def to_s
+        "FOREACH #{@sources.first.alias} GENERATE #{field_expressions_string}"
+      end
+    
+    private
+  
+      def field_expressions_string
+        @field_expressions.map { |fe| fe.to_s }.join(', ')
+      end
+    end
+  end
+end

data/lib/piglet/relation/group.rb

@@ -0,0 +1,23 @@
+module Piglet
+  module Relation
+    class Group # :nodoc:
+      include Relation
+    
+      def initialize(relation, grouping, options={})
+        options ||= {}
+        @sources, @grouping, @parallel = [relation], grouping, options[:parallel]
+      end
+    
+      def to_s
+        str = "GROUP #{@sources.first.alias} BY "
+        if @grouping.size > 1
+          str << "(#{@grouping.join(', ')})"
+        else
+          str << @grouping.first.to_s
+        end
+        str << " PARALLEL #{@parallel}" if @parallel
+        str
+      end
+    end
+  end
+end