piglet 0.1.2 → 0.2.0

data/.gitignore

@@ -20,3 +20,4 @@ doc
 pkg
 
 ## PROJECT::SPECIFIC
+pig_*.log

data/README.rdoc

@@ -209,6 +209,21 @@ and then use them just as any other operator:
   
 nifty, huh?
 
+=== Types & schemas
+
+Piglet knows the schema of relations, so you can do something else that Pig lacks: introspection. This lets you do things like like this code, which counts the unique values of all fields in a relation:
+
+  relation = load('in', :schema => [:a, :b, :c])
+  relation.schema.field_names.each do |field|
+    grouped = relation.group(field)
+    counted = grouped.foreach { |r| [r[1].count] }
+    store(counted, "out-#{field}")
+  end
+  
+This feature obviously only works if you have specified a schema in the call to #load.
+
+There are currently many limitations to this feature, so use it with caution. Since the schema support isn't completely reliable Piglet does not enforce schemas, and it does not warn you if you try to access a field that doesn't exist. If it had, it would probably be more annoying and limiting than it would be worth.
+
 == Limitations
   
 The aim is to support most of Pig Latin, but currently there are some limitations.

data/lib/piglet.rb

@@ -1,6 +1,9 @@
 # :main: README.rdoc
 module Piglet # :nodoc:
-  VERSION = '0.1.2'
+  VERSION = '0.2.0'
+  
+  class PigletError < StandardError; end
+  class NotSupportedError < PigletError; end
   
   autoload :Interpreter, 'piglet/interpreter'
   
@@ -37,12 +40,17 @@ module Piglet # :nodoc:
     autoload :CallExpression, 'piglet/field/call_expression'
     autoload :InfixExpression, 'piglet/field/infix_expression'
     autoload :Literal, 'piglet/field/literal'
-    autoload :Operators, 'piglet/field/operators'
+    autoload :Field, 'piglet/field/field'
     autoload :PrefixExpression, 'piglet/field/prefix_expression'
     autoload :Reference, 'piglet/field/reference'
     autoload :Rename, 'piglet/field/rename'
     autoload :SuffixExpression, 'piglet/field/suffix_expression'
   end
-
-  class NotSupportedError < StandardError; end
+  
+  module Schema
+    autoload :Bag, 'piglet/schema/bag'
+    autoload :Tuple, 'piglet/schema/tuple'
+    
+    class SchemaError < PigletError; end
+  end
 end

data/lib/piglet/field/binary_conditional.rb

@@ -1,12 +1,13 @@
 module Piglet
   module Field
-    include Operators
-    
     class BinaryConditional
+      include Field
+      
       def initialize(test, if_true, if_false)
         @test, @if_true, @if_false = test, if_true, if_false
+        @type = expression_type(@if_true)
       end
-    
+          
       def to_s
         "(#{@test} ? #{@if_true} : #{@if_false})"
       end

data/lib/piglet/field/call_expression.rb

@@ -1,20 +1,20 @@
 module Piglet
   module Field
     class CallExpression # :nodoc:
-      include Operators
+      include Field
     
-      def initialize(name, inner_expression, options=nil)
+      def initialize(function_name, inner_expression, options=nil)
         options ||= {}
-        @name, @inner_expression = name, inner_expression
-        @new_name = options[:as]
+        @function_name, @inner_expression = function_name, inner_expression
+        @type = options[:type] || inner_expression.type
       end
-    
+      
       def simple?
         false
       end
     
       def to_s
-        "#{@name}(#{@inner_expression})"
+        "#{@function_name}(#{@inner_expression})"
       end
     end
   end

data/lib/piglet/field/field.rb

@@ -0,0 +1,134 @@
+module Piglet
+  module Field
+    module Field # :nodoc:
+      SYMBOLIC_OPERATORS = [:==, :>, :<, :>=, :<=, :%, :+, :-, :*, :/]
+      FUNCTIONS = [:avg, :count, :max, :min, :size, :sum, :tokenize]
+
+      attr_reader :name, :type
+    
+      FUNCTIONS.each do |fun|
+        define_method(fun) do
+          CallExpression.new(fun.to_s.upcase, self, :type => function_return_type(fun, self.type))
+        end
+      end
+
+      def empty?
+        CallExpression.new('IsEmpty', self, :type => :boolean)
+      end
+      
+      def diff(other)
+        raise NotSupportedError
+      end
+    
+      def as(new_name)
+        Rename.new(new_name, self)
+      end
+    
+      def not
+        PrefixExpression.new('NOT', self, true, :type => :boolean)
+      end
+    
+      def null?
+        SuffixExpression.new('is null', self, :type => :boolean)
+      end
+    
+      def not_null?
+        SuffixExpression.new('is not null', self, :type => :boolean)
+      end
+    
+      def cast(type)
+        PrefixExpression.new("(#{type.to_s})", self, true, :type => type.to_sym)
+      end
+    
+      def matches(pattern)
+        regex_options_pattern = /^\(\?.+?:(.*)\)$/
+        pattern = pattern.to_s.sub(regex_options_pattern, '\1') if pattern.is_a?(Regexp) && pattern.to_s =~ regex_options_pattern
+        InfixExpression.new('matches', self, "'#{pattern.to_s}'", :type => :boolean)
+      end
+    
+      def neg
+        PrefixExpression.new('-', self, false, :type => self.type)
+      end
+    
+      def ne(other)
+        InfixExpression.new('!=', self, other, :type => :boolean)
+      end
+    
+      def and(other)
+        InfixExpression.new('AND', self, other, :type => :boolean)
+      end
+    
+      def or(other)
+        InfixExpression.new('OR', self, other, :type => :boolean)
+      end
+    
+      SYMBOLIC_OPERATORS.each do |op|
+        define_method(op) do |other|
+          InfixExpression.new(op.to_s, self, other, :type => symbolic_operator_return_type(op, self, other))
+        end
+      end
+    
+    protected
+  
+      def parenthesise(expr)
+        if expr.respond_to?(:simple?) && ! expr.simple?
+          "(#{expr})"
+        else
+          expr.to_s
+        end
+      end
+      
+      def escape(str)
+        str.gsub(/("|'|\\)/) { |m| "\\#{$1}" }
+      end
+      
+      def function_return_type(function_name, expression_type)
+        case function_name
+        when :avg, :sum
+          case expression_type
+          when :int, :long
+            :long
+          when :float, :double, :bytearray
+            :double
+          else
+            nil
+          end
+        when :count, :size
+          :long
+        when :max, :min
+          expression_type
+        when :tokenize
+          :bag
+        else
+          nil
+        end
+      end
+      
+      def symbolic_operator_return_type(operator, left_expression, right_expression)
+        case operator
+        when :==, :>, :<, :>=, :<=
+          :boolean
+        when :%
+          :int
+        else # :+, :-, :*, :/
+          nil
+        end
+      end
+      
+      def expression_type(expression)
+        case expression
+        when Field
+          expression.type
+        when Integer
+          :int
+        when Numeric
+          :float
+        when true, false
+          :boolean
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/piglet/field/infix_expression.rb

@@ -1,10 +1,16 @@
 module Piglet
   module Field
     class InfixExpression # :nodoc:
-      include Operators
+      include Field
     
-      def initialize(operator, left_expression, right_expression)
+      def initialize(operator, left_expression, right_expression, options=nil)
+        options ||= {}
         @operator, @left_expression, @right_expression = operator, left_expression, right_expression
+        if options[:type]
+          @type = options[:type]
+        else
+          @type = determine_type(@left_expression, @right_expression)
+        end
       end
     
       def simple?
@@ -14,6 +20,23 @@ module Piglet
       def to_s
         "#{parenthesise(@left_expression)} #{@operator} #{parenthesise(@right_expression)}"
       end
+      
+    private
+    
+      def determine_type(left, right)
+        left_type = expression_type(left)
+        right_type = expression_type(right)
+
+        if left_type == :double || right_type == :double
+          :double
+        elsif left_type == :float || right_type == :float
+          :float
+        elsif left_type == :long || right_type == :long
+          :long
+        else
+          left_type
+        end
+      end
     end
   end
 end

data/lib/piglet/field/literal.rb

@@ -1,10 +1,12 @@
 module Piglet
   module Field
     class Literal
-      include Operators
+      include Field
     
-      def initialize(obj)
+      def initialize(obj, options=nil)
+        options ||= {}
         @obj = obj
+        @type = options[:type] || literal_type(obj)
       end
     
       def to_s
@@ -15,6 +17,21 @@ module Piglet
           "'#{escape(@obj.to_s)}'"
         end
       end
+      
+    private
+    
+      def literal_type(obj)
+        case obj
+        when String
+          :chararray
+        when Integer
+          :int
+        when Numeric
+          :double
+        else
+          :bytearray
+        end
+      end
     end
   end
 end

data/lib/piglet/field/prefix_expression.rb

@@ -1,10 +1,12 @@
 module Piglet
   module Field
     class PrefixExpression # :nodoc:
-      include Operators
+      include Field
     
-      def initialize(operator, expression, space_between=true)
+      def initialize(operator, expression, space_between=true, options=nil)
+        options ||= {}
         @operator, @expression, @space_between = operator, expression, space_between
+        @type = options[:type] || expression.type
       end
     
       def simple?

data/lib/piglet/field/reference.rb

@@ -1,28 +1,33 @@
 module Piglet
   module Field
     class Reference # :nodoc:
-      include Operators
+      include Field
     
       def initialize(name, relation=nil, options=nil)
         options ||= {}
         @name, @parent = name, relation
         @explicit_ancestry = options[:explicit_ancestry] || false
+        @type = options[:type]
       end
     
       def simple?
         true
       end
     
+      def field(name)
+        Reference.new(name, self, :explicit_ancestry => true)
+      end
+    
       def method_missing(name, *args)
         if name.to_s =~ /^\w+$/ && args.empty?
-          Reference.new(name, self, :explicit_ancestry => true)
+          field(name)
         else
           super
         end
       end
     
       def [](n)
-        Reference.new("\$#{n}", self, :explicit_ancestry => true)
+        field("\$#{n}")
       end
     
       def to_s

data/lib/piglet/field/rename.rb

@@ -1,12 +1,14 @@
 module Piglet
   module Field
     class Rename # :nodoc:
+      attr_reader :name, :type
+      
       def initialize(new_name, field_expression)
-        @new_name, @field_expression = new_name, field_expression
+        @name, @field_expression, @type = new_name, field_expression, field_expression.type
       end
-    
+      
       def to_s
-        "#{@field_expression} AS #{@new_name}"
+        "#{@field_expression} AS #{@name}"
       end
     end
   end

data/lib/piglet/field/suffix_expression.rb

@@ -1,10 +1,12 @@
 module Piglet
   module Field
     class SuffixExpression # :nodoc:
-      include Operators
+      include Field
     
-      def initialize(operator, expression)
+      def initialize(operator, expression, options=nil)
+        options ||= {}
         @operator, @expression = operator, expression
+        @type = options[:type] || expression.type
       end
     
       def simple?

data/lib/piglet/inout/load.rb

@@ -1,6 +1,7 @@
 module Piglet
   module Inout
     class Load # :nodoc:
+      include Piglet::Relation::Relation
       include StorageTypes
     
       def initialize(path, options={})
@@ -8,6 +9,10 @@ module Piglet
         @path, @using, @schema = path, options[:using], options[:schema]
       end
     
+      def schema
+        Piglet::Schema::Tuple.parse(@schema) if @schema
+      end
+    
       def to_s
         str  = "LOAD '#{@path}'"
         str << " USING #{resolve_load_store_function(@using)}" if @using

data/lib/piglet/interpreter.rb

@@ -27,7 +27,6 @@ module Piglet
         unless store.relation.nil?
           assignments(store.relation, handled_relations).each do |assignment|
             statements << assignment
-            handled_relations << assignment.target
           end
         end
         statements << store
@@ -52,9 +51,7 @@ module Piglet
     # NOTE: the syntax load('path', :schema => {:a => :chararray, :b => :int})
     # would be nice, but the order of the keys can't be guaranteed in Ruby 1.8.
     def load(path, options={})
-      load = Inout::Load.new(path, options)
-      load.extend Piglet::Relation::Relation
-      load
+      Inout::Load.new(path, options)
     end
   
     # STORE
@@ -116,6 +113,7 @@ module Piglet
     def assignments(relation, ignore_set)
       return [] if ignore_set.include?(relation)
       assignment = Assignment.new(relation)
+      ignore_set << relation
       if relation.sources
         (relation.sources.map { |source| assignments(source, ignore_set) } + [assignment]).flatten
       else