piglet 0.1.2 → 0.2.0

data/lib/piglet/relation/cogroup.rb

@@ -8,6 +8,21 @@ module Piglet
         @sources = @join_fields.keys
         @parallel = description[:parallel]
       end
+      
+      def schema
+        first_schema = @sources.first.schema
+        join_fields = @join_fields[@sources.first]
+        if join_fields.is_a?(Enumerable) && join_fields.size > 1
+          group_type = join_fields.map { |f| [f, first_schema.field_type[f]] }
+          description = [[:group, :tuple, group_type]]
+        else
+          description = [[:group, *join_fields]]
+        end
+        @sources.each do |source|
+          description << [source.alias.to_sym, Piglet::Schema::Bag.new(source.schema)]
+        end
+        Piglet::Schema::Tuple.parse(description)
+      end
     
       def to_s
         joins = @sources.map do |s|

data/lib/piglet/relation/cross.rb

@@ -8,6 +8,11 @@ module Piglet
         @sources, @parallel = relations, options[:parallel]
       end
     
+      def schema
+        schemas = @sources.map { |s| s.schema }
+        schemas.first.union(schemas[1..-1])
+      end
+    
       def to_s
         str  = "CROSS #{source_aliases.join(', ')}"
         str << " PARALLEL #{@parallel}" if @parallel

data/lib/piglet/relation/foreach.rb

@@ -6,6 +6,11 @@ module Piglet
       def initialize(relation, field_expressions)
         @sources, @field_expressions = [relation], [field_expressions].flatten
       end
+      
+      def schema
+        description = @field_expressions.map { |expr| [expr.name, expr.type] }
+        Piglet::Schema::Tuple.parse(description)
+      end
     
       def to_s
         "FOREACH #{@sources.first.alias} GENERATE #{field_expressions_string}"

data/lib/piglet/relation/group.rb

@@ -7,6 +7,22 @@ module Piglet
         options ||= {}
         @sources, @grouping, @parallel = [relation], grouping, options[:parallel]
       end
+      
+      def schema
+        parent = @sources.first
+        parent_schema = parent.schema
+        if @grouping.size == 1
+          group_type = parent.schema.field_type(@grouping.first)
+        else
+          group_type = Piglet::Schema::Tuple.parse(
+            @grouping.map { |field| [field, parent_schema.field_type(field)] }
+          )
+        end
+        Piglet::Schema::Tuple.parse([
+          [:group, group_type],
+          [parent.alias.to_sym, Piglet::Schema::Bag.new(parent_schema)]
+        ])
+      end
     
       def to_s
         str = "GROUP #{@sources.first.alias} BY "

data/lib/piglet/relation/join.rb

@@ -9,6 +9,11 @@ module Piglet
         @using = description[:using]
         @parallel = description[:parallel]
       end
+      
+      def schema
+        schemas = @sources.map { |s| s.schema }
+        schemas.first.union(schemas[1..-1])
+      end
     
       def to_s
         joins = @sources.map { |s| "#{s.alias} BY #{@join_fields[s]}" }.join(', ')

data/lib/piglet/relation/relation.rb

@@ -139,16 +139,31 @@ module Piglet
         Union.new(*([self] + relations))
       end
 
+      def field(name)
+        type = schema.field_type(name) rescue nil
+        Field::Reference.new(name, self, :type => type)
+      end
+      
+      def schema
+        if @sources.nil?
+          raise Piglet::Schema::SchemaError, 'Could not determine the schema since there was no source relation and this relation does not define its own schema'
+        elsif @sources.size > 1
+          raise Piglet::Schema::SchemaError, 'Could not determine the schema since there were more than one source relation'
+        else
+          @sources.first.schema
+        end
+      end
+
       def method_missing(name, *args)
         if name.to_s =~ /^\w+$/ && args.empty?
-          Field::Reference.new(name, self)
+          field(name)
         else
           super
         end
       end
     
       def [](n)
-        Field::Reference.new("\$#{n}", self)
+        field("\$#{n}")
       end
 
       def hash

data/lib/piglet/relation/union.rb

@@ -4,7 +4,7 @@ module Piglet
       include Relation
     
       def initialize(*relations)
-        @sources = relations
+        @sources = [relations].flatten
       end
     
       def to_s

data/lib/piglet/schema/bag.rb

@@ -0,0 +1,21 @@
+module Piglet
+  module Schema
+    class Bag
+      def initialize(tuple)
+        @tuple = tuple
+      end
+      
+      def field_names
+        @tuple.field_names
+      end
+      
+      def field_type(name)
+        @tuple.field_type(name)
+      end
+      
+      def to_s
+        @tuple.to_s.sub(/^\((.*)\)$/, '{\1}')
+      end
+    end
+  end
+end

data/lib/piglet/schema/tuple.rb

@@ -0,0 +1,111 @@
+module Piglet
+  module Schema
+    class Tuple
+      attr_reader :field_names
+      
+      def initialize(field_names, type_map)
+        @field_names = [ ]
+        @field_names = field_names.dup if field_names
+        @type_map = { }
+        @type_map = type_map.dup if type_map
+      end
+      
+      # Returns a new Tuple with a schema described by the specified array.
+      #
+      # The array will be interpreted as follows: each element defines a field,
+      # and a field can have an optional type. To define a typeless field simply
+      # use a symbol, to define a typed field use an array with two values: the
+      # first is the name and the second is the type.
+      #
+      # The type of a field can be one of the following:
+      # * <code>:int</code>
+      # * <code>:long</code>
+      # * <code>:float</code>
+      # * <code>:double</code>
+      # * <code>:chararray</code>
+      # * <code>:bytearray</code>
+      # * <code>:tuple</code> or Piglet::Schema::Tuple
+      # * <code>:bag</code> or Piglet::Schema::Bag
+      #
+      # If a type is not given it defaults to <code>:bytearray</code>. To define
+      # a tuple field either pass a Piglet::Schema::Tuple object as the type, or
+      # use <code>:tuple</code> and a thrid element, which is the schema of the
+      # tuple, e.g. <code>[[:a, :tuple, [:b, :c]]]</code>.
+      #
+      # Maps are currently not supported.
+      #
+      # Examples (Piglet schema description to the left with the Pig Latin
+      # schema definition to the right):
+      #
+      #   [:a, :b, :c]                     # => (a:bytearray, b:bytearray, c:bytearray)
+      #   [[:a, :chararray], [:b, :float]] # => (a:chararray, b:float)
+      #   [[:a, Tuple.parse(:b, :c)]]      # => (a:tuple (b:bytearray, c:bytearray))
+      #   [[:a, :bag, [:b, :c]]]           # => (a:bag {x:tuple (b:bytearray, c:bytearray)})
+      def self.parse(description)
+        field_names = [ ]
+        type_map = { }
+        index = 0
+        description.map do |component|
+          case component
+          when Enumerable
+            head = component.first
+            tail = component[1..-1]
+            case tail.first
+            when :tuple
+              type_map[head || index] = parse(*tail[1..-1])
+            when :bag
+              type_map[head || index] = Bag.new(parse(*tail[1..-1]))
+            else
+              type_map[head || index] = tail.first
+            end
+            field_names << head
+          else
+            type_map[component] = :bytearray
+            field_names << component
+          end
+          index += 1
+        end
+        Tuple.new(field_names, type_map)
+      end
+      
+      def union(*tuples)
+        field_names = @field_names.dup
+        type_map = @type_map.dup
+        tuples.flatten.each do |tuple|
+          tuple.field_names.each do |f|
+            field_names << f
+            type_map[f] = tuple.field_type(f)
+          end
+        end
+        Tuple.new(field_names, type_map)
+      end
+      
+      def field_type(field_name)
+        if Integer === field_name
+          field_name = @field_names[field_name] || field_name
+        end
+        @type_map[field_name]
+      end
+
+      def to_s
+        field_declarations = @field_names.map do |field_name|
+          type = field_type(field_name)
+          type_str = case type
+          when Tuple
+            "tuple #{type}"
+          when Bag
+            "bag #{type}"
+          else
+            type.to_s
+          end
+          if field_name
+            "#{field_name}:#{type_str}"
+          else
+            type_str
+          end
+        end
+        "(#{field_declarations.join(', ')})"
+      end
+    end
+  end
+end

data/spec/piglet/field/binary_conditional_spec.rb

@@ -0,0 +1,47 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+
+include Piglet::Field
+
+
+describe BinaryConditional do
+
+  before do
+    @true_test = mock('test expression')
+    @true_test.stub!(:to_s).and_return('true')
+    @expressions = {}
+    [:int, :long, :float, :double, :chararray, :bytearray, :bag, :tuple].each do |type|
+      @expressions[type] = mock("#{type} expression")
+      @expressions[type].extend Field
+      @expressions[type].stub!(:type).and_return(type)
+    end
+  end
+  
+  describe '#type' do
+    it 'returns the type of the true expression' do
+      bincond = BinaryConditional.new(@true_test, @expressions[:int], @expressions[:float])
+      bincond.type.should == :int
+    end
+    
+    it 'returns int if the true expression is an Integer' do
+      bincond = BinaryConditional.new(@true_test, 3, @expressions[:float])
+      bincond.type.should == :int
+    end
+    
+    it 'returns float if the true expression is a Float' do
+      bincond = BinaryConditional.new(@true_test, 3.14, @expressions[:float])
+      bincond.type.should == :float
+    end
+    
+    it 'returns boolean if the true expression is true' do
+      bincond = BinaryConditional.new(@true_test, true, @expressions[:float])
+      bincond.type.should == :boolean
+    end
+
+    it 'returns boolean if the true expression is false' do
+      bincond = BinaryConditional.new(@true_test, false, @expressions[:float])
+      bincond.type.should == :boolean
+    end
+  end
+  
+end

data/spec/piglet/field/field_spec.rb

@@ -0,0 +1,103 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+
+include Piglet::Field
+
+
+describe Field do
+
+  before do
+    @field = mock('field')
+    @field.extend Field
+    @expressions = {}
+    [:int, :long, :float, :double, :chararray, :bytearray, :bag, :tuple].each do |type|
+      @expressions[type] = mock("#{type} expression")
+      @expressions[type].extend Field
+      @expressions[type].stub!(:type).and_return(type)
+    end
+  end
+  
+  describe '#type' do
+    [:==, :ne, :<, :>, :<=, :>=, :and, :or].each do |op|
+      op_str = (op == :ne ? '!=' : op).to_s.upcase
+      
+      it "knows that a #{op_str} expression always is of type boolean" do
+        (@field.send(op, @field)).type.should eql(:boolean)
+      end
+    end
+
+    it 'knows that % yields an integer' do
+      (@field % 5).type.should eql(:int)
+    end
+    
+    it 'knows that a call to IsEmpty is of type boolean' do
+      @field.empty?.type.should eql(:boolean)
+    end
+    
+    it 'knows that the NOT operator yields a boolean' do
+      @field.not.type.should eql(:boolean)
+    end
+
+    it 'knows that the "is null" operator yields a boolean' do
+      @field.null?.type.should eql(:boolean)
+    end
+
+    it 'knows that the "is not null" operator yields a boolean' do
+      @field.not_null?.type.should eql(:boolean)
+    end
+    
+    [:int, :long, :float, :double, :chararray, :bytearray].each do |type|
+      it "knows that a cast to #{type} is of type #{type}" do
+        @field.cast(type).type.should eql(type)
+      end
+    end
+    
+    it 'knows that a "matches" expression is always of type boolean' do
+      @field.matches(/hello world/).type.should eql(:boolean)
+    end
+    
+    [:int, :long, :float, :double].each do |type|
+      it "knows that negating a #{type} yields a #{type}" do
+        @expressions[type].neg.type.should eql(type)
+      end
+    end
+
+    [:+, :-, :*].each do |op|
+      it "knows that int #{op} int yields an int" do
+        (@expressions[:int].send(op, @expressions[:int])).type.should eql(:int)
+      end
+
+      it "knows that int #{op} long yields a long" do
+        (@expressions[:int].send(op, @expressions[:long])).type.should eql(:long)
+      end
+
+      it "knows that int #{op} float yields a float" do
+        (@expressions[:int].send(op, @expressions[:float])).type.should eql(:float)
+      end
+
+      it "knows that int #{op} double yields a double" do
+        (@expressions[:int].send(op, @expressions[:double])).type.should eql(:double)
+      end
+    end
+    
+    combos = {
+      [:int, :int] => :int,
+      [:int, :long] => :long,
+      [:int, :float] => :float,
+      [:int, :double] => :double,
+      [:long, :float] => :float,
+      [:long, :double] => :double
+    }
+    
+    combos.each do |operands, result|
+      it "knows that #{operands[0]}/#{operands[1]} yields a #{result}" do
+        (@expressions[operands[0]] / @expressions[operands[1]]).type.should eql(result)
+      end
+
+      it "knows that #{operands[1]}/#{operands[0]} yields a #{result}" do
+        (@expressions[operands[1]] / @expressions[operands[0]]).type.should eql(result)
+      end
+    end
+  end
+  
+end

data/spec/piglet/field/infix_expression_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+
+include Piglet::Field
+
+
+describe InfixExpression do
+
+  before do
+    @expressions = {}
+    [:int, :long, :float, :double, :chararray, :bytearray, :bag, :tuple].each do |type|
+      @expressions[type] = mock("#{type} expression")
+      @expressions[type].extend Field
+      @expressions[type].stub!(:type).and_return(type)
+    end
+  end
+  
+  describe '#type' do
+    context 'specified' do
+      it 'returns the type specified in the options' do
+        expr = InfixExpression.new(@true_test, @expressions[:int], @expressions[:chararray], :type => :long)
+        expr.type.should == :long
+      end
+    end
+    
+    context 'inferred' do
+      it 'returns the type of the left expression if no other rules apply' do
+        expr = InfixExpression.new('x', @expressions[:chararray], @expressions[:bytearray])
+        expr.type.should == :chararray
+      end
+    
+      it 'returns double if the lefthand type is a double' do
+        expr = InfixExpression.new(@true_test, @expressions[:double], @expressions[:int])
+        expr.type.should == :double
+      end
+    
+      it 'returns double if the righthand type is a double' do
+        expr = InfixExpression.new(@true_test, @expressions[:float], @expressions[:double])
+        expr.type.should == :double
+      end
+    
+      it 'returns double when the other operand is of type long' do
+        expr = InfixExpression.new(@true_test, @expressions[:double], @expressions[:long])
+        expr.type.should == :double
+      end
+    
+      it 'returns float if one type is long and the other is a float' do
+        expr = InfixExpression.new(@true_test, @expressions[:long], @expressions[:float])
+        expr.type.should == :float
+      end
+    
+      it 'returns long if the lefthand type is long, and the righthand is an int' do
+        expr = InfixExpression.new(@true_test, @expressions[:long], @expressions[:int])
+        expr.type.should == :long
+      end
+    
+      it 'returns long if the righthand type is long, and the lefthand is an int' do
+        expr = InfixExpression.new(@true_test, @expressions[:int], @expressions[:long])
+        expr.type.should == :long
+      end
+        
+      it 'returns float if one operand is of type int and the other is a float' do
+        expr = InfixExpression.new(@true_test, @expressions[:int], @expressions[:float])
+        expr.type.should == :float
+      end
+    end
+  end
+  
+end