piglet 0.1.1 → 0.1.2

data/lib/piglet/relation/join.rb

@@ -0,0 +1,22 @@
+module Piglet
+  module Relation
+    class Join # :nodoc:
+      include Relation
+    
+      def initialize(relation, description)
+        @join_fields = Hash[*description.select { |k, v| k.is_a?(Relation) }.flatten]
+        @sources = @join_fields.keys
+        @using = description[:using]
+        @parallel = description[:parallel]
+      end
+    
+      def to_s
+        joins = @sources.map { |s| "#{s.alias} BY #{@join_fields[s]}" }.join(', ')
+        str  = "JOIN #{joins}"
+        str << " USING \"#{@using.to_s}\"" if @using
+        str << " PARALLEL #{@parallel}" if @parallel
+        str
+      end
+    end
+  end
+end

data/lib/piglet/relation/limit.rb

@@ -0,0 +1,15 @@
+module Piglet
+  module Relation
+    class Limit # :nodoc:
+      include Relation
+    
+      def initialize(relation, n)
+        @sources, @n = [relation], n
+      end
+    
+      def to_s
+        "LIMIT #{@sources.first.alias} #{@n}"
+      end
+    end
+  end
+end

data/lib/piglet/relation/order.rb

@@ -0,0 +1,31 @@
+module Piglet
+  module Relation
+    class Order # :nodoc:
+      include Relation
+    
+      def initialize(relation, fields, options)
+        options ||= {}
+        @sources, @parallel = [relation], options[:parallel]
+        @fields = fields.is_a?(Enumerable) ? fields : [fields]
+      end
+    
+      def to_s
+        "ORDER #{@sources.first.alias} BY #{field_strings}"
+      end
+    
+    private
+  
+      def field_strings
+        @fields.map { |f| field_string(f) }.join(', ')
+      end
+    
+      def field_string(f)
+        if f.is_a?(Enumerable)
+          "#{f[0]} #{f[1].to_s.upcase}"
+        else
+          f.to_s
+        end
+      end
+    end
+  end
+end

data/lib/piglet/relation/relation.rb

@@ -0,0 +1,179 @@
+module Piglet
+  module Relation
+    module Relation
+      attr_reader :sources
+
+      # The name this relation will get in Pig Latin. Then name is generated when
+      # the relation is outputed by the interpreter, and will be unique.
+      def alias
+        @alias ||= Relation.next_alias
+      end
+  
+      # GROUP
+      #
+      #   x.group(:a)                           # => GROUP x By a
+      #   x.group(:a, :b, :c)                   # => GROUP x BY (a, b, c)
+      #   x.group([:a, :b, :c], :parallel => 3) # => GROUP x BY (a, b, c) PARALLEL 3
+      def group(*args)
+        grouping, options = split_at_options(args)
+        Group.new(self, [grouping].flatten, options)
+      end
+  
+      # DISTINCT
+      #
+      #   x.distinct                 # => DISTINCT x
+      #   x.distinct(:parallel => 5) # => DISTINCT x PARALLEL 5
+      def distinct(options={})
+        Distinct.new(self, options)
+      end
+
+      # COGROUP
+      #
+      #   x.cogroup(x => :a, y => :b)                 # => COGROUP x BY a, y BY b
+      #   x.cogroup(x => :a, y => :b, z => :c)        # => COGROUP x BY a, y BY b, z BY c
+      #   x.cogroup(x => [:a, :b], y => [:c, :d])     # => COGROUP x BY (a, b), y BY (c, d)
+      #   x.cogroup(x => :a, y => [:b, :inner])       # => COGROUP x BY a, y BY b INNER
+      #   x.cogroup(x => :a, y => :b, :parallel => 5) # => COGROUP x BY a, y BY b PARALLEL 5
+      def cogroup(description)
+        Cogroup.new(self, description)
+      end
+  
+      # CROSS
+      #
+      #   x.cross(y)                      # => CROSS x, y
+      #   x.cross(y, z, w)                # => CROSS x, y, z, w
+      #   x.cross([y, z], :parallel => 5) # => CROSS x, y, z, w PARALLEL 5
+      def cross(*args)
+        relations, options = split_at_options(args)
+        Cross.new(([self] + relations).flatten, options)
+      end
+  
+      # FILTER
+      #
+      #   x.filter { |r| r.a == r.b }            # => FILTER x BY a == b
+      #   x.filter { |r| r.a > r.b && r.c != 3 } # => FILTER x BY a > b AND c != 3
+      def filter
+        Filter.new(self, yield(self))
+      end
+  
+      # FOREACH ... GENERATE
+      #
+      #   x.foreach { |r| r.a }            # => FOREACH x GENERATE a
+      #   x.foreach { |r| [r.a, r.b] }     # => FOREACH x GENERATE a, b
+      #   x.foreach { |r| r.a.max }        # => FOREACH x GENERATE MAX(a)
+      #   x.foreach { |r| r.a.avg.as(:b) } # => FOREACH x GENERATE AVG(a) AS b
+      #
+      #--
+      #
+      # TODO: FOREACH a { b GENERATE c }
+      def foreach
+        Foreach.new(self, yield(self))
+      end
+  
+      # JOIN
+      #
+      #   x.join(x => :a, y => :b)                        # => JOIN x BY a, y BY b
+      #   x.join(x => :a, y => :b, z => :c)               # => JOIN x BY a, y BY b, z BY c
+      #   x.join(x => :a, y => :b, :using => :replicated) # => JOIN x BY a, y BY b USING "replicated"
+      #   x.join(x => :a, y => :b, :parallel => 5)        # => JOIN x BY a, y BY b PARALLEL 5
+      def join(description)
+        Join.new(self, description)
+      end
+  
+      # LIMIT
+      #
+      #   x.limit(10) # => LIMIT x 10
+      def limit(n)
+        Limit.new(self, n)
+      end
+  
+      # ORDER
+      #
+      #   x.order(:a)                      # => ORDER x BY a
+      #   x.order(:a, :b)                  # => ORDER x BY a, b
+      #   x.order([:a, :asc], [:b, :desc]) # => ORDER x BY a ASC, b DESC
+      #   x.order(:a, :parallel => 5)      # => ORDER x BY a PARALLEL 5
+      #
+      #--
+      #
+      # NOTE: the syntax x.order(:a => :asc, :b => :desc) would be nice, but in
+      # Ruby 1.8 the order of the keys cannot be guaranteed.
+      def order(*args)
+        fields, options = split_at_options(args)
+        fields = *fields
+        Order.new(self, fields, options)
+      end
+  
+      # SAMPLE
+      #
+      #   x.sample(5) # => SAMPLE x 5;
+      def sample(n)
+        Sample.new(self, n)
+      end
+    
+      # SPLIT
+      #
+      #   y, z = x.split { |r| [r.a <= 3, r.b > 4]} # => SPLIT x INTO y IF a <= 3, z IF a > 4
+      def split
+        Split.new(self, yield(self)).shards
+      end
+  
+      # STREAM
+      #
+      #   x.stream(x, 'cut -f 3')                         # => STREAM x THROUGH `cut -f 3`
+      #   x.stream([x, y], 'cut -f 3')                    # => STREAM x, y THROUGH `cut -f 3`
+      #   x.stream(x, 'cut -f 3', :schema => [%w(a int)]) # => STREAM x THROUGH `cut -f 3` AS (a:int)
+      #
+      #--
+      #
+      # TODO: how to handle DEFINE'd commands?
+      def stream(relations, command, options={})
+        raise NotSupportedError
+      end
+  
+      # UNION
+      #
+      #   x.union(y)    # => UNION x, y
+      #   x.union(y, z) # => UNION x, y, z
+      def union(*relations)
+        Union.new(*([self] + relations))
+      end
+
+      def method_missing(name, *args)
+        if name.to_s =~ /^\w+$/ && args.empty?
+          Field::Reference.new(name, self)
+        else
+          super
+        end
+      end
+    
+      def [](n)
+        Field::Reference.new("\$#{n}", self)
+      end
+
+      def hash
+        self.alias.hash
+      end
+  
+      def eql?(other)
+        other.is_a?(Relation) && other.alias == self.alias
+      end
+  
+    private
+
+      def split_at_options(parameters)
+        if parameters.last.is_a? Hash
+          [parameters[0..-2], parameters.last]
+        else
+          [parameters, nil]
+        end
+      end
+
+      def self.next_alias
+        @counter ||= 0
+        @counter += 1
+        "relation_#{@counter}"
+      end
+    end
+  end
+end

data/lib/piglet/relation/sample.rb

@@ -0,0 +1,15 @@
+module Piglet
+  module Relation
+    class Sample # :nodoc:
+      include Relation
+    
+      def initialize(relation, n)
+        @sources, @n = [relation], n
+      end
+    
+      def to_s
+        "SAMPLE #{@sources.first.alias} #{@n}"
+      end
+    end
+  end
+end

data/lib/piglet/relation/split.rb

@@ -0,0 +1,45 @@
+module Piglet
+  module Relation
+    class Split # :nodoc:
+      include Relation
+    
+      def initialize(relation, expressions)
+        @sources, @expressions = [relation], expressions
+        @shard_map = create_shards
+      end
+    
+      def shards
+        @shard_map.values_at(*@expressions)
+      end
+    
+      def to_s
+        split_strings = @expressions.map do |expression|
+          "#{@shard_map[expression].alias} IF #{expression}"
+        end
+        
+        "SPLIT #{@sources.first.alias} INTO #{split_strings.join(', ')}"
+      end
+    
+    private
+  
+      def create_shards
+        @expressions.inject({}) do |map, expr|
+          map[expr] = RelationShard.new(self)
+          map
+        end
+      end
+    end
+  
+    class RelationShard # :nodoc:
+      include Relation
+    
+      def initialize(split)
+        @sources = [split]
+      end
+    
+      def to_s
+        self.alias
+      end
+    end
+  end
+end

data/lib/piglet/relation/stream.rb

@@ -0,0 +1,7 @@
+module Piglet
+  module Relation
+    class Stream # :nodoc:
+      include Relation
+    end
+  end
+end

data/lib/piglet/relation/union.rb

@@ -0,0 +1,21 @@
+module Piglet
+  module Relation
+    class Union # :nodoc:
+      include Relation
+    
+      def initialize(*relations)
+        @sources = relations
+      end
+    
+      def to_s
+        "UNION #{source_aliases.join(', ')}"
+      end
+    
+    private
+  
+      def source_aliases
+        @sources.map { |s| s.alias }
+      end
+    end
+  end
+end

data/lib/piglet.rb

@@ -1,45 +1,47 @@
 # :main: README.rdoc
 module Piglet # :nodoc:
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
   
-  autoload_files = %w(
-    assignment
-    cogroup
-    cross
-    describe
-    distinct
-    dump
-    explain
-    field
-    field_expression_functions
-    field_function_expression
-    field_infix_expression
-    field_prefix_expression
-    field_rename
-    field_suffix_expression
-    filter
-    foreach
-    group
-    illustrate
-    interpreter
-    join
-    limit
-    load
-    load_and_store
-    order
-    relation
-    sample
-    split
-    store
-    storing
-    stream
-    union
-  )
+  autoload :Interpreter, 'piglet/interpreter'
   
-  autoload_files.each do |f|
-    c = f.split('_').map { |s| s.capitalize }.join.to_sym
-    p = "piglet/#{f}"
-    autoload c, p
+  module Inout
+    autoload :Describe, 'piglet/inout/describe'
+    autoload :Dump, 'piglet/inout/dump'
+    autoload :Explain, 'piglet/inout/explain'
+    autoload :Illustrate, 'piglet/inout/illustrate'
+    autoload :Load, 'piglet/inout/load'
+    autoload :Output, 'piglet/inout/output'
+    autoload :StorageTypes, 'piglet/inout/storage_types'
+    autoload :Store, 'piglet/inout/store'
+  end
+  
+  module Relation
+    autoload :Cogroup, 'piglet/relation/cogroup'
+    autoload :Cross, 'piglet/relation/cross'
+    autoload :Distinct, 'piglet/relation/distinct'
+    autoload :Filter, 'piglet/relation/filter'
+    autoload :Foreach, 'piglet/relation/foreach'
+    autoload :Group, 'piglet/relation/group'
+    autoload :Join, 'piglet/relation/join'
+    autoload :Limit, 'piglet/relation/limit'
+    autoload :Order, 'piglet/relation/order'
+    autoload :Relation, 'piglet/relation/relation'
+    autoload :Sample, 'piglet/relation/sample'
+    autoload :Split, 'piglet/relation/split'
+    autoload :Stream, 'piglet/relation/stream'
+    autoload :Union, 'piglet/relation/union'
+  end
+  
+  module Field
+    autoload :BinaryConditional, 'piglet/field/binary_conditional'
+    autoload :CallExpression, 'piglet/field/call_expression'
+    autoload :InfixExpression, 'piglet/field/infix_expression'
+    autoload :Literal, 'piglet/field/literal'
+    autoload :Operators, 'piglet/field/operators'
+    autoload :PrefixExpression, 'piglet/field/prefix_expression'
+    autoload :Reference, 'piglet/field/reference'
+    autoload :Rename, 'piglet/field/rename'
+    autoload :SuffixExpression, 'piglet/field/suffix_expression'
   end
 
   class NotSupportedError < StandardError; end

data/spec/piglet/{field_spec.rb → field/reference_spec.rb}

@@ -1,10 +1,10 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 
-describe Piglet::Field do
+describe Piglet::Field::Reference do
 
   before do
-    @field = Piglet::Field.new('field')
+    @field = Piglet::Field::Reference.new('field')
   end
   
   describe '#to_s' do
@@ -13,7 +13,7 @@ describe Piglet::Field do
     end
     
     it 'returns a string with the field name (as a symbol)' do
-      @field = Piglet::Field.new(:field)
+      @field = Piglet::Field::Reference.new(:field)
       @field.to_s.should eql("field")
     end
   end
@@ -52,8 +52,8 @@ describe Piglet::Field do
   
   context 'infix and unary operators' do
     before do
-      @field1 = Piglet::Field.new('field1')
-      @field2 = Piglet::Field.new('field2')
+      @field1 = Piglet::Field::Reference.new('field1')
+      @field2 = Piglet::Field::Reference.new('field2')
     end
     
     [:==, :>, :<, :>=, :<=, :%, :+, :-, :*, :/].each do |op|
@@ -125,6 +125,22 @@ describe Piglet::Field do
     it 'supports casts on an expression' do
       (@field1 + @field2).cast(:chararray).to_s.should eql("(chararray) (field1 + field2)")
     end
+
+    it 'supports AND though #and on fields' do
+      @field1.and(@field2).to_s.should eql("field1 AND field2")
+    end
+    
+    it 'supports AND through #and on expressions' do
+      (@field1 + @field2).and(@field1 - @field2).to_s.should eql("(field1 + field2) AND (field1 - field2)")
+    end
+
+    it 'supports OR though #or on fields' do
+      @field1.or(@field2).to_s.should eql("field1 OR field2")
+    end
+    
+    it 'supports OR through #or on expressions' do
+      (@field1 + @field2).or(@field1 - @field2).to_s.should eql("(field1 + field2) OR (field1 - field2)")
+    end
   end
   
 end

data/spec/piglet/interpreter_spec.rb

@@ -40,7 +40,7 @@ describe Piglet::Interpreter do
         @interpreter.to_pig_latin.should include("LOAD 'some/path' USING XYZ;")
       end
   
-      Piglet::LoadAndStore::LOAD_STORE_FUNCTIONS.each do |symbolic_name, function|
+      Piglet::Inout::StorageTypes::LOAD_STORE_FUNCTIONS.each do |symbolic_name, function|
         it "knows that the load method :#{symbolic_name} means #{function}" do
           @interpreter.interpret { store(load('some/path', :using => symbolic_name), 'out') }
           @interpreter.to_pig_latin.should include("LOAD 'some/path' USING #{function};")
@@ -267,10 +267,10 @@ describe Piglet::Interpreter do
         @interpreter.to_pig_latin.should match(/FILTER \w+ BY a == 3/)
       end
 
-      # it 'outputs a FILTER statement with a complex test' do
-      #   @interpreter.interpret { dump(load('in').filter { |r| r.a > r.b && r.c != 3 }) }
-      #   @interpreter.to_pig_latin.should match(/FILTER \w+ BY a > b AND c != 3/)
-      # end
+      it 'outputs a FILTER statement with a complex test' do
+        @interpreter.interpret { dump(load('in').filter { |r| (r.a > r.b).and(r.c.ne(3)) }) }
+        @interpreter.to_pig_latin.should match(/FILTER \w+ BY \(a > b\) AND \(c != 3\)/)
+      end
     end
     
     describe 'SPLIT' do
@@ -410,4 +410,50 @@ describe Piglet::Interpreter do
     end
   end
 
+  context 'misc. operators' do
+    it 'outputs a binary conditional when using #test' do
+      @interpreter.interpret do
+        dump(load('in').foreach { |r| [test(r.a == r.b, r.a, r.b)]})
+      end
+      @interpreter.to_pig_latin.should include('(a == b ? a : b)')
+    end
+  end
+
+  context 'literals' do
+    it 'outputs a literal string when passing a string to #literal' do
+      @interpreter.interpret do
+        dump(load('in').foreach { |r| [literal('hello').as(:world)]})
+      end
+      @interpreter.to_pig_latin.should include("'hello' AS world")
+    end
+    
+    it 'outputs a literal integer when passing an integer to #literal' do
+      @interpreter.interpret do
+        dump(load('in').foreach { |r| [literal(3).as(:n)]})
+      end
+      @interpreter.to_pig_latin.should include("3 AS n")
+    end
+    
+    it 'outputs a literal float when passing a float to #literal' do
+      @interpreter.interpret do
+        dump(load('in').foreach { |r| [literal(3.14).as(:pi)]})
+      end
+      @interpreter.to_pig_latin.should include("3.14 AS pi")
+    end
+    
+    it 'outputs a literal string when passing an arbitrary object to #literal' do
+      @interpreter.interpret do
+        dump(load('in').foreach { |r| [literal(self).as(:interpreter)]})
+      end
+      @interpreter.to_pig_latin.should match(/'[^']+' AS interpreter/)
+    end
+    
+    it 'escapes single quotes in literal strings' do
+      @interpreter.interpret do
+        dump(load('in').foreach { |r| [literal("hello 'world'").as(:str)]})
+      end
+      @interpreter.to_pig_latin.should include("'hello \\'world\\'' AS str")
+    end
+  end
+
 end

data/spec/piglet/{relation_spec.rb → relation/relation_spec.rb}

@@ -1,11 +1,11 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 
-describe Piglet::Relation do
+describe Piglet::Relation::Relation do
   
   before do
     @relation = Object.new
-    @relation.extend Piglet::Relation
+    @relation.extend Piglet::Relation::Relation
   end
   
   it 'has a alias' do
@@ -16,7 +16,7 @@ describe Piglet::Relation do
     aliases = { }
     1000.times do
       @relation = Object.new
-      @relation.extend Piglet::Relation
+      @relation.extend Piglet::Relation::Relation
       aliases.should_not have_key(@relation.alias)
       aliases[@relation.alias] = @relation
     end
@@ -37,7 +37,7 @@ describe Piglet::Relation do
   describe '#cross' do
     it 'returns a new relation with the target relation as one of the sources' do
       other = Object.new
-      other.extend Piglet::Relation
+      other.extend Piglet::Relation::Relation
       @relation.cross(other).sources.should include(@relation)
     end
   end
@@ -45,7 +45,7 @@ describe Piglet::Relation do
   describe '#union' do
     it 'returns a new relation with the target relation as one of the sources' do
       other = Object.new
-      other.extend Piglet::Relation
+      other.extend Piglet::Relation::Relation
       @relation.union(other).sources.should include(@relation)
     end
   end

data/spec/piglet/{split_spec.rb → relation/split_spec.rb}

@@ -1,27 +1,27 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 
-describe Piglet::Split do
+describe Piglet::Relation::Split do
 
   before do
     @relation = mock('source')
     @expr1 = mock('expr1')
     @expr2 = mock('expr2')
     @relation.stub!(:alias).and_return('rel')
-    @expr1.stub!(:to_s).and_return('y')
-    @expr2.stub!(:to_s).and_return('w')
-    @split = Piglet::Split.new(@relation, [@expr1, @expr2])
+    @expr1.stub!(:to_s).and_return('expr1')
+    @expr2.stub!(:to_s).and_return('expr2')
+    @split = Piglet::Relation::Split.new(@relation, [@expr1, @expr2])
   end
 
   describe '#to_s' do
     it 'outputs all x IF y expressions' do
-      @split.to_s.should match(/SPLIT rel INTO \w+ IF [yw], \w+ IF [yw]/)
+      @split.to_s.should match(/SPLIT rel INTO \w+ IF expr[12], \w+ IF expr[12]/)
     end
     
     it 'contains the names of all the shard relations' do
       @shards = @split.shards
-      @split.to_s.should include("#{@shards[0].alias} IF y")
-      @split.to_s.should include("#{@shards[1].alias} IF w")
+      @split.to_s.should include("#{@shards[0].alias} IF expr1")
+      @split.to_s.should include("#{@shards[1].alias} IF expr2")
     end
   end
   

data/spec/spec_helper.rb

@@ -7,8 +7,6 @@ require 'spec'
 require 'spec/autorun'
 
 
-require 'piglet/interpreter'
-
 Spec::Runner.configure do |config|
   
 end