piglet 0.2.5 → 0.3.0

data/spec/piglet/relation/relation_spec.rb

@@ -6,8 +6,10 @@ require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 describe Piglet::Relation::Relation do
   
   before do
-    @relation = Object.new
-    @relation.extend Piglet::Relation::Relation
+    @interpreter = double()
+    @interpreter.stub(:next_relation_alias).and_return(rand(10000))
+    @relation = PlainRelation.new(@interpreter)
+    @relation.extend(Piglet::Relation::Relation)
   end
   
   it 'has a alias' do
@@ -16,9 +18,10 @@ describe Piglet::Relation::Relation do
   
   it 'has a unique alias' do
     aliases = { }
+    interpreter = double()
+    interpreter.stub(:next_relation_alias).and_return(*(0..1000).to_a)
     1000.times do
-      @relation = Object.new
-      @relation.extend Piglet::Relation::Relation
+      @relation = PlainRelation.new(interpreter)
       aliases.should_not have_key(@relation.alias)
       aliases[@relation.alias] = @relation
     end

data/spec/piglet/relation/split_spec.rb

@@ -12,7 +12,9 @@ describe Piglet::Relation::Split do
     @relation.stub!(:alias).and_return('rel')
     @expr1.stub!(:to_s).and_return('expr1')
     @expr2.stub!(:to_s).and_return('expr2')
-    @split = Piglet::Relation::Split.new(@relation, [@expr1, @expr2])
+    @interpreter = mock('Interpreter')
+    @interpreter.stub(:next_relation_alias).and_return(3)
+    @split = Piglet::Relation::Split.new(@relation, @interpreter, [@expr1, @expr2])
   end
 
   describe '#to_s' do

data/spec/piglet/relation/union_spec.rb

@@ -6,13 +6,11 @@ require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 describe Piglet::Relation::Union do
 
   before do
-    @relation1 = Object.new
-    @relation1.extend Piglet::Relation::Relation
-    @relation2 = mock('relation2')
-    @relation3 = mock('relation3')
-    @relation1.stub!(:alias).and_return('relation1')
-    @relation2.stub!(:alias).and_return('relation2')
-    @relation3.stub!(:alias).and_return('relation3')
+    @interpreter = stub()
+    @interpreter.stub(:next_relation_alias).and_return('relation1', 'relation2', 'relation3')
+    @relation1 = PlainRelation.new(@interpreter)
+    @relation2 = PlainRelation.new(@interpreter)
+    @relation3 = PlainRelation.new(@interpreter)
   end
 
   describe '#to_s' do    

data/spec/piglet_spec.rb

@@ -243,49 +243,94 @@ describe Piglet do
 
     describe 'FOREACH … GENERATE' do
       it 'outputs a FOREACH … GENERATE statement' do
-        @interpreter.interpret { dump(load('in').foreach { |r| :a }) }
+        @interpreter.interpret { dump(load('in').foreach { :a }) }
         @interpreter.to_pig_latin.should match(/FOREACH \w+ GENERATE a/)
       end
       
       it 'outputs a FOREACH … GENERATE statement with a list of fields' do
-        @interpreter.interpret { dump(load('in').foreach { |r| [:a, :b, :c] }) }
+        @interpreter.interpret { dump(load('in').foreach { [:a, :b, :c] }) }
         @interpreter.to_pig_latin.should match(/FOREACH \w+ GENERATE a, b, c/)
       end
       
       it 'outputs a FOREACH … GENERATE statement with fields resolved from the relation' do
-        @interpreter.interpret { dump(load('in').foreach { |r| [r.a, r.b, r.c] }) }
+        @interpreter.interpret { dump(load('in').foreach { [a, b, c] }) }
         @interpreter.to_pig_latin.should match(/FOREACH (\w+) GENERATE a, b, c/)
       end
       
       it 'outputs a FOREACH … GENERATE statement with fields resolved from the relation with positional syntax' do
-        @interpreter.interpret { dump(load('in').foreach { |r| [r[0], r[1], r[2]] }) }
+        @interpreter.interpret { dump(load('in').foreach { [self[0], self[1], self[2]] }) }
         @interpreter.to_pig_latin.should match(/FOREACH (\w+) GENERATE \$0, \$1, \$2/)
       end
       
       it 'outputs a FOREACH … GENERATE statement with aggregate functions applied to the fields' do
-        @interpreter.interpret { dump(load('in').foreach { |r| [r.a.max, r.b.min, r.c.avg] }) }
+        @interpreter.interpret { dump(load('in').foreach { [a.max, b.min, c.avg] }) }
         @interpreter.to_pig_latin.should match(/FOREACH (\w+) GENERATE MAX\(a\), MIN\(b\), AVG\(c\)/)
       end
       
       it 'outputs a FOREACH … GENERATE statement with fields that access inner fields' do
-        @interpreter.interpret { dump(load('in').foreach { |r| [r.a.b, r.b.c, r.c.d] }) }
+        @interpreter.interpret { dump(load('in').foreach { [a.b, b.c, c.d] }) }
         @interpreter.to_pig_latin.should match(/FOREACH (\w+) GENERATE a.b, b.c, c.d/)
       end
       
       it 'outputs a FOREACH … GENERATE statement that includes field aliasing' do
-        @interpreter.interpret { dump(load('in').foreach { |r| [r.a.b.as(:c), r.a.b.as(:d)] }) }
+        @interpreter.interpret { dump(load('in').foreach { [a.b.as(:c), a.b.as(:d)] }) }
         @interpreter.to_pig_latin.should match(/FOREACH (\w+) GENERATE a.b AS c, a.b AS d/)
       end
     end
+    
+    describe 'FOREACH ... { ... GENERATE }' do
+      it 'outputs a FOREACH ... { ... GENERATE } statement for named fields' do
+        @interpreter.interpret { dump(load('in').nested_foreach { [a, b, c] }) }
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = a;\s+(\w+) = b;\s+(\w+) = c;\s+GENERATE \1, \2, \3;\s+\}/m)
+      end
+      
+      it 'outputs a FOREACH ... { ... GENERATE } statement for positional fields' do
+        @interpreter.interpret { dump(load('in').nested_foreach { [self[0], self[1], self[2]] }) }
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = \$0\;\s+(\w+) = \$1\;\s+(\w+) = \$2\;\s+GENERATE \1, \2, \3\;\s+\}/m)
+      end
+      
+      it 'outputs a FOREACH ... { ... GENERATE } statement with aggregate functions applied to fields' do
+        @interpreter.interpret { dump(load('in').nested_foreach { [a.max, b.min, c.avg] }) }
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = a;\s+(\w+) = MAX\(\1\);\s+(\w+) = b;\s+(\w+) = MIN\(\3\);\s+(\w+) = c;\s+(\w+) = AVG\(\5\);\s+GENERATE \2, \4, \6;\s+\}/m)
+      end
+      
+      it 'outputs a FOREACH ... { ... GENERATE } statement with fields that access inner fields' do
+        @interpreter.interpret { dump(load('in').nested_foreach { [a.b, b.c]}) }
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = a;\s+(\w+) = \1.b;\s+(\w+) = b;\s+(\w+) = \3.c;\s+GENERATE \2, \4;\s+\}/m)
+      end
+      
+      it 'outputs a FOREACH ... { ... GENERATE } statement with user defined functions' do
+        @interpreter.interpret do 
+          define('my_udf', :function => 'com.example.My')
+          dump(load('in').nested_foreach { [my_udf(a, 3, "hello")] })
+        end
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = a;\s+(\w+) = my_udf\(\1, 3, 'hello'\);\s+GENERATE \2;\s+\}/)
+      end
+      
+      it 'outputs a FOREACH ... { ... GENERATE } statement with bag methods' do
+        @interpreter.interpret { dump(load('in').nested_foreach { [self[1].distinct.sample(0.3).limit(5).order(:x).filter { x == 5 }] }) }
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = \$1;\s+(\w+) = DISTINCT \1;\s+(\w+) = SAMPLE \2 0.3;\s+(\w+) = LIMIT \3 5;\s+(\w+) = ORDER \4 BY x;\s+(\w+) = FILTER \5 BY x == 5;\s+GENERATE \6;\s+\}/m)
+      end
+      
+      it 'outputs a FOREACH ... { ... GENERATE } statement with field aliasing' do
+        @interpreter.interpret { dump(load('in').nested_foreach { a = b.distinct; [a.as(:c)] }) }
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = b;\s+(\w+) = DISTINCT \1;\s+GENERATE \2 AS c;\s+\}/)
+      end
+      
+      it 'outputs a FOREACH ... { ... GENERATE } statement with flatten' do
+        @interpreter.interpret { dump(load('in').nested_foreach { [a.flatten] }) }
+        @interpreter.to_pig_latin.should match(/FOREACH \w+ \{\s+(\w+) = a;\s+(\w+) = FLATTEN\(\1\);\s+GENERATE \2;\s+\}/m)
+      end
+    end
 
     describe 'FILTER' do
       it 'outputs a FILTER statement' do
-        @interpreter.interpret { dump(load('in').filter { |r| r.a == 3 }) }
+        @interpreter.interpret { dump(load('in').filter { a == 3 }) }
         @interpreter.to_pig_latin.should match(/FILTER \w+ BY a == 3/)
       end
 
       it 'outputs a FILTER statement with a complex test' do
-        @interpreter.interpret { dump(load('in').filter { |r| (r.a > r.b).and(r.c.ne(3)) }) }
+        @interpreter.interpret { dump(load('in').filter { (a > b).and(c.ne(3)) }) }
         @interpreter.to_pig_latin.should match(/FILTER \w+ BY \(a > b\) AND \(c != 3\)/)
       end
     end
@@ -293,11 +338,11 @@ describe Piglet do
     describe 'SPLIT' do
       it 'outputs a SPLIT statement' do
         @interpreter.interpret do
-          a, b = load('in').split { |r| [r.a >= 0, r.a < 0]}
+          a, b = load('in').split { [first >= 0, second < 0] }
           dump(a)
           dump(b)
         end
-        @interpreter.to_pig_latin.should match(/SPLIT \w+ INTO \w+ IF a >= 0, \w+ IF a < 0/)
+        @interpreter.to_pig_latin.should match(/SPLIT \w+ INTO \w+ IF first >= 0, \w+ IF second < 0/)
       end
     end
     
@@ -517,7 +562,7 @@ describe Piglet do
         output = @interpreter.to_pig_latin do
           define('my_udf', :function => 'com.example.My')
           a = load('in')
-          b = a.foreach { |r| [my_udf('foo', 3, 'hello \'world\'', r[0]).as(:bar)]}
+          b = a.foreach { [my_udf('foo', 3, 'hello \'world\'', self[0]).as(:bar)]}
           store(b, 'out')
         end
         output.should match(/FOREACH \w+ GENERATE my_udf\('foo', 3, 'hello \\'world\\'', \$0\) AS bar/)
@@ -568,35 +613,35 @@ describe Piglet do
   context 'field expressions' do
     it 'parenthesizes expressions with different operators' do
       output = @interpreter.to_pig_latin do
-        store(load('in').filter { |r| r.x.and(r.y.or(r.z)).and(r.w) }, 'out')
+        store(load('in').filter { self.x.and(self.y.or(self.z)).and(self.w) }, 'out')
       end
       output.should include('x AND (y OR z) AND w')
     end
     
     it 'doesn\'t parenthesizes expressions with the same operator' do
       output = @interpreter.to_pig_latin do
-        store(load('in').filter { |r| r.x.and(r.y.and(r.z)).and(r.w) }, 'out')
+        store(load('in').filter { self.x.and(self.y.and(self.z)).and(self.w) }, 'out')
       end
       output.should include('x AND y AND z AND w')
     end
 
     it 'doesn\'t parenthesize function calls' do
       output = @interpreter.to_pig_latin do
-        store(load('in').foreach { |r| [r.x.max + r.y.min] }, 'out')
+        store(load('in').foreach { [self.x.max + self.y.min] }, 'out')
       end
       output.should include('MAX(x) + MIN(y)')
     end
 
     it 'doesn\'t parenthesize a suffix expression followed by an infix expression' do
       output = @interpreter.to_pig_latin do
-        store(load('in').foreach { |r| [r.x.null?.or(r.y)] }, 'out')
+        store(load('in').foreach { [self.x.null?.or(self.y)] }, 'out')
       end
       output.should include('x is null OR y')
     end
 
     it 'parenthesizes a prefix expression followed by an infix expression' do
       output = @interpreter.to_pig_latin do
-        store(load('in').foreach { |r| [r.x.not.and(r.y)] }, 'out')
+        store(load('in').foreach { [self.x.not.and(self.y)] }, 'out')
       end
       output.should include('(NOT x) AND y')
     end
@@ -615,15 +660,15 @@ describe Piglet do
           [:click_thru, :int]
         ])
         %w(site size name).each do |dimension|
-          result = sessions.group(:ad_id, dimension).foreach do |r|
+          result = sessions.group(:ad_id, dimension).foreach do
             [
-              r[0].ad_id.as(:ad_id),
+              self[0].ad_id.as(:ad_id),
               literal(dimension).as(:dimension),
-              r[0].field(dimension).as(:value),
-              r[1].exposure.sum.as(:exposures),
-              r[1].impression.sum.as(:impressions),
-              r[1].engagement.sum.as(:engagements),
-              r[1].click_thru.sum.as(:click_thrus)
+              self[0].field(dimension).as(:value),
+              self[1].exposure.sum.as(:exposures),
+              self[1].impression.sum.as(:impressions),
+              self[1].engagement.sum.as(:engagements),
+              self[1].click_thru.sum.as(:click_thrus)
             ]
           end
           store(result, "report_metrics-#{dimension}")
@@ -800,7 +845,7 @@ describe Piglet do
       schema = catch(:schema) do
         @interpreter.interpret do
           relation1 = load('in1', :schema => [[:a, :float], [:b, :int]])
-          relation2 = relation1.foreach { |r| [r.a] }
+          relation2 = relation1.foreach { [a] }
           throw :schema, relation2.schema
         end
       end
@@ -812,7 +857,7 @@ describe Piglet do
       schema = catch(:schema) do
         @interpreter.interpret do
           relation1 = load('in1', :schema => [[:a, :float], [:b, :int]])
-          relation2 = relation1.foreach { |r| [r.a.max] }
+          relation2 = relation1.foreach { [a.max] }
           throw :schema, relation2.schema
         end
       end
@@ -824,7 +869,7 @@ describe Piglet do
       schema = catch(:schema) do
         @interpreter.interpret do
           relation1 = load('in1', :schema => [[:a, :float], [:b, :int]])
-          relation2 = relation1.foreach { |r| [r.a.count] }
+          relation2 = relation1.foreach { [a.count] }
           throw :schema, relation2.schema
         end
       end
@@ -836,7 +881,7 @@ describe Piglet do
       schema = catch(:schema) do
         @interpreter.interpret do
           relation1 = load('in1', :schema => [[:a, :float], [:b, :int]])
-          relation2 = relation1.foreach { |r| [r.a.count.as(:x)] }
+          relation2 = relation1.foreach { [a.count.as(:x)] }
           throw :schema, relation2.schema
         end
       end
@@ -847,7 +892,7 @@ describe Piglet do
       schema = catch(:schema) do
         @interpreter.interpret do
           relation1 = load('in1', :schema => [[:a, :float], [:b, :int]])
-          relation2 = relation1.foreach { |r| [literal('blipp')] }
+          relation2 = relation1.foreach { [literal('blipp')] }
           throw :schema, relation2.schema
         end
       end
@@ -858,7 +903,7 @@ describe Piglet do
       schema = catch(:schema) do
         @interpreter.interpret do
           relation1 = load('in1', :schema => [[:a, :float], [:b, :int]])
-          relation2 = relation1.foreach { |r| [literal(4)] }
+          relation2 = relation1.foreach { [literal(4)] }
           throw :schema, relation2.schema
         end
       end
@@ -869,7 +914,7 @@ describe Piglet do
       schema = catch(:schema) do
         @interpreter.interpret do
           relation1 = load('in1', :schema => [[:a, :float], [:b, :int]])
-          relation2 = relation1.foreach { |r| [literal(3.14)] }
+          relation2 = relation1.foreach { [literal(3.14)] }
           throw :schema, relation2.schema
         end
       end

data/spec/spec_helper.rb

@@ -9,6 +9,15 @@ require 'spec'
 require 'spec/autorun'
 
 
+class PlainRelation
+  include Piglet::Relation::Relation
+  
+  def initialize(interpreter)
+    @interpreter = interpreter
+  end
+end
+
+
 Spec::Runner.configure do |config|
   
 end

data/tasks/gem.rake

@@ -1,21 +1,18 @@
 # encoding: utf-8
 
-begin
-  require 'jeweler'
-  
-  Jeweler::Tasks.new do |gem|
-    gem.name = "piglet"
-    gem.summary = %Q{Piglet is a DSL for Pig scripts}
-    gem.description = %Q{Piglet aims to look like Pig Latin while allowing for things like loops and control of flow that are missing from Pig.}
-    gem.email = "theo@iconara.net"
-    gem.homepage = "http://github.com/iconara/piglet"
-    gem.authors = ["Theo Hultberg"]
-    gem.add_development_dependency "rspec", ">= 1.2.9"
-    gem.version = Piglet::VERSION
-    gem.test_files = FileList['spec/**/*.rb']
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
-  
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-end
+require 'jeweler'
+
+Jeweler::Tasks.new do |gem|
+  gem.name = "piglet"
+  gem.summary = %Q{Piglet is a DSL for Pig scripts}
+  gem.description = %Q{Piglet aims to look like Pig Latin while allowing for things like loops and control of flow that are missing from Pig.}
+  gem.email = "theo@iconara.net"
+  gem.homepage = "http://github.com/iconara/piglet"
+  gem.authors = ["Theo Hultberg"]
+  gem.add_development_dependency "rspec", ">= 1.2.9"
+  gem.version = Piglet::VERSION
+  gem.test_files = FileList['spec/**/*.rb']
+  # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+end
+
+Jeweler::GemcutterTasks.new

data/tasks/rdoc.rake

@@ -1,9 +1,7 @@
 # encoding: utf-8
 
-gem 'rdoc', '>= 2.4.0'
-
 require 'rake/rdoctask'
-require 'sdoc' rescue LoadError
+require 'sdoc'
 
 
 Rake::RDocTask.new do |rdoc|

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: piglet
 version: !ruby/object:Gem::Version 
-  version: 0.2.5
+  hash: 19
+  prerelease: false
+  segments: 
+  - 0
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Theo Hultberg
@@ -9,19 +15,25 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-16 00:00:00 +01:00
+date: 2010-07-09 00:00:00 +02:00
 default_executable: piglet
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: rspec
   type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  name: rspec
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
         version: 1.2.9
-    version: 
+  requirement: *id001
 description: Piglet aims to look like Pig Latin while allowing for things like loops and control of flow that are missing from Pig.
 email: theo@iconara.net
 executables: 
@@ -31,10 +43,11 @@ extensions: []
 extra_rdoc_files: 
 - LICENSE
 - README.rdoc
-- TODO
 files: 
 - .document
 - .gitignore
+- Gemfile
+- Gemfile.lock
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -42,9 +55,11 @@ files:
 - lib/piglet.rb
 - lib/piglet/field/binary_conditional.rb
 - lib/piglet/field/call_expression.rb
+- lib/piglet/field/direct_expression.rb
 - lib/piglet/field/field.rb
 - lib/piglet/field/infix_expression.rb
 - lib/piglet/field/literal.rb
+- lib/piglet/field/map_value.rb
 - lib/piglet/field/prefix_expression.rb
 - lib/piglet/field/reference.rb
 - lib/piglet/field/rename.rb
@@ -62,6 +77,7 @@ files:
 - lib/piglet/param/declare.rb
 - lib/piglet/param/default.rb
 - lib/piglet/param/parameter_statement.rb
+- lib/piglet/relation/block_context.rb
 - lib/piglet/relation/cogroup.rb
 - lib/piglet/relation/cross.rb
 - lib/piglet/relation/distinct.rb
@@ -70,6 +86,7 @@ files:
 - lib/piglet/relation/group.rb
 - lib/piglet/relation/join.rb
 - lib/piglet/relation/limit.rb
+- lib/piglet/relation/nested_foreach.rb
 - lib/piglet/relation/order.rb
 - lib/piglet/relation/relation.rb
 - lib/piglet/relation/sample.rb
@@ -80,6 +97,7 @@ files:
 - lib/piglet/schema/tuple.rb
 - lib/piglet/udf/define.rb
 - lib/piglet/udf/register.rb
+- piglet.gemspec
 - spec/piglet/field/binary_conditional_spec.rb
 - spec/piglet/field/field_spec.rb
 - spec/piglet/field/infix_expression_spec.rb
@@ -96,7 +114,6 @@ files:
 - tasks/gem.rake
 - tasks/rdoc.rake
 - tasks/spec.rake
-- TODO
 has_rdoc: true
 homepage: http://github.com/iconara/piglet
 licenses: []
@@ -107,21 +124,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Piglet is a DSL for Pig scripts