piglet 0.2.4 → 0.2.5

data/README.rdoc

@@ -14,6 +14,14 @@ The code above will be translated to the following Pig Latin:
 
 Piglet aims to look like Pig Latin while allowing for things like loops and control of flow that are missing from Pig. I started working on Piglet out of frustration that my Pig scripts started to be very repetitive. Pig lacks control of flow and mechanisms to apply the same set of operations on multiple relations. Piglet is my way of adding those features.
 
+== Installation
+
+If you have Gemcutter.org as a source, just do
+
+  gem install piglet
+
+there are no dependencies.
+
 == Usage
 
 It can be used either as a command line tool for translating a file of Piglet code into Pig Latin, or you can use it inline in a Ruby script:
@@ -257,6 +265,12 @@ You can also use +define+ to define function references:
 
   define(:hello, :function => 'com.example.Hello') # => DEFINE hello com.example.Hello
 
+When you define a UDF it becomes available as a method in the interpreter scope. This means that you can refer to it by name in, for example, a <code>FOREACH … GENERATE</code> statement:
+
+  define :awesome, :function => 'my.awesome.Function' # => DEFINE awesome my.awesome.Function
+  …
+  b = a.foreach { |r| [awesome(r[0]).as(:something_special)] } # => b = FOREACH a GENERATE awesome($0) AS something_special
+
 If you need to register a JAR you can use +register+:
 
   register('path/to/lib.jar') # => REGISTER path/to/lib.jar
@@ -277,6 +291,17 @@ For some operators in Pig Latin you can specify the <code>PARALLEL</code> keywor
 
 For the +cogroup+, +cross+, +distinct+, +group+, +join+ and +order+ you can pass  <code>:parallel => <em>n</em></code> as the last parameter to specify the amount of parallelism, e.g. <code>a.group(:x, :y, :z, :parallel => 5)</code>.
 
+== <code>%declare</code> & <code>%default</code>
+
+The <code>%declare</code> and <code>%default</code> preprocessor macros are available as +declare+ and +default+. Each take two parameters, a name and a value:
+
+  declare(:foo, 'bar')     # => %declare foo 'bar'
+  default('hello', :world) # => %default hello 'world'
+
+If you want to quote the value with backticks, pass <code>:backticks => true</code> as the third parameter:
+
+  default 'CMD', 'uniq', :backticks => true
+
 === Putting it all together
 
 Let's look at a more complex example:
@@ -404,16 +429,12 @@ The following Pig operators are supported:
 * +STREAM+
 * +UNION+
 
-The following are currently not supported (but will be soon):
+The following is currently not supported (but will be soon):
 
 * <code>FOREACH { … } GENERATE</code>
-* <code>%DECLARE</code>
-* <code>%DEFAULT</code>
 
 The file commands (+cd+, +cat+, etc.) will probably not be supported for the forseeable future.
 
-UDF:s are sort of supported. You can declare a UDF with +declare+ and you can use it in some situations by using a string, but some situations are trickier (like renaming the result of a UDF call in a <code>FOREACH … GENERATE</code> body).
-
 All the aggregate functions except two are supported:
 
 * +AVG+

data/Rakefile

@@ -2,4 +2,4 @@ require 'lib/piglet'
 
 task :default => :spec
 
-Dir['tasks/*.rake'].each { |t| load t }
+Dir[File.join(File.dirname(__FILE__), 'tasks', '*.rake')].each { |t| load t }

data/TODO

@@ -1,4 +1,2 @@
-* STREAM
 * DEFINE + gör så att funktionen går att köra från Piglet i ett foreach-block
-* REGISTER
 * DIFF

data/lib/piglet/field/binary_conditional.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class BinaryConditional

data/lib/piglet/field/call_expression.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class CallExpression # :nodoc:

data/lib/piglet/field/field.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     module Field # :nodoc:

data/lib/piglet/field/infix_expression.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class InfixExpression # :nodoc:

data/lib/piglet/field/literal.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class Literal

data/lib/piglet/field/prefix_expression.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class PrefixExpression # :nodoc:

data/lib/piglet/field/reference.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class Reference # :nodoc:

data/lib/piglet/field/rename.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class Rename # :nodoc:

data/lib/piglet/field/suffix_expression.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Field
     class SuffixExpression # :nodoc:

data/lib/piglet/field/udf_expression.rb

@@ -0,0 +1,28 @@
+module Piglet
+  module Field
+    class UdfExpression
+      include Field
+      
+      def initialize(ali4s, *args)
+        @alias, @args = ali4s, args
+      end
+      
+      def to_s
+        "#{@alias}(#{args_to_s(@args)})"
+      end
+      
+    private
+      
+      def args_to_s(arg)
+        case arg
+        when String
+          "'#{escape(arg)}'"
+        when Enumerable
+          arg.map { |a| args_to_s(a) }.join(', ')
+        else
+          arg
+        end
+      end
+    end
+  end
+end

data/lib/piglet/inout/describe.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Inout
     class Describe # :nodoc:

data/lib/piglet/inout/dump.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Inout
     class Dump # :nodoc:

data/lib/piglet/inout/explain.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Inout
     class Explain # :nodoc:

data/lib/piglet/inout/illustrate.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Inout
     class Illustrate # :nodoc:

data/lib/piglet/inout/load.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Inout
     class Load # :nodoc:

data/lib/piglet/inout/output.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Inout
     module Output # :nodoc:

data/lib/piglet/inout/storage_types.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet  
   module Inout
     module StorageTypes # :nodoc:

data/lib/piglet/inout/store.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Inout
     class Store # :nodoc:

data/lib/piglet/interpreter.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'set'
 
 
@@ -119,6 +121,34 @@ module Piglet
     #   :output => [{:to => :stdout, :using => 'MySerializer'}, 'some/path'] # => OUTPUT(stdout USING MySerializer, 'some/path')
     def define(ali4s, options=nil)
       @top_level_statements << Udf::Define.new(ali4s, options)
+      unless respond_to?(ali4s)
+        def metaclass
+          class << self
+            return self
+          end
+        end
+        metaclass.send(:define_method, ali4s) do |*args|
+          Field::UdfExpression.new(ali4s, *args)
+        end
+      end
+    end
+    
+    # %declare
+    #
+    #   declare(:my_var, 'value')                  # => %declare my_var 'value'
+    #   declare('quote', "He said 'hello!'")       # => %declare quote 'He said \'hello!\''
+    #   declare('cmd', 'uniq', :backticks => true) # => %declare cmd `uniq`
+    def declare(name, value, options=nil)
+      @top_level_statements << Param::Declare.new(name, value, options)
+    end
+    
+    # %default
+    #
+    #   default(:my_var, 'value')                  # => %default my_var 'value'
+    #   default('quote', "He said 'hello!'")       # => %default quote 'He said \'hello!\''
+    #   default('cmd', 'uniq', :backticks => true) # => %default cmd `uniq`
+    def default(name, value, options=nil)
+      @top_level_statements << Param::Default.new(name, value, options)
     end
     
     # Support for binary conditions, a.k.a. the ternary operator.

data/lib/piglet/param/declare.rb

@@ -0,0 +1,11 @@
+module Piglet
+  module Param
+    class Declare
+      include ParameterStatement
+      def initialize(name, value, options=nil)
+        options ||= {}
+        @kind, @name, @value, @backticks = 'declare', name, value, options[:backticks]
+      end
+    end
+  end
+end

data/lib/piglet/param/default.rb

@@ -0,0 +1,11 @@
+module Piglet
+  module Param
+    class Default
+      include ParameterStatement
+      def initialize(name, value, options=nil)
+        options ||= {}
+        @kind, @name, @value, @backticks = 'default', name, value, options[:backticks]
+      end
+    end
+  end
+end

data/lib/piglet/param/parameter_statement.rb

@@ -0,0 +1,25 @@
+module Piglet
+  module Param
+    module ParameterStatement
+      def to_s
+        if @backticks
+          v = "`#{@value}`"
+        else
+          case @value
+          when String, Symbol
+            v = "'#{escape(@value)}'"
+          else
+            v = @value
+          end
+        end
+        "%#{@kind} #{@name} #{v}"
+      end
+      
+    private
+    
+      def escape(str)
+        str.to_s.gsub(/('|\\)/) { |m| "\\#{$1}" }
+      end
+    end
+  end
+end

data/lib/piglet/relation/cogroup.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Cogroup # :nodoc:
@@ -31,7 +33,10 @@ module Piglet
             inout = fields.last.to_s.upcase
             fields = fields[0..-2]
           end
-          if fields.is_a?(Enumerable) && fields.size > 1
+          if fields.is_a?(Enumerable) && fields.size == 1
+            fields = fields.first
+          end
+          if fields.is_a?(Enumerable)
             str = "#{s.alias} BY (#{fields.join(', ')})"
           else
             str = "#{s.alias} BY #{fields}"

data/lib/piglet/relation/cross.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Cross # :nodoc:

data/lib/piglet/relation/distinct.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Distinct # :nodoc:

data/lib/piglet/relation/filter.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Filter # :nodoc:

data/lib/piglet/relation/foreach.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Foreach # :nodoc:

data/lib/piglet/relation/group.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Group # :nodoc:

data/lib/piglet/relation/join.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Join # :nodoc:

data/lib/piglet/relation/limit.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Limit # :nodoc:

data/lib/piglet/relation/order.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Order # :nodoc:

data/lib/piglet/relation/relation.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     module Relation

data/lib/piglet/relation/sample.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Sample # :nodoc:

data/lib/piglet/relation/split.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Split # :nodoc:

data/lib/piglet/relation/stream.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Stream # :nodoc:

data/lib/piglet/relation/union.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Relation
     class Union # :nodoc:

data/lib/piglet/schema/bag.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Schema
     class Bag

data/lib/piglet/schema/tuple.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Schema
     class Tuple

data/lib/piglet/udf/define.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Udf
     class Define

data/lib/piglet/udf/register.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Piglet
   module Udf
     class Register

data/lib/piglet.rb

@@ -1,6 +1,8 @@
+# encoding: utf-8
+
 # :main: README.rdoc
 module Piglet # :nodoc:
-  VERSION = '0.2.4'
+  VERSION = '0.2.5'
   
   class PigletError < StandardError; end
   class NotSupportedError < PigletError; end
@@ -45,6 +47,7 @@ module Piglet # :nodoc:
     autoload :Reference, 'piglet/field/reference'
     autoload :Rename, 'piglet/field/rename'
     autoload :SuffixExpression, 'piglet/field/suffix_expression'
+    autoload :UdfExpression, 'piglet/field/udf_expression'
   end
   
   module Schema
@@ -58,4 +61,10 @@ module Piglet # :nodoc:
     autoload :Define, 'piglet/udf/define'
     autoload :Register, 'piglet/udf/register'
   end
+  
+  module Param
+    autoload :Declare, 'piglet/param/declare'
+    autoload :Default, 'piglet/param/default'
+    autoload :ParameterStatement, 'piglet/param/parameter_statement'
+  end
 end

data/spec/piglet/field/binary_conditional_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/field/field_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/field/infix_expression_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/field/literal_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/field/reference_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/interpreter_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 

data/spec/piglet/relation/relation_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/relation/split_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/relation/union_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
 

data/spec/piglet/schema/tuple_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.dirname(__FILE__) + '/../../spec_helper'
 
 

data/spec/piglet_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 
@@ -104,6 +106,37 @@ describe Piglet do
       end
     end
   end
+
+  context 'parameter declarations:' do
+    %w(declare default).each do |op|
+      describe "%#{op}" do
+        it "outputs a %#{op} statement" do
+          @interpreter.interpret { self.send(op, :my_var, 'my_value') }
+          @interpreter.to_pig_latin.should match(/%#{op} my_var 'my_value'/)
+        end
+      
+        it "outputs a %#{op} statement with single quotes escaped" do
+          @interpreter.interpret { self.send(op, :my_var, "my 'value'") }
+          @interpreter.to_pig_latin.should match(/%#{op} my_var 'my \\'value\\''/)
+        end
+
+        it "outputs a %#{op} statement with an numeric value unquoted" do
+          @interpreter.interpret { self.send(op, :my_var, 1) }
+          @interpreter.to_pig_latin.should match(/%#{op} my_var 1/)
+        end
+
+        it "outputs a %#{op} statement with an symbol value quoted" do
+          @interpreter.interpret { self.send(op, :my_var, :x) }
+          @interpreter.to_pig_latin.should match(/%#{op} my_var 'x'/)
+        end
+
+        it "outputs a %#{op} statement with the value quoted in backticks, if the option :backticks => true is passed" do
+          @interpreter.interpret { self.send(op, :my_var, 'cut -f 4', :backticks => true) }
+          @interpreter.to_pig_latin.should match(/%#{op} my_var `cut -f 4`/)
+        end
+      end
+    end
+  end
   
   context 'relation operators:' do
     describe 'GROUP' do
@@ -479,6 +512,16 @@ describe Piglet do
         end
         output.should include('DEFINE plunk `plunk.rb` INPUT(\'some/path\' USING PigStorage, stdin USING HelloWorld(\'test\')) OUTPUT(\'some/other/path\' USING BinStorage, stdout USING SomeOtherMechanism()) SHIP(\'to/here\') CACHE(\'first\', \'second\', \'third\')')
       end
+      
+      it 'makes the defined UDF available as a method in the interpreter scope, so that it can be used in a FOREACH and it\'s result renamed using AS' do
+        output = @interpreter.to_pig_latin do
+          define('my_udf', :function => 'com.example.My')
+          a = load('in')
+          b = a.foreach { |r| [my_udf('foo', 3, 'hello \'world\'', r[0]).as(:bar)]}
+          store(b, 'out')
+        end
+        output.should match(/FOREACH \w+ GENERATE my_udf\('foo', 3, 'hello \\'world\\'', \$0\) AS bar/)
+      end
     end
     
     describe 'REGISTER' do

data/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 

data/tasks/gem.rake

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 begin
   require 'jeweler'
   

data/tasks/rdoc.rake

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 gem 'rdoc', '>= 2.4.0'
 
 require 'rake/rdoctask'

data/tasks/spec.rake

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'spec/rake/spectask'
 
 Spec::Rake::SpecTask.new(:spec) do |spec|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: piglet
 version: !ruby/object:Gem::Version 
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors: 
 - Theo Hultberg
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-14 00:00:00 +01:00
+date: 2010-01-16 00:00:00 +01:00
 default_executable: piglet
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -49,6 +49,7 @@ files:
 - lib/piglet/field/reference.rb
 - lib/piglet/field/rename.rb
 - lib/piglet/field/suffix_expression.rb
+- lib/piglet/field/udf_expression.rb
 - lib/piglet/inout/describe.rb
 - lib/piglet/inout/dump.rb
 - lib/piglet/inout/explain.rb
@@ -58,6 +59,9 @@ files:
 - lib/piglet/inout/storage_types.rb
 - lib/piglet/inout/store.rb
 - lib/piglet/interpreter.rb
+- lib/piglet/param/declare.rb
+- lib/piglet/param/default.rb
+- lib/piglet/param/parameter_statement.rb
 - lib/piglet/relation/cogroup.rb
 - lib/piglet/relation/cross.rb
 - lib/piglet/relation/distinct.rb