piglet 0.1.0

data/lib/piglet/field_prefix_expression.rb

@@ -0,0 +1,21 @@
+module Piglet
+  class FieldPrefixExpression # :nodoc:
+    include FieldExpressionFunctions
+    
+    def initialize(operator, expression, space_between=true)
+      @operator, @expression, @space_between = operator, expression, space_between
+    end
+    
+    def simple?
+      true
+    end
+    
+    def to_s
+      if @space_between
+        "#{@operator} #{parenthesise(@expression)}"
+      else
+        "#{@operator}#{parenthesise(@expression)}"
+      end
+    end
+  end
+end

data/lib/piglet/field_rename.rb

@@ -0,0 +1,11 @@
+module Piglet
+  class FieldRename # :nodoc:
+    def initialize(new_name, field_expression)
+      @new_name, @field_expression = new_name, field_expression
+    end
+    
+    def to_s
+      "#{@field_expression} AS #{@new_name}"
+    end
+  end
+end

data/lib/piglet/field_suffix_expression.rb

@@ -0,0 +1,17 @@
+module Piglet
+  class FieldSuffixExpression # :nodoc:
+    include FieldExpressionFunctions
+    
+    def initialize(operator, expression)
+      @operator, @expression = operator, expression
+    end
+    
+    def simple?
+      false
+    end
+    
+    def to_s
+      "#{parenthesise(@expression)} #{@operator}"
+    end
+  end
+end

data/lib/piglet/filter.rb

@@ -0,0 +1,13 @@
+module Piglet
+  class Filter # :nodoc:
+    include Relation
+    
+    def initialize(relation, expression)
+      @sources, @expression = [relation], expression
+    end
+    
+    def to_s
+      "FILTER #{@sources.first.alias} BY #{@expression}"
+    end
+  end
+end

data/lib/piglet/foreach.rb

@@ -0,0 +1,19 @@
+module Piglet
+  class Foreach # :nodoc:
+    include Relation
+    
+    def initialize(relation, field_expressions)
+      @sources, @field_expressions = [relation], [field_expressions].flatten
+    end
+    
+    def to_s
+      "FOREACH #{@sources.first.alias} GENERATE #{field_expressions_string}"
+    end
+    
+  private
+  
+    def field_expressions_string
+      @field_expressions.map { |fe| fe.to_s }.join(', ')
+    end
+  end
+end

data/lib/piglet/group.rb

@@ -0,0 +1,21 @@
+module Piglet
+  class Group # :nodoc:
+    include Relation
+    
+    def initialize(relation, grouping, options={})
+      options ||= {}
+      @sources, @grouping, @parallel = [relation], grouping, options[:parallel]
+    end
+    
+    def to_s
+      str = "GROUP #{@sources.first.alias} BY "
+      if @grouping.size > 1
+        str << "(#{@grouping.join(', ')})"
+      else
+        str << @grouping.first.to_s
+      end
+      str << " PARALLEL #{@parallel}" if @parallel
+      str
+    end
+  end
+end

data/lib/piglet/illustrate.rb

@@ -0,0 +1,5 @@
+module Piglet
+  class Illustrate # :nodoc:
+    include Storing
+  end
+end

data/lib/piglet/interpreter.rb

@@ -0,0 +1,108 @@
+require 'set'
+
+
+module Piglet
+  class Interpreter
+    def initialize(&block)
+      @stores = [ ]
+      
+      interpret(&block) if block_given?
+    end
+  
+    def interpret(&block)
+      if block_given?
+        instance_eval(&block)
+      end
+    
+      self
+    end
+    
+    def to_pig_latin
+      return '' if @stores.empty?
+      
+      handled_relations = Set.new
+      statements = [ ]
+      
+      @stores.each do |store|
+        unless store.relation.nil?
+          assignments(store.relation, handled_relations).each do |assignment|
+            statements << assignment
+            handled_relations << assignment.target
+          end
+        end
+        statements << store
+      end
+      
+      statements.flatten.map { |s| s.to_s }.join(";\n") + ";\n"
+    end
+  
+  protected
+
+    # LOAD
+    #
+    #   load('some/path')                                         # => LOAD 'some/path'
+    #   load('some/path', :using => 'Xyz')                        # => LOAD 'some/path' USING Xyz
+    #   load('some/path', :using => :pig_storage)                 # => LOAD 'some/path' USING PigStorage
+    #   load('some/path', :schema => [:a, :b])                    # => LOAD 'some/path' AS (a, b)
+    #   load('some/path', :schema => %w(a b c d))                 # => LOAD 'some/path' AS (a, b, c, d)
+    #   load('some/path', :schema => [%w(a chararray), %(b int)]) # => LOAD 'some/path' AS (a:chararray, b:int)
+    #
+    #--
+    #
+    # NOTE: the syntax load('path', :schema => {:a => :chararray, :b => :int})
+    # would be nice, but the order of the keys can't be guaranteed in Ruby 1.8.
+    def load(path, options={})
+      Load.new(path, options)
+    end
+  
+    # STORE
+    #
+    #   store(x, 'some/path') # => STORE x INTO 'some/path'
+    #   store(x, 'some/path', :using => 'Xyz') # => STORE x INTO 'some/path' USING Xyz
+    #   store(x, 'some/path', :using => :pig_storage) # => STORE x INTO 'some/path' USING PigStorage
+    def store(relation, path, options={})
+      @stores << Store.new(relation, path, options)
+    end
+  
+    # DUMP
+    #
+    #   dump(x) # => DUMP x
+    def dump(relation)
+      @stores << Dump.new(relation)
+    end
+  
+    # ILLUSTRATE
+    #
+    #   illustrate(x) # => ILLUSTRATE x
+    def illustrate(relation)
+      @stores << Illustrate.new(relation)
+    end
+  
+    # DESCRIBE
+    #
+    #   describe(x) # => DESCRIBE x
+    def describe(relation)
+      @stores << Describe.new(relation)
+    end
+  
+    # EXPLAIN
+    #
+    #   explain    # => EXPLAIN
+    #   explain(x) # => EXPLAIN(x)
+    def explain(relation=nil)
+      @stores << Explain.new(relation)
+    end
+  
+  private
+  
+    def assignments(relation, ignore_set)
+      return [] if ignore_set.include?(relation)
+      assignment = Assignment.new(relation)
+      if relation.sources
+        (relation.sources.map { |source| assignments(source, ignore_set) } + [assignment]).flatten
+      else
+        [assignment]
+      end
+    end
+  end  
+end

data/lib/piglet/join.rb

@@ -0,0 +1,20 @@
+module Piglet
+  class Join # :nodoc:
+    include Relation
+    
+    def initialize(relation, description)
+      @join_fields = Hash[*description.select { |k, v| k.is_a?(Relation) }.flatten]
+      @sources = @join_fields.keys
+      @using = description[:using]
+      @parallel = description[:parallel]
+    end
+    
+    def to_s
+      joins = @sources.map { |s| "#{s.alias} BY #{@join_fields[s]}" }.join(', ')
+      str  = "JOIN #{joins}"
+      str << " USING \"#{@using.to_s}\"" if @using
+      str << " PARALLEL #{@parallel}" if @parallel
+      str
+    end
+  end
+end

data/lib/piglet/limit.rb

@@ -0,0 +1,13 @@
+module Piglet
+  class Limit # :nodoc:
+    include Relation
+    
+    def initialize(relation, n)
+      @sources, @n = [relation], n
+    end
+    
+    def to_s
+      "LIMIT #{@sources.first.alias} #{@n}"
+    end
+  end
+end

data/lib/piglet/load.rb

@@ -0,0 +1,31 @@
+module Piglet
+  class Load # :nodoc:
+    include Relation
+    include LoadAndStore
+    
+    def initialize(path, options={})
+      options ||= {}
+      @path, @using, @schema = path, options[:using], options[:schema]
+    end
+    
+    def to_s
+      str  = "LOAD '#{@path}'"
+      str << " USING #{resolve_load_store_function(@using)}" if @using
+      str << " AS (#{schema_string})" if @schema
+      str
+    end
+    
+  private
+        
+    def schema_string
+      @schema.map do |field|
+        if field.is_a?(Enumerable)
+          field.map { |f| f.to_s }.join(':')
+        else
+          field.to_s
+        end
+      end.join(', ')
+    end
+    
+  end
+end

data/lib/piglet/load_and_store.rb

@@ -0,0 +1,16 @@
+module Piglet  
+  module LoadAndStore # :nodoc:
+    LOAD_STORE_FUNCTIONS = {
+      :binary_serializer => 'BinarySerializer',
+      :binary_deserializer => 'BinaryDeserializer',
+      :bin_storage => 'BinStorage',
+      :pig_storage => 'PigStorage',
+      :pig_dump => 'PigDump',
+      :text_loader => 'TextLoader'
+    }
+    
+    def resolve_load_store_function(name)
+      LOAD_STORE_FUNCTIONS[name] || name.to_s
+    end
+  end
+end

data/lib/piglet/order.rb

@@ -0,0 +1,29 @@
+module Piglet
+  class Order # :nodoc:
+    include Relation
+    
+    def initialize(relation, fields, options)
+      options ||= {}
+      @sources, @parallel = [relation], options[:parallel]
+      @fields = fields.is_a?(Enumerable) ? fields : [fields]
+    end
+    
+    def to_s
+      "ORDER #{@sources.first.alias} BY #{field_strings}"
+    end
+    
+  private
+  
+    def field_strings
+      @fields.map { |f| field_string(f) }.join(', ')
+    end
+    
+    def field_string(f)
+      if f.is_a?(Enumerable)
+        "#{f[0]} #{f[1].to_s.upcase}"
+      else
+        f.to_s
+      end
+    end
+  end
+end

data/lib/piglet/relation.rb

@@ -0,0 +1,177 @@
+module Piglet
+  module Relation
+    attr_reader :sources
+
+    # The name this relation will get in Pig Latin. Then name is generated when
+    # the relation is outputed by the interpreter, and will be unique.
+    def alias
+      @alias ||= Relation.next_alias
+    end
+  
+    # GROUP
+    #
+    #   x.group(:a)                           # => GROUP x By a
+    #   x.group(:a, :b, :c)                   # => GROUP x BY (a, b, c)
+    #   x.group([:a, :b, :c], :parallel => 3) # => GROUP x BY (a, b, c) PARALLEL 3
+    def group(*args)
+      grouping, options = split_at_options(args)
+      Group.new(self, [grouping].flatten, options)
+    end
+  
+    # DISTINCT
+    #
+    #   x.distinct                 # => DISTINCT x
+    #   x.distinct(:parallel => 5) # => DISTINCT x PARALLEL 5
+    def distinct(options={})
+      Distinct.new(self, options)
+    end
+
+    # COGROUP
+    #
+    #   x.cogroup(x => :a, y => :b)                 # => COGROUP x BY a, y BY b
+    #   x.cogroup(x => :a, y => :b, z => :c)        # => COGROUP x BY a, y BY b, z BY c
+    #   x.cogroup(x => [:a, :b], y => [:c, :d])     # => COGROUP x BY (a, b), y BY (c, d)
+    #   x.cogroup(x => :a, y => [:b, :inner])       # => COGROUP x BY a, y BY b INNER
+    #   x.cogroup(x => :a, y => :b, :parallel => 5) # => COGROUP x BY a, y BY b PARALLEL 5
+    def cogroup(description)
+      Cogroup.new(self, description)
+    end
+  
+    # CROSS
+    #
+    #   x.cross(y)                      # => CROSS x, y
+    #   x.cross(y, z, w)                # => CROSS x, y, z, w
+    #   x.cross([y, z], :parallel => 5) # => CROSS x, y, z, w PARALLEL 5
+    def cross(*args)
+      relations, options = split_at_options(args)
+      Cross.new(([self] + relations).flatten, options)
+    end
+  
+    # FILTER
+    #
+    #   x.filter { |r| r.a == r.b }            # => FILTER x BY a == b
+    #   x.filter { |r| r.a > r.b && r.c != 3 } # => FILTER x BY a > b AND c != 3
+    def filter
+      Filter.new(self, yield(self))
+    end
+  
+    # FOREACH ... GENERATE
+    #
+    #   x.foreach { |r| r.a }            # => FOREACH x GENERATE a
+    #   x.foreach { |r| [r.a, r.b] }     # => FOREACH x GENERATE a, b
+    #   x.foreach { |r| r.a.max }        # => FOREACH x GENERATE MAX(a)
+    #   x.foreach { |r| r.a.avg.as(:b) } # => FOREACH x GENERATE AVG(a) AS b
+    #
+    #--
+    #
+    # TODO: FOREACH a { b GENERATE c }
+    def foreach
+      Foreach.new(self, yield(self))
+    end
+  
+    # JOIN
+    #
+    #   x.join(x => :a, y => :b)                        # => JOIN x BY a, y BY b
+    #   x.join(x => :a, y => :b, z => :c)               # => JOIN x BY a, y BY b, z BY c
+    #   x.join(x => :a, y => :b, :using => :replicated) # => JOIN x BY a, y BY b USING "replicated"
+    #   x.join(x => :a, y => :b, :parallel => 5)        # => JOIN x BY a, y BY b PARALLEL 5
+    def join(description)
+      Join.new(self, description)
+    end
+  
+    # LIMIT
+    #
+    #   x.limit(10) # => LIMIT x 10
+    def limit(n)
+      Limit.new(self, n)
+    end
+  
+    # ORDER
+    #
+    #   x.order(:a)                      # => ORDER x BY a
+    #   x.order(:a, :b)                  # => ORDER x BY a, b
+    #   x.order([:a, :asc], [:b, :desc]) # => ORDER x BY a ASC, b DESC
+    #   x.order(:a, :parallel => 5)      # => ORDER x BY a PARALLEL 5
+    #
+    #--
+    #
+    # NOTE: the syntax x.order(:a => :asc, :b => :desc) would be nice, but in
+    # Ruby 1.8 the order of the keys cannot be guaranteed.
+    def order(*args)
+      fields, options = split_at_options(args)
+      fields = *fields
+      Order.new(self, fields, options)
+    end
+  
+    # SAMPLE
+    #
+    #   x.sample(5) # => SAMPLE x 5;
+    def sample(n)
+      Sample.new(self, n)
+    end
+    
+    # SPLIT
+    #
+    #   y, z = x.split { |r| [r.a <= 3, r.b > 4]} # => SPLIT x INTO y IF a <= 3, z IF a > 4
+    def split
+      Split.new(self, yield(self)).shards
+    end
+  
+    # STREAM
+    #
+    #   x.stream(x, 'cut -f 3')                         # => STREAM x THROUGH `cut -f 3`
+    #   x.stream([x, y], 'cut -f 3')                    # => STREAM x, y THROUGH `cut -f 3`
+    #   x.stream(x, 'cut -f 3', :schema => [%w(a int)]) # => STREAM x THROUGH `cut -f 3` AS (a:int)
+    #
+    #--
+    #
+    # TODO: how to handle DEFINE'd commands?
+    def stream(relations, command, options={})
+      raise NotSupportedError
+    end
+  
+    # UNION
+    #
+    #   x.union(y)    # => UNION x, y
+    #   x.union(y, z) # => UNION x, y, z
+    def union(*relations)
+      Union.new(*([self] + relations))
+    end
+
+    def method_missing(name, *args)
+      if name.to_s =~ /^\w+$/ && args.empty?
+        Field.new(name, self)
+      else
+        super
+      end
+    end
+    
+    def [](n)
+      Field.new("\$#{n}", self)
+    end
+
+    def hash
+      self.alias.hash
+    end
+  
+    def eql?(other)
+      other.is_a?(Relation) && other.alias == self.alias
+    end
+  
+  private
+
+    def split_at_options(parameters)
+      if parameters.last.is_a? Hash
+        [parameters[0..-2], parameters.last]
+      else
+        [parameters, nil]
+      end
+    end
+
+    def self.next_alias
+      @counter ||= 0
+      @counter += 1
+      "relation_#{@counter}"
+    end
+  end
+end

data/lib/piglet/sample.rb

@@ -0,0 +1,13 @@
+module Piglet
+  class Sample # :nodoc:
+    include Relation
+    
+    def initialize(relation, n)
+      @sources, @n = [relation], n
+    end
+    
+    def to_s
+      "SAMPLE #{@sources.first.alias} #{@n}"
+    end
+  end
+end

data/lib/piglet/split.rb

@@ -0,0 +1,41 @@
+module Piglet
+  class Split # :nodoc:
+    include Relation
+    
+    
+    def initialize(relation, expressions)
+      @sources, @expressions = [relation], expressions
+      @shard_map = create_shards
+    end
+    
+    def shards
+      @shard_map.keys
+    end
+    
+    def to_s
+      "SPLIT #{@sources.first.alias} INTO #{split_strings}"
+    end
+    
+  private
+  
+    def create_shards
+      Hash[*@expressions.map { |expr| [RelationShard.new(self), expr] }.flatten]
+    end
+  
+    def split_strings
+      shards.map { |relation| "#{relation.alias} IF #{@shard_map[relation]}" }.join(', ')
+    end
+  end
+  
+  class RelationShard # :nodoc:
+    include Relation
+    
+    def initialize(split)
+      @sources = [split]
+    end
+    
+    def to_s
+      self.alias
+    end
+  end
+end

data/lib/piglet/store.rb

@@ -0,0 +1,17 @@
+module Piglet
+  class Store # :nodoc:
+    include LoadAndStore
+    include Storing
+    
+    def initialize(relation, path, options={})
+      @relation, @path, @using = relation, path, options[:using]
+    end
+    
+    def to_s
+      str  = super
+      str << " INTO '#{@path}'"
+      str << " USING #{resolve_load_store_function(@using)}" if @using
+      str
+    end
+  end
+end

data/lib/piglet/storing.rb

@@ -0,0 +1,13 @@
+module Piglet
+  module Storing # :nodoc:
+    attr_reader :relation
+    
+    def initialize(relation)
+      @relation = relation
+    end
+    
+    def to_s
+      "#{self.class.name.split(/::/).last.upcase} #{@relation.alias}"
+    end
+  end
+end

data/lib/piglet/stream.rb

@@ -0,0 +1,5 @@
+module Piglet
+  class Stream # :nodoc:
+    include Relation
+  end
+end

data/lib/piglet/union.rb

@@ -0,0 +1,19 @@
+module Piglet
+  class Union # :nodoc:
+    include Relation
+    
+    def initialize(*relations)
+      @sources = relations
+    end
+    
+    def to_s
+      "UNION #{source_aliases.join(', ')}"
+    end
+    
+  private
+  
+    def source_aliases
+      @sources.map { |s| s.alias }
+    end
+  end
+end

data/lib/piglet.rb

@@ -0,0 +1,45 @@
+module Piglet
+  VERSION = '0.1.0'
+  
+  autoload_files = %w(
+    assignment
+    cogroup
+    cross
+    describe
+    distinct
+    dump
+    explain
+    field
+    field_expression_functions
+    field_function_expression
+    field_infix_expression
+    field_prefix_expression
+    field_rename
+    field_suffix_expression
+    filter
+    foreach
+    group
+    illustrate
+    interpreter
+    join
+    limit
+    load
+    load_and_store
+    order
+    relation
+    sample
+    split
+    store
+    storing
+    stream
+    union
+  )
+  
+  autoload_files.each do |f|
+    c = f.split('_').map { |s| s.capitalize }.join.to_sym
+    p = "piglet/#{f}"
+    autoload c, p
+  end
+
+  class NotSupportedError < StandardError; end
+end