piglet 0.1.1 → 0.1.2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: piglet
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Theo Hultberg
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-03 00:00:00 +01:00
+date: 2010-01-07 00:00:00 +01:00
 default_executable: piglet
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -38,44 +38,43 @@ files:
 - README.rdoc
 - Rakefile
 - bin/piglet
-- examples/spike1.rb
-- examples/spike2.rb
 - lib/piglet.rb
-- lib/piglet/assignment.rb
-- lib/piglet/cogroup.rb
-- lib/piglet/cross.rb
-- lib/piglet/describe.rb
-- lib/piglet/distinct.rb
-- lib/piglet/dump.rb
-- lib/piglet/explain.rb
-- lib/piglet/field.rb
-- lib/piglet/field_expression_functions.rb
-- lib/piglet/field_function_expression.rb
-- lib/piglet/field_infix_expression.rb
-- lib/piglet/field_prefix_expression.rb
-- lib/piglet/field_rename.rb
-- lib/piglet/field_suffix_expression.rb
-- lib/piglet/filter.rb
-- lib/piglet/foreach.rb
-- lib/piglet/group.rb
-- lib/piglet/illustrate.rb
+- lib/piglet/field/binary_conditional.rb
+- lib/piglet/field/call_expression.rb
+- lib/piglet/field/infix_expression.rb
+- lib/piglet/field/literal.rb
+- lib/piglet/field/operators.rb
+- lib/piglet/field/prefix_expression.rb
+- lib/piglet/field/reference.rb
+- lib/piglet/field/rename.rb
+- lib/piglet/field/suffix_expression.rb
+- lib/piglet/inout/describe.rb
+- lib/piglet/inout/dump.rb
+- lib/piglet/inout/explain.rb
+- lib/piglet/inout/illustrate.rb
+- lib/piglet/inout/load.rb
+- lib/piglet/inout/output.rb
+- lib/piglet/inout/storage_types.rb
+- lib/piglet/inout/store.rb
 - lib/piglet/interpreter.rb
-- lib/piglet/join.rb
-- lib/piglet/limit.rb
-- lib/piglet/load.rb
-- lib/piglet/load_and_store.rb
-- lib/piglet/order.rb
-- lib/piglet/relation.rb
-- lib/piglet/sample.rb
-- lib/piglet/split.rb
-- lib/piglet/store.rb
-- lib/piglet/storing.rb
-- lib/piglet/stream.rb
-- lib/piglet/union.rb
-- spec/piglet/field_spec.rb
+- lib/piglet/relation/cogroup.rb
+- lib/piglet/relation/cross.rb
+- lib/piglet/relation/distinct.rb
+- lib/piglet/relation/filter.rb
+- lib/piglet/relation/foreach.rb
+- lib/piglet/relation/group.rb
+- lib/piglet/relation/join.rb
+- lib/piglet/relation/limit.rb
+- lib/piglet/relation/order.rb
+- lib/piglet/relation/relation.rb
+- lib/piglet/relation/sample.rb
+- lib/piglet/relation/split.rb
+- lib/piglet/relation/stream.rb
+- lib/piglet/relation/union.rb
+- spec/piglet/field/reference_spec.rb
 - spec/piglet/interpreter_spec.rb
-- spec/piglet/relation_spec.rb
-- spec/piglet/split_spec.rb
+- spec/piglet/relation/relation_spec.rb
+- spec/piglet/relation/split_spec.rb
 - spec/piglet_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
@@ -111,9 +110,9 @@ signing_key:
 specification_version: 3
 summary: Piglet is a DSL for Pig scripts
 test_files: 
-- spec/piglet/field_spec.rb
+- spec/piglet/field/reference_spec.rb
 - spec/piglet/interpreter_spec.rb
-- spec/piglet/relation_spec.rb
-- spec/piglet/split_spec.rb
+- spec/piglet/relation/relation_spec.rb
+- spec/piglet/relation/split_spec.rb
 - spec/piglet_spec.rb
 - spec/spec_helper.rb

data/examples/spike1.rb

@@ -1,43 +0,0 @@
-# raw_ads =
-#   LOAD '$INPUT/ads*'
-#   USING PigStorage AS (
-#     ad_id:chararray,
-#     api_key:chararray,
-#     name:chararray,
-#     dimensions:chararray,
-#     destination:chararray,
-#     agent_version:chararray
-#   );
-raw_ads << load('$INPUT/ads*').using(:pig_storage).as(
-  [:ad_id, :chararray],
-  [:api_key, :chararray],
-  [:name, :chararray],
-  [:dimensions, :chararray],
-  [:destination, :chararray],
-  [:agent_version, :chararray]
-)
-
-# ads =
-#   FOREACH
-#     (GROUP raw_ads BY ad_id PARALLEL $PARALLELISM)
-#   GENERATE
-#     $0 AS ad_id,
-#     MAX($1.api_key) AS api_key,
-#     MAX($1.name) AS name,
-#     MAX($1.dimensions) AS dimensions,
-#     MAX($1.destination) AS destination,
-#     MAX($1.agent_version) AS agent_version
-#   ;
-ads << (raw_ads.group(:ad_id)).foreach do |relation|
-  [
-    relation[0].as(:ad_id),
-    relation[1].api_key.max.as(:api_key),
-    relation[1].name.max.as(:name),
-    relation[1].dimensions.max.as(:dimensions),
-    relation[1].destination.max.as(:destination),
-    relation[1].agent_version.max.as(:agent_version)
-  ]
-end
-
-# STORE ads INTO '$OUTPUT/ads' USING PigStorage;
-ads.store('$OUTPUT/ads').using(:pig_storage)

data/examples/spike2.rb

@@ -1,40 +0,0 @@
-# raw_ads =
-#   LOAD '$INPUT/ads*'
-#   USING PigStorage AS (
-#     ad_id:chararray,
-#     api_key:chararray,
-#     name:chararray,
-#     dimensions:chararray,
-#     destination:chararray,
-#     agent_version:chararray
-#   );
-raw_ads = load(
-  '$INPUT/ads*',
-  :using => :pig_storage,
-  :schema => %w(ad_id api_key name dimensions destination agent_version)
-)
-
-# ads =
-#   FOREACH
-#     (GROUP raw_ads BY ad_id PARALLEL $PARALLELISM)
-#   GENERATE
-#     $0 AS ad_id,
-#     MAX($1.api_key) AS api_key,
-#     MAX($1.name) AS name,
-#     MAX($1.dimensions) AS dimensions,
-#     MAX($1.destination) AS destination,
-#     MAX($1.agent_version) AS agent_version
-#   ;
-ads = raw_ads.group(:ad_id, :parallel => 2).foreach do |relation|
-  [
-    relation[0].as(:ad_id),
-    relation[1].api_key.as(:api_key)
-    relation[1].name.max.as(:name)
-    relation[1].dimensions.max.as(:dimensions)
-    relation[1].destination.max.as(:destination)
-    relation[1].agent_version.max.as(:agent_version)
-  ]
-end
-
-# STORE ads INTO '$OUTPUT/ads' USING PigStorage;
-store(ads, '$OUTPUT/ads', :using => :pig_storage)

data/lib/piglet/assignment.rb

@@ -1,13 +0,0 @@
-module Piglet
-  class Assignment # :nodoc:
-    attr_reader :target
-    
-    def initialize(relation)
-      @target = relation
-    end
-    
-    def to_s
-      "#{@target.alias} = #{@target.to_s}"
-    end
-  end
-end

data/lib/piglet/cogroup.rb

@@ -1,31 +0,0 @@
-module Piglet
-  class Cogroup # :nodoc:
-    include Relation
-    
-    def initialize(relation, description)
-      @join_fields = description.reject { |k, v| ! (k.is_a?(Relation)) }
-      @sources = @join_fields.keys
-      @parallel = description[:parallel]
-    end
-    
-    def to_s
-      joins = @sources.map do |s|
-        fields = @join_fields[s]
-        if fields.is_a?(Enumerable) && fields.size > 1 && (fields.last == :inner || fields.last == :outer)
-          inout = fields.last.to_s.upcase
-          fields = fields[0..-2]
-        end
-        if fields.is_a?(Enumerable) && fields.size > 1
-          str = "#{s.alias} BY (#{fields.join(', ')})"
-        else
-          str = "#{s.alias} BY #{fields}"
-        end
-        str << " #{inout}" if inout
-        str
-      end
-      str  = "COGROUP #{joins.join(', ')}"
-      str << " PARALLEL #{@parallel}" if @parallel
-      str
-    end
-  end
-end

data/lib/piglet/cross.rb

@@ -1,22 +0,0 @@
-module Piglet
-  class Cross # :nodoc:
-    include Relation
-    
-    def initialize(relations, options={})
-      options ||= {}
-      @sources, @parallel = relations, options[:parallel]
-    end
-    
-    def to_s
-      str  = "CROSS #{source_aliases.join(', ')}"
-      str << " PARALLEL #{@parallel}" if @parallel
-      str
-    end
-  
-  private
-    
-    def source_aliases
-      @sources.map { |s| s.alias }
-    end
-  end
-end

data/lib/piglet/describe.rb

@@ -1,5 +0,0 @@
-module Piglet
-  class Describe # :nodoc:
-    include Storing
-  end
-end

data/lib/piglet/distinct.rb

@@ -1,16 +0,0 @@
-module Piglet
-  class Distinct # :nodoc:
-    include Relation
-    
-    def initialize(relation, options={})
-      options ||= {}
-      @sources, @parallel = [relation], options[:parallel]
-    end
-    
-    def to_s
-      str  = "DISTINCT #{@sources.first.alias}"
-      str << " PARALLEL #{@parallel}" if @parallel
-      str
-    end
-  end
-end

data/lib/piglet/dump.rb

@@ -1,5 +0,0 @@
-module Piglet
-  class Dump # :nodoc:
-    include Storing
-  end
-end

data/lib/piglet/explain.rb

@@ -1,13 +0,0 @@
-module Piglet
-  class Explain # :nodoc:
-    include Storing
-    
-    def to_s
-      if relation.nil?
-        "EXPLAIN"
-      else
-        super
-      end
-    end
-  end
-end

data/lib/piglet/field.rb

@@ -1,40 +0,0 @@
-module Piglet
-  class Field # :nodoc:
-    include FieldExpressionFunctions
-    
-    def initialize(name, relation=nil, options=nil)
-      options ||= {}
-      @name, @parent = name, relation
-      @explicit_ancestry = options[:explicit_ancestry] || false
-    end
-    
-    def simple?
-      true
-    end
-    
-    def method_missing(name, *args)
-      if name.to_s =~ /^\w+$/ && args.empty?
-        Field.new(name, self, :explicit_ancestry => true)
-      else
-        super
-      end
-    end
-    
-    def [](n)
-      Field.new("\$#{n}", self, :explicit_ancestry => true)
-    end
-    
-    def to_s
-      if @explicit_ancestry
-        if @parent.respond_to?(:alias)
-          "#{@parent.alias}.#{@name.to_s}"
-        else
-          "#{@parent}.#{@name.to_s}"
-        end
-      else
-        @name.to_s
-      end
-    end
-  end
-
-end

data/lib/piglet/field_expression_functions.rb

@@ -1,62 +0,0 @@
-module Piglet
-  module FieldExpressionFunctions # :nodoc:
-    SYMBOLIC_OPERATORS = [:==, :>, :<, :>=, :<=, :%, :+, :-, :*, :/]
-    FUNCTIONS = [:avg, :count, :diff, :max, :min, :size, :sum, :tokenize]
-    
-    FUNCTIONS.each do |fun|
-      define_method(fun) { FieldFunctionExpression.new(fun.to_s.upcase, self) }
-    end
-
-    def empty?
-      FieldFunctionExpression.new('IsEmpty', self)
-    end
-    
-    def as(new_name)
-      FieldRename.new(new_name, self)
-    end
-    
-    def not
-      FieldPrefixExpression.new('NOT', self)
-    end
-    
-    def null?
-      FieldSuffixExpression.new('is null', self)
-    end
-    
-    def not_null?
-      FieldSuffixExpression.new('is not null', self)
-    end
-    
-    def cast(type)
-      FieldPrefixExpression.new("(#{type.to_s})", self)
-    end
-    
-    def matches(pattern)
-      regex_options_pattern = /^\(\?.+?:(.*)\)$/
-      pattern = pattern.to_s.sub(regex_options_pattern, '\1') if pattern.is_a?(Regexp) && pattern.to_s =~ regex_options_pattern
-      FieldInfixExpression.new('matches', self, "'#{pattern.to_s}'")
-    end
-    
-    def neg
-      FieldPrefixExpression.new('-', self, false)
-    end
-    
-    def ne(other)
-      FieldInfixExpression.new('!=', self, other)
-    end
-    
-    SYMBOLIC_OPERATORS.each do |op|
-      define_method(op) { |other| FieldInfixExpression.new(op.to_s, self, other) }
-    end
-    
-  protected
-  
-    def parenthesise(expr)
-      if expr.respond_to?(:simple?) && ! expr.simple?
-        "(#{expr})"
-      else
-        expr.to_s
-      end
-    end
-  end
-end

data/lib/piglet/field_function_expression.rb

@@ -1,19 +0,0 @@
-module Piglet
-  class FieldFunctionExpression # :nodoc:
-    include FieldExpressionFunctions
-    
-    def initialize(name, inner_expression, options=nil)
-      options ||= {}
-      @name, @inner_expression = name, inner_expression
-      @new_name = options[:as]
-    end
-    
-    def simple?
-      false
-    end
-    
-    def to_s
-      "#{@name}(#{@inner_expression})"
-    end
-  end
-end

data/lib/piglet/field_infix_expression.rb

@@ -1,17 +0,0 @@
-module Piglet
-  class FieldInfixExpression # :nodoc:
-    include FieldExpressionFunctions
-    
-    def initialize(operator, left_expression, right_expression)
-      @operator, @left_expression, @right_expression = operator, left_expression, right_expression
-    end
-    
-    def simple?
-      false
-    end
-    
-    def to_s
-      "#{parenthesise(@left_expression)} #{@operator} #{parenthesise(@right_expression)}"
-    end
-  end
-end

data/lib/piglet/field_prefix_expression.rb

@@ -1,21 +0,0 @@
-module Piglet
-  class FieldPrefixExpression # :nodoc:
-    include FieldExpressionFunctions
-    
-    def initialize(operator, expression, space_between=true)
-      @operator, @expression, @space_between = operator, expression, space_between
-    end
-    
-    def simple?
-      true
-    end
-    
-    def to_s
-      if @space_between
-        "#{@operator} #{parenthesise(@expression)}"
-      else
-        "#{@operator}#{parenthesise(@expression)}"
-      end
-    end
-  end
-end

data/lib/piglet/field_rename.rb

@@ -1,11 +0,0 @@
-module Piglet
-  class FieldRename # :nodoc:
-    def initialize(new_name, field_expression)
-      @new_name, @field_expression = new_name, field_expression
-    end
-    
-    def to_s
-      "#{@field_expression} AS #{@new_name}"
-    end
-  end
-end

data/lib/piglet/field_suffix_expression.rb

@@ -1,17 +0,0 @@
-module Piglet
-  class FieldSuffixExpression # :nodoc:
-    include FieldExpressionFunctions
-    
-    def initialize(operator, expression)
-      @operator, @expression = operator, expression
-    end
-    
-    def simple?
-      false
-    end
-    
-    def to_s
-      "#{parenthesise(@expression)} #{@operator}"
-    end
-  end
-end

data/lib/piglet/filter.rb

@@ -1,13 +0,0 @@
-module Piglet
-  class Filter # :nodoc:
-    include Relation
-    
-    def initialize(relation, expression)
-      @sources, @expression = [relation], expression
-    end
-    
-    def to_s
-      "FILTER #{@sources.first.alias} BY #{@expression}"
-    end
-  end
-end

data/lib/piglet/foreach.rb

@@ -1,19 +0,0 @@
-module Piglet
-  class Foreach # :nodoc:
-    include Relation
-    
-    def initialize(relation, field_expressions)
-      @sources, @field_expressions = [relation], [field_expressions].flatten
-    end
-    
-    def to_s
-      "FOREACH #{@sources.first.alias} GENERATE #{field_expressions_string}"
-    end
-    
-  private
-  
-    def field_expressions_string
-      @field_expressions.map { |fe| fe.to_s }.join(', ')
-    end
-  end
-end

data/lib/piglet/group.rb

@@ -1,21 +0,0 @@
-module Piglet
-  class Group # :nodoc:
-    include Relation
-    
-    def initialize(relation, grouping, options={})
-      options ||= {}
-      @sources, @grouping, @parallel = [relation], grouping, options[:parallel]
-    end
-    
-    def to_s
-      str = "GROUP #{@sources.first.alias} BY "
-      if @grouping.size > 1
-        str << "(#{@grouping.join(', ')})"
-      else
-        str << @grouping.first.to_s
-      end
-      str << " PARALLEL #{@parallel}" if @parallel
-      str
-    end
-  end
-end

data/lib/piglet/illustrate.rb

@@ -1,5 +0,0 @@
-module Piglet
-  class Illustrate # :nodoc:
-    include Storing
-  end
-end

data/lib/piglet/join.rb

@@ -1,20 +0,0 @@
-module Piglet
-  class Join # :nodoc:
-    include Relation
-    
-    def initialize(relation, description)
-      @join_fields = Hash[*description.select { |k, v| k.is_a?(Relation) }.flatten]
-      @sources = @join_fields.keys
-      @using = description[:using]
-      @parallel = description[:parallel]
-    end
-    
-    def to_s
-      joins = @sources.map { |s| "#{s.alias} BY #{@join_fields[s]}" }.join(', ')
-      str  = "JOIN #{joins}"
-      str << " USING \"#{@using.to_s}\"" if @using
-      str << " PARALLEL #{@parallel}" if @parallel
-      str
-    end
-  end
-end

data/lib/piglet/limit.rb

@@ -1,13 +0,0 @@
-module Piglet
-  class Limit # :nodoc:
-    include Relation
-    
-    def initialize(relation, n)
-      @sources, @n = [relation], n
-    end
-    
-    def to_s
-      "LIMIT #{@sources.first.alias} #{@n}"
-    end
-  end
-end

data/lib/piglet/load.rb

@@ -1,31 +0,0 @@
-module Piglet
-  class Load # :nodoc:
-    include Relation
-    include LoadAndStore
-    
-    def initialize(path, options={})
-      options ||= {}
-      @path, @using, @schema = path, options[:using], options[:schema]
-    end
-    
-    def to_s
-      str  = "LOAD '#{@path}'"
-      str << " USING #{resolve_load_store_function(@using)}" if @using
-      str << " AS (#{schema_string})" if @schema
-      str
-    end
-    
-  private
-        
-    def schema_string
-      @schema.map do |field|
-        if field.is_a?(Enumerable)
-          field.map { |f| f.to_s }.join(':')
-        else
-          field.to_s
-        end
-      end.join(', ')
-    end
-    
-  end
-end

data/lib/piglet/load_and_store.rb

@@ -1,16 +0,0 @@
-module Piglet  
-  module LoadAndStore # :nodoc:
-    LOAD_STORE_FUNCTIONS = {
-      :binary_serializer => 'BinarySerializer',
-      :binary_deserializer => 'BinaryDeserializer',
-      :bin_storage => 'BinStorage',
-      :pig_storage => 'PigStorage',
-      :pig_dump => 'PigDump',
-      :text_loader => 'TextLoader'
-    }
-    
-    def resolve_load_store_function(name)
-      LOAD_STORE_FUNCTIONS[name] || name.to_s
-    end
-  end
-end

data/lib/piglet/order.rb

@@ -1,29 +0,0 @@
-module Piglet
-  class Order # :nodoc:
-    include Relation
-    
-    def initialize(relation, fields, options)
-      options ||= {}
-      @sources, @parallel = [relation], options[:parallel]
-      @fields = fields.is_a?(Enumerable) ? fields : [fields]
-    end
-    
-    def to_s
-      "ORDER #{@sources.first.alias} BY #{field_strings}"
-    end
-    
-  private
-  
-    def field_strings
-      @fields.map { |f| field_string(f) }.join(', ')
-    end
-    
-    def field_string(f)
-      if f.is_a?(Enumerable)
-        "#{f[0]} #{f[1].to_s.upcase}"
-      else
-        f.to_s
-      end
-    end
-  end
-end