piglet 0.1.0

data/examples/analysis.rb

@@ -0,0 +1,311 @@
+# raw_sessions = 
+#   LOAD '$INPUT/sessions*'
+#   USING PigStorage AS (
+#     date:chararray,
+#     api_key:chararray,
+#     ad_id:chararray,
+#     user_id:chararray,
+#     site:chararray,
+#     size:chararray,
+#     name:chararray,
+#     destination:chararray,
+#     indeterminate_visibility:int,
+#     impression:int,
+#     engagement:int,
+#     click_thru:int,
+#     extra:int,
+#     session_time:int,
+#     visible_time:int,
+#     engagement_time:int
+#   );
+raw_sessions = load('$INPUT/sessions*', :schema => [
+  [:date :chararray],
+  [:api_key :chararray],
+  [:ad_id :chararray],
+  [:user_id :chararray],
+  [:site :chararray],
+  [:size :chararray],
+  [:name :chararray],
+  [:destination :chararray],
+  [:indeterminate_visibility :int],
+  [:impression :int],
+  [:engagement :int],
+  [:click_thru :int],
+  [:extra :int],
+  [:session_time :int],
+  [:visible_time :int],
+  [:engagement_time :int]
+])
+
+# raw_actions = 
+#   LOAD '$INPUT/actions*'
+#   USING PigStorage AS (
+#     date:chararray,
+#     api_key:chararray,
+#     ad_id:chararray,
+#     user_id:chararray,
+#     action:chararray,
+#     site:chararray,
+#     size:chararray,
+#     name:chararray,
+#     destination:chararray,
+#     extra:int
+#   );
+raw_actions = load('$INPUT/actions*', :schema => 
+  [:date :chararray],
+  [:api_key :chararray],
+  [:ad_id :chararray],
+  [:user_id :chararray],
+  [:action :chararray],
+  [:site :chararray],
+  [:size :chararray],
+  [:name :chararray],
+  [:destination :chararray],
+  [:extra :int]
+)
+
+#sessions = FILTER raw_sessions BY date is not null;
+sessions = raw_sessions.filter { |r| r.date.not_null? }
+
+#actions = FILTER raw_actions  BY date is not null;
+actions = raw_actions.filter { |r| r.date.not_null? }
+
+# /*
+#  * Modify each session and action based on whether or not it's an extra session
+#  * (a session that was logged only because it was a click thru). Extra sessions
+#  * should affect only the total number of click thrus, not the number of
+#  * exposures, impressions, etc. nor the durations. By setting these values to
+#  * zero and introducing a field for whether or not the session was an exposure
+#  * (zero for extra sessions, one for all other), the calculations below can
+#  * filter out extra sessions without too much work. 
+#  */
+# sessions =
+#   FOREACH
+#     sessions
+#   GENERATE
+#     date,
+#     api_key,
+#     ad_id,
+#     user_id,
+#     site,
+#     size,
+#     name,
+#     destination,
+#     (extra == 1 ? 0 : indeterminate_visibility) AS indeterminate_visibility,
+#     (extra == 1 ? 0 : 1) AS exposure,
+#     (extra == 1 ? 0 : impression) AS impression,
+#     (extra == 1 ? 0 : engagement) AS engagement,
+#     click_thru,
+#     (extra == 1 ? 0 : session_time) AS session_time,
+#     (extra == 1 ? 0 : visible_time) AS visible_time,
+#     (extra == 1 ? 0 : engagement_time) AS engagement_time;
+sessions = sessions.foreach do |r|
+  [
+    r.date, 
+    r.api_key, 
+    r.ad_id, 
+    r.user_id,
+    r.site,
+    r.size,
+    r.name,
+    r.destination,
+    r.test(r.extra == 1, 0, r.indeterminate_visibility).as(:indeterminate_visibility),
+    r.test(r.extra == 1, 0, 1).as(:exposure),
+    r.test(r.extra == 1, 0, r.impression).as(:impression),
+    r.test(r.extra == 1, 0, r.engagement).as(:engagement),
+    r.click_thru,
+    r.test(r.extra == 1, 0, r.session_time).as(:session_time),
+    r.test(r.extra == 1, 0, r.visible_time).as(:visible_time),
+    r.test(r.extra == 1, 0, r.engagement_time).as(:engagement_time)
+  ]
+end
+    
+# actions =
+#   FOREACH
+#     actions
+#   GENERATE
+#     date,
+#     api_key,
+#     ad_id,
+#     user_id,
+#     action,
+#     site,
+#     size,
+#     name,
+#     destination,
+#     (extra == 1 ? 0 : 1) AS exposure;
+actions = actions.foreach do |r|
+  [
+    r.date,
+    r.api_key,
+    r.ad_id,
+    r.user_id,
+    r.action,
+    r.site,
+    r.size,
+    r.name,
+    r.destination,
+    r.test(r.extra == 1, 0, 1).as(:exposure)
+  ]
+end
+
+%w(all site size name).each do |name|
+  # session_category_<%= name %> =
+  #   FOREACH 
+  #     (GROUP sessions BY (date, ad_id, api_key, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
+  #   GENERATE 
+  #     $0.date AS date,
+  #     $0.ad_id AS ad_id,
+  #     $0.api_key AS api_key,
+  #     '<%= name %>' AS category,
+  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
+  #     SUM($1.exposure) AS exposures,
+  #     SUM($1.impression) AS impressions,
+  #     SUM($1.engagement) AS engagements,
+  #     SUM($1.click_thru) AS click_thrus,
+  #     SUM($1.indeterminate_visibility) AS indeterminate_visibility,
+  #     SUM($1.session_time) AS session_time,
+  #     SUM($1.visible_time) AS visible_time,
+  #     SUM($1.engagement_time) AS engagement_time;
+  session_category = sessions.group(:date, :ad_id, :api_key, (name == 'all' ? 'all' : name), :parallel => '$PARALLELISM')
+  session_category = session_category.foreach do |r|
+    [
+      r[0].date.as(:date),
+      r[0].ad_id.as(:ad_id),
+      r[0].api_key.as(:api_key),
+      name.as(:category),
+      (name == 'all' ? "'all'" : r[0].name).as(:segment),
+      r[1].sum.as(:exposure),
+      r[1].sum.as(:impression),
+      r[1].sum.as(:engagement),
+      r[1].sum.as(:click_thru),
+      r[1].sum.as(:indeterminate_visibility),
+      r[1].sum.as(:session_time),
+      r[1].sum.as(:visible_time),
+      r[1].sum.as(:engagement_time)
+    ]
+  end
+
+  # session_category_<%= name %>_by_user_id =
+  #   FOREACH
+  #     (GROUP sessions BY (date, ad_id, api_key, user_id, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
+  #   GENERATE
+  #     $0.date AS date,
+  #     $0.ad_id AS ad_id,
+  #     $0.api_key AS api_key,
+  #     '<%= name %>' AS category,
+  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
+  #     1 AS exposures,
+  #     MAX($1.impression) AS impressions,
+  #     MAX($1.engagement) AS engagements,
+  #     MAX($1.click_thru) AS click_thrus;
+  session_category_by_user_id = sessions.group(:date, :ad_id, :api_key, (name == 'all' ? 'all' : name), :parallel => '$PARALLELISM')
+  session_category_by_user_id = session_category_by_user_id.foreach do |r|
+    r[0].date.as(:date),
+    r[0].ad_id.as(:ad_id),
+    r[0].api_key.as(:api_key),
+    name.as(:category),
+    (name == 'all' ? "'all'" : r[0].name).as(:segment)
+    1.as(:exposures),
+    r[1].impression.max.as(:impressions),
+    r[1].engagement.max.as(:engagements),
+    r[1].click_thru.max.as(:click_thrus)
+  end
+
+  # unique_session_category_<%= name %> =
+  #   FOREACH
+  #     (GROUP session_category_<%= name %>_by_user_id BY (date, ad_id, api_key, category, segment) PARALLEL $PARALLELISM)
+  #   GENERATE
+  #     $0.date AS date,
+  #     $0.ad_id AS ad_id,
+  #     $0.api_key AS api_key,
+  #     $0.category,
+  #     $0.segment,
+  #     COUNT($1.ad_id)     AS unique_exposures,
+  #     SUM($1.impressions) AS unique_impressions,
+  #     SUM($1.engagements) AS unique_engagements,
+  #     SUM($1.click_thrus) AS unique_click_thrus;
+  # 
+  # action_category_<%= name %> =
+  #   FOREACH 
+  #     (GROUP actions BY (date, ad_id, api_key, action, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
+  #   GENERATE 
+  #     $0.date AS date,
+  #     $0.ad_id AS ad_id,
+  #     $0.api_key AS api_key,
+  #     $0.action AS action,
+  #     '<%= name %>' AS category,
+  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
+  #     SUM($1.exposure) AS engagements;
+  # 
+  # action_category_<%= name %>_by_user_id =
+  #   FOREACH
+  #     (GROUP actions BY (date, ad_id, api_key, action, user_id, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
+  #   GENERATE
+  #     $0.date AS date,
+  #     $0.ad_id AS ad_id,
+  #     $0.api_key AS api_key,
+  #     $0.action AS action,
+  #     '<%= name %>' AS category,
+  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
+  #     1 AS exposures,
+  #     1 AS engagements;
+  # 
+  # unique_action_category_<%= name %> =
+  #   FOREACH
+  #     (GROUP action_category_<%= name %>_by_user_id BY (date, ad_id, api_key, action, category, segment) PARALLEL $PARALLELISM)
+  #   GENERATE
+  #     $0.date AS date,
+  #     $0.ad_id AS ad_id,
+  #     $0.api_key AS api_key,
+  #     $0.action AS action,
+  #     $0.category,
+  #     $0.segment,
+  #     SUM($1.engagements) AS unique_engagements;
+end
+
+-- unions ----------------------------------------------------------------------
+-- -----------------------------------------------------------------------------
+
+<% if @categories.size > 1 -%>
+report_metrics =
+  UNION
+    <%= @categories.map { |name| "session_category_#{name}" }.join(",\n    ") %>;
+<% else -%>
+report_metrics = FILTER session_category_<%= @categories.first %> BY 1 == 1;
+<% end -%>
+
+<% if @categories.size > 1 -%>
+unique_report_metrics =
+  UNION
+    <%= @categories.map { |name| "unique_session_category_#{name}" }.join(",\n    ") %>;
+<% else -%>
+unique_report_metrics = FILTER unique_session_category_<%= @categories.first %> BY 1 == 1;
+<% end -%>
+<% if @categories.size > 1 -%>  
+report_action_metrics =
+  UNION
+    <%= @categories.map { |name| "action_category_#{name}" }.join(",\n    ") %>;
+<% else -%>
+report_action_metrics = FILTER action_category_<%= @categories.first %> BY 1 == 1;
+<% end -%>
+<% if @categories.size > 1 -%>  
+unique_report_action_metrics =
+  UNION
+    <%= @categories.map { |name| "unique_action_category_#{name}" }.join(",\n    ") %>;
+<% else -%>
+unique_report_action_metrics = FILTER unique_action_category_<%= @categories.first %> BY 1 == 1;
+<% end %>
+
+-- complete output -------------------------------------------------------------
+-- -----------------------------------------------------------------------------
+
+
+<% %w(report_metrics unique_report_metrics report_action_metrics unique_report_action_metrics).each do |relation| -%>
+<%= relation %> = FILTER <%= relation %> BY date is not null AND date != '' AND api_key is not null AND api_key != '';
+<% end -%>
+
+STORE report_metrics INTO '$OUTPUT/report_metrics' USING PigStorage;
+STORE unique_report_metrics INTO '$OUTPUT/unique_report_metrics' USING PigStorage;
+STORE report_action_metrics INTO '$OUTPUT/report_action_metrics' USING PigStorage;
+STORE unique_report_action_metrics INTO '$OUTPUT/unique_report_action_metrics' USING PigStorage;

data/examples/scratch.rb

@@ -0,0 +1,11 @@
+module Piglet::Relation
+  def samples(*sizes)
+    sizes.map { |s| sample(s) }
+  end
+end
+
+input = load('input', :schema => %w(country browser site visit_duration))
+a, b, c = input.samples(0.1, 0.2, 0.3)
+store(a, 'output1')
+store(b, 'output2')
+store(c, 'output3')

data/examples/spike1.rb

@@ -0,0 +1,43 @@
+# raw_ads =
+#   LOAD '$INPUT/ads*'
+#   USING PigStorage AS (
+#     ad_id:chararray,
+#     api_key:chararray,
+#     name:chararray,
+#     dimensions:chararray,
+#     destination:chararray,
+#     agent_version:chararray
+#   );
+raw_ads << load('$INPUT/ads*').using(:pig_storage).as(
+  [:ad_id, :chararray],
+  [:api_key, :chararray],
+  [:name, :chararray],
+  [:dimensions, :chararray],
+  [:destination, :chararray],
+  [:agent_version, :chararray]
+)
+
+# ads =
+#   FOREACH
+#     (GROUP raw_ads BY ad_id PARALLEL $PARALLELISM)
+#   GENERATE
+#     $0 AS ad_id,
+#     MAX($1.api_key) AS api_key,
+#     MAX($1.name) AS name,
+#     MAX($1.dimensions) AS dimensions,
+#     MAX($1.destination) AS destination,
+#     MAX($1.agent_version) AS agent_version
+#   ;
+ads << (raw_ads.group(:ad_id)).foreach do |relation|
+  [
+    relation[0].as(:ad_id),
+    relation[1].api_key.max.as(:api_key),
+    relation[1].name.max.as(:name),
+    relation[1].dimensions.max.as(:dimensions),
+    relation[1].destination.max.as(:destination),
+    relation[1].agent_version.max.as(:agent_version)
+  ]
+end
+
+# STORE ads INTO '$OUTPUT/ads' USING PigStorage;
+ads.store('$OUTPUT/ads').using(:pig_storage)

data/examples/spike2.rb

@@ -0,0 +1,40 @@
+# raw_ads =
+#   LOAD '$INPUT/ads*'
+#   USING PigStorage AS (
+#     ad_id:chararray,
+#     api_key:chararray,
+#     name:chararray,
+#     dimensions:chararray,
+#     destination:chararray,
+#     agent_version:chararray
+#   );
+raw_ads = load(
+  '$INPUT/ads*',
+  :using => :pig_storage,
+  :schema => %w(ad_id api_key name dimensions destination agent_version)
+)
+
+# ads =
+#   FOREACH
+#     (GROUP raw_ads BY ad_id PARALLEL $PARALLELISM)
+#   GENERATE
+#     $0 AS ad_id,
+#     MAX($1.api_key) AS api_key,
+#     MAX($1.name) AS name,
+#     MAX($1.dimensions) AS dimensions,
+#     MAX($1.destination) AS destination,
+#     MAX($1.agent_version) AS agent_version
+#   ;
+ads = raw_ads.group(:ad_id, :parallel => 2).foreach do |relation|
+  [
+    relation[0].as(:ad_id),
+    relation[1].api_key.as(:api_key)
+    relation[1].name.max.as(:name)
+    relation[1].dimensions.max.as(:dimensions)
+    relation[1].destination.max.as(:destination)
+    relation[1].agent_version.max.as(:agent_version)
+  ]
+end
+
+# STORE ads INTO '$OUTPUT/ads' USING PigStorage;
+store(ads, '$OUTPUT/ads', :using => :pig_storage)

data/examples/test1.rb

@@ -0,0 +1,3 @@
+raw_data = load 'test1-data.txt', :schema => %w(name city country)
+grouped_by_country = raw_data.group :country
+dump grouped_by_country

data/examples/test2.rb

@@ -0,0 +1,5 @@
+a = load('in', :schema => %w(x y z w))
+%w(x y z w).each do |f|
+  r = a.group(f)
+  store(r, 'out-' + f)
+end

data/examples/test3.rb

@@ -0,0 +1,4 @@
+a = load 'input', :schema => [:a, :b, :c]
+b = a.group :c
+c = b.foreach { |r| [r[0], r[1].a.max, r[1].b.max] }
+store c, 'output'

data/lib/piglet/assignment.rb

@@ -0,0 +1,13 @@
+module Piglet
+  class Assignment # :nodoc:
+    attr_reader :target
+    
+    def initialize(relation)
+      @target = relation
+    end
+    
+    def to_s
+      "#{@target.alias} = #{@target.to_s}"
+    end
+  end
+end

data/lib/piglet/cogroup.rb

@@ -0,0 +1,31 @@
+module Piglet
+  class Cogroup # :nodoc:
+    include Relation
+    
+    def initialize(relation, description)
+      @join_fields = description.reject { |k, v| ! (k.is_a?(Relation)) }
+      @sources = @join_fields.keys
+      @parallel = description[:parallel]
+    end
+    
+    def to_s
+      joins = @sources.map do |s|
+        fields = @join_fields[s]
+        if fields.is_a?(Enumerable) && fields.size > 1 && (fields.last == :inner || fields.last == :outer)
+          inout = fields.last.to_s.upcase
+          fields = fields[0..-2]
+        end
+        if fields.is_a?(Enumerable) && fields.size > 1
+          str = "#{s.alias} BY (#{fields.join(', ')})"
+        else
+          str = "#{s.alias} BY #{fields}"
+        end
+        str << " #{inout}" if inout
+        str
+      end
+      str  = "COGROUP #{joins.join(', ')}"
+      str << " PARALLEL #{@parallel}" if @parallel
+      str
+    end
+  end
+end

data/lib/piglet/cross.rb

@@ -0,0 +1,22 @@
+module Piglet
+  class Cross # :nodoc:
+    include Relation
+    
+    def initialize(relations, options={})
+      options ||= {}
+      @sources, @parallel = relations, options[:parallel]
+    end
+    
+    def to_s
+      str  = "CROSS #{source_aliases.join(', ')}"
+      str << " PARALLEL #{@parallel}" if @parallel
+      str
+    end
+  
+  private
+    
+    def source_aliases
+      @sources.map { |s| s.alias }
+    end
+  end
+end

data/lib/piglet/describe.rb

@@ -0,0 +1,5 @@
+module Piglet
+  class Describe # :nodoc:
+    include Storing
+  end
+end

data/lib/piglet/distinct.rb

@@ -0,0 +1,16 @@
+module Piglet
+  class Distinct # :nodoc:
+    include Relation
+    
+    def initialize(relation, options={})
+      options ||= {}
+      @sources, @parallel = [relation], options[:parallel]
+    end
+    
+    def to_s
+      str  = "DISTINCT #{@sources.first.alias}"
+      str << " PARALLEL #{@parallel}" if @parallel
+      str
+    end
+  end
+end

data/lib/piglet/dump.rb

@@ -0,0 +1,5 @@
+module Piglet
+  class Dump # :nodoc:
+    include Storing
+  end
+end

data/lib/piglet/explain.rb

@@ -0,0 +1,13 @@
+module Piglet
+  class Explain # :nodoc:
+    include Storing
+    
+    def to_s
+      if relation.nil?
+        "EXPLAIN"
+      else
+        super
+      end
+    end
+  end
+end

data/lib/piglet/field.rb

@@ -0,0 +1,40 @@
+module Piglet
+  class Field # :nodoc:
+    include FieldExpressionFunctions
+    
+    def initialize(name, relation=nil, options=nil)
+      options ||= {}
+      @name, @parent = name, relation
+      @explicit_ancestry = options[:explicit_ancestry] || false
+    end
+    
+    def simple?
+      true
+    end
+    
+    def method_missing(name, *args)
+      if name.to_s =~ /^\w+$/ && args.empty?
+        Field.new(name, self, :explicit_ancestry => true)
+      else
+        super
+      end
+    end
+    
+    def [](n)
+      Field.new("\$#{n}", self, :explicit_ancestry => true)
+    end
+    
+    def to_s
+      if @explicit_ancestry
+        if @parent.respond_to?(:alias)
+          "#{@parent.alias}.#{@name.to_s}"
+        else
+          "#{@parent}.#{@name.to_s}"
+        end
+      else
+        @name.to_s
+      end
+    end
+  end
+
+end

data/lib/piglet/field_expression_functions.rb

@@ -0,0 +1,62 @@
+module Piglet
+  module FieldExpressionFunctions # :nodoc:
+    SYMBOLIC_OPERATORS = [:==, :>, :<, :>=, :<=, :%, :+, :-, :*, :/]
+    FUNCTIONS = [:avg, :count, :diff, :max, :min, :size, :sum, :tokenize]
+    
+    FUNCTIONS.each do |fun|
+      define_method(fun) { FieldFunctionExpression.new(fun.to_s.upcase, self) }
+    end
+
+    def empty?
+      FieldFunctionExpression.new('IsEmpty', self)
+    end
+    
+    def as(new_name)
+      FieldRename.new(new_name, self)
+    end
+    
+    def not
+      FieldPrefixExpression.new('NOT', self)
+    end
+    
+    def null?
+      FieldSuffixExpression.new('is null', self)
+    end
+    
+    def not_null?
+      FieldSuffixExpression.new('is not null', self)
+    end
+    
+    def cast(type)
+      FieldPrefixExpression.new("(#{type.to_s})", self)
+    end
+    
+    def matches(pattern)
+      regex_options_pattern = /^\(\?.+?:(.*)\)$/
+      pattern = pattern.to_s.sub(regex_options_pattern, '\1') if pattern.is_a?(Regexp) && pattern.to_s =~ regex_options_pattern
+      FieldInfixExpression.new('matches', self, "'#{pattern.to_s}'")
+    end
+    
+    def neg
+      FieldPrefixExpression.new('-', self, false)
+    end
+    
+    def ne(other)
+      FieldInfixExpression.new('!=', self, other)
+    end
+    
+    SYMBOLIC_OPERATORS.each do |op|
+      define_method(op) { |other| FieldInfixExpression.new(op.to_s, self, other) }
+    end
+    
+  protected
+  
+    def parenthesise(expr)
+      if expr.respond_to?(:simple?) && ! expr.simple?
+        "(#{expr})"
+      else
+        expr.to_s
+      end
+    end
+  end
+end

data/lib/piglet/field_function_expression.rb

@@ -0,0 +1,19 @@
+module Piglet
+  class FieldFunctionExpression # :nodoc:
+    include FieldExpressionFunctions
+    
+    def initialize(name, inner_expression, options=nil)
+      options ||= {}
+      @name, @inner_expression = name, inner_expression
+      @new_name = options[:as]
+    end
+    
+    def simple?
+      false
+    end
+    
+    def to_s
+      "#{@name}(#{@inner_expression})"
+    end
+  end
+end

data/lib/piglet/field_infix_expression.rb

@@ -0,0 +1,17 @@
+module Piglet
+  class FieldInfixExpression # :nodoc:
+    include FieldExpressionFunctions
+    
+    def initialize(operator, left_expression, right_expression)
+      @operator, @left_expression, @right_expression = operator, left_expression, right_expression
+    end
+    
+    def simple?
+      false
+    end
+    
+    def to_s
+      "#{parenthesise(@left_expression)} #{@operator} #{parenthesise(@right_expression)}"
+    end
+  end
+end