piglet 0.1.0 → 0.1.1

data/Rakefile

@@ -1,50 +1,5 @@
-require 'rubygems'
-require 'rake'
 require 'lib/piglet'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "piglet"
-    gem.summary = %Q{Piglet is a DSL for Pig scripts}
-    gem.description = %Q{Piglet aims to look like Pig Latin while allowing for things like loops and control of flow that are missing from Pig.}
-    gem.email = "theo@iconara.net"
-    gem.homepage = "http://github.com/iconara/piglet"
-    gem.authors = ["Theo Hultberg"]
-    gem.add_development_dependency "rspec", ">= 1.2.9"
-    gem.version = Piglet::VERSION
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
-
-require 'spec/rake/spectask'
-
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
-end
-
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-
-task :spec => :check_dependencies
-
 task :default => :spec
 
-require 'rake/rdoctask'
-
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "piglet #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-  rdoc.options << '--charset' << 'utf-8'
-end
+Dir['tasks/*.rake'].each { |t| load t }

data/lib/piglet.rb

@@ -1,5 +1,6 @@
-module Piglet
-  VERSION = '0.1.0'
+# :main: README.rdoc
+module Piglet # :nodoc:
+  VERSION = '0.1.1'
   
   autoload_files = %w(
     assignment

data/tasks/gem.rake

@@ -0,0 +1,19 @@
+begin
+  require 'jeweler'
+  
+  Jeweler::Tasks.new do |gem|
+    gem.name = "piglet"
+    gem.summary = %Q{Piglet is a DSL for Pig scripts}
+    gem.description = %Q{Piglet aims to look like Pig Latin while allowing for things like loops and control of flow that are missing from Pig.}
+    gem.email = "theo@iconara.net"
+    gem.homepage = "http://github.com/iconara/piglet"
+    gem.authors = ["Theo Hultberg"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.version = Piglet::VERSION
+    gem.test_files = FileList['spec/**/*.rb']
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+end

data/tasks/rdoc.rake

@@ -0,0 +1,13 @@
+gem 'rdoc', '>= 2.4.0'
+
+require 'rake/rdoctask'
+require 'sdoc' rescue LoadError
+
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "piglet #{Piglet::VERSION}"
+  rdoc.rdoc_files.include('lib/**/*.rb')
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.options << '--charset' << 'utf-8'
+end

data/tasks/spec.rake

@@ -0,0 +1,15 @@
+require 'spec/rake/spectask'
+
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: piglet
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - Theo Hultberg
@@ -79,6 +79,9 @@ files:
 - spec/piglet_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- tasks/gem.rake
+- tasks/rdoc.rake
+- tasks/spec.rake
 has_rdoc: true
 homepage: http://github.com/iconara/piglet
 licenses: []
@@ -114,10 +117,3 @@ test_files:
 - spec/piglet/split_spec.rb
 - spec/piglet_spec.rb
 - spec/spec_helper.rb
-- examples/analysis.rb
-- examples/scratch.rb
-- examples/spike1.rb
-- examples/spike2.rb
-- examples/test1.rb
-- examples/test2.rb
-- examples/test3.rb

data/examples/analysis.rb

@@ -1,311 +0,0 @@
-# raw_sessions = 
-#   LOAD '$INPUT/sessions*'
-#   USING PigStorage AS (
-#     date:chararray,
-#     api_key:chararray,
-#     ad_id:chararray,
-#     user_id:chararray,
-#     site:chararray,
-#     size:chararray,
-#     name:chararray,
-#     destination:chararray,
-#     indeterminate_visibility:int,
-#     impression:int,
-#     engagement:int,
-#     click_thru:int,
-#     extra:int,
-#     session_time:int,
-#     visible_time:int,
-#     engagement_time:int
-#   );
-raw_sessions = load('$INPUT/sessions*', :schema => [
-  [:date :chararray],
-  [:api_key :chararray],
-  [:ad_id :chararray],
-  [:user_id :chararray],
-  [:site :chararray],
-  [:size :chararray],
-  [:name :chararray],
-  [:destination :chararray],
-  [:indeterminate_visibility :int],
-  [:impression :int],
-  [:engagement :int],
-  [:click_thru :int],
-  [:extra :int],
-  [:session_time :int],
-  [:visible_time :int],
-  [:engagement_time :int]
-])
-
-# raw_actions = 
-#   LOAD '$INPUT/actions*'
-#   USING PigStorage AS (
-#     date:chararray,
-#     api_key:chararray,
-#     ad_id:chararray,
-#     user_id:chararray,
-#     action:chararray,
-#     site:chararray,
-#     size:chararray,
-#     name:chararray,
-#     destination:chararray,
-#     extra:int
-#   );
-raw_actions = load('$INPUT/actions*', :schema => 
-  [:date :chararray],
-  [:api_key :chararray],
-  [:ad_id :chararray],
-  [:user_id :chararray],
-  [:action :chararray],
-  [:site :chararray],
-  [:size :chararray],
-  [:name :chararray],
-  [:destination :chararray],
-  [:extra :int]
-)
-
-#sessions = FILTER raw_sessions BY date is not null;
-sessions = raw_sessions.filter { |r| r.date.not_null? }
-
-#actions = FILTER raw_actions  BY date is not null;
-actions = raw_actions.filter { |r| r.date.not_null? }
-
-# /*
-#  * Modify each session and action based on whether or not it's an extra session
-#  * (a session that was logged only because it was a click thru). Extra sessions
-#  * should affect only the total number of click thrus, not the number of
-#  * exposures, impressions, etc. nor the durations. By setting these values to
-#  * zero and introducing a field for whether or not the session was an exposure
-#  * (zero for extra sessions, one for all other), the calculations below can
-#  * filter out extra sessions without too much work. 
-#  */
-# sessions =
-#   FOREACH
-#     sessions
-#   GENERATE
-#     date,
-#     api_key,
-#     ad_id,
-#     user_id,
-#     site,
-#     size,
-#     name,
-#     destination,
-#     (extra == 1 ? 0 : indeterminate_visibility) AS indeterminate_visibility,
-#     (extra == 1 ? 0 : 1) AS exposure,
-#     (extra == 1 ? 0 : impression) AS impression,
-#     (extra == 1 ? 0 : engagement) AS engagement,
-#     click_thru,
-#     (extra == 1 ? 0 : session_time) AS session_time,
-#     (extra == 1 ? 0 : visible_time) AS visible_time,
-#     (extra == 1 ? 0 : engagement_time) AS engagement_time;
-sessions = sessions.foreach do |r|
-  [
-    r.date, 
-    r.api_key, 
-    r.ad_id, 
-    r.user_id,
-    r.site,
-    r.size,
-    r.name,
-    r.destination,
-    r.test(r.extra == 1, 0, r.indeterminate_visibility).as(:indeterminate_visibility),
-    r.test(r.extra == 1, 0, 1).as(:exposure),
-    r.test(r.extra == 1, 0, r.impression).as(:impression),
-    r.test(r.extra == 1, 0, r.engagement).as(:engagement),
-    r.click_thru,
-    r.test(r.extra == 1, 0, r.session_time).as(:session_time),
-    r.test(r.extra == 1, 0, r.visible_time).as(:visible_time),
-    r.test(r.extra == 1, 0, r.engagement_time).as(:engagement_time)
-  ]
-end
-    
-# actions =
-#   FOREACH
-#     actions
-#   GENERATE
-#     date,
-#     api_key,
-#     ad_id,
-#     user_id,
-#     action,
-#     site,
-#     size,
-#     name,
-#     destination,
-#     (extra == 1 ? 0 : 1) AS exposure;
-actions = actions.foreach do |r|
-  [
-    r.date,
-    r.api_key,
-    r.ad_id,
-    r.user_id,
-    r.action,
-    r.site,
-    r.size,
-    r.name,
-    r.destination,
-    r.test(r.extra == 1, 0, 1).as(:exposure)
-  ]
-end
-
-%w(all site size name).each do |name|
-  # session_category_<%= name %> =
-  #   FOREACH 
-  #     (GROUP sessions BY (date, ad_id, api_key, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
-  #   GENERATE 
-  #     $0.date AS date,
-  #     $0.ad_id AS ad_id,
-  #     $0.api_key AS api_key,
-  #     '<%= name %>' AS category,
-  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
-  #     SUM($1.exposure) AS exposures,
-  #     SUM($1.impression) AS impressions,
-  #     SUM($1.engagement) AS engagements,
-  #     SUM($1.click_thru) AS click_thrus,
-  #     SUM($1.indeterminate_visibility) AS indeterminate_visibility,
-  #     SUM($1.session_time) AS session_time,
-  #     SUM($1.visible_time) AS visible_time,
-  #     SUM($1.engagement_time) AS engagement_time;
-  session_category = sessions.group(:date, :ad_id, :api_key, (name == 'all' ? 'all' : name), :parallel => '$PARALLELISM')
-  session_category = session_category.foreach do |r|
-    [
-      r[0].date.as(:date),
-      r[0].ad_id.as(:ad_id),
-      r[0].api_key.as(:api_key),
-      name.as(:category),
-      (name == 'all' ? "'all'" : r[0].name).as(:segment),
-      r[1].sum.as(:exposure),
-      r[1].sum.as(:impression),
-      r[1].sum.as(:engagement),
-      r[1].sum.as(:click_thru),
-      r[1].sum.as(:indeterminate_visibility),
-      r[1].sum.as(:session_time),
-      r[1].sum.as(:visible_time),
-      r[1].sum.as(:engagement_time)
-    ]
-  end
-
-  # session_category_<%= name %>_by_user_id =
-  #   FOREACH
-  #     (GROUP sessions BY (date, ad_id, api_key, user_id, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
-  #   GENERATE
-  #     $0.date AS date,
-  #     $0.ad_id AS ad_id,
-  #     $0.api_key AS api_key,
-  #     '<%= name %>' AS category,
-  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
-  #     1 AS exposures,
-  #     MAX($1.impression) AS impressions,
-  #     MAX($1.engagement) AS engagements,
-  #     MAX($1.click_thru) AS click_thrus;
-  session_category_by_user_id = sessions.group(:date, :ad_id, :api_key, (name == 'all' ? 'all' : name), :parallel => '$PARALLELISM')
-  session_category_by_user_id = session_category_by_user_id.foreach do |r|
-    r[0].date.as(:date),
-    r[0].ad_id.as(:ad_id),
-    r[0].api_key.as(:api_key),
-    name.as(:category),
-    (name == 'all' ? "'all'" : r[0].name).as(:segment)
-    1.as(:exposures),
-    r[1].impression.max.as(:impressions),
-    r[1].engagement.max.as(:engagements),
-    r[1].click_thru.max.as(:click_thrus)
-  end
-
-  # unique_session_category_<%= name %> =
-  #   FOREACH
-  #     (GROUP session_category_<%= name %>_by_user_id BY (date, ad_id, api_key, category, segment) PARALLEL $PARALLELISM)
-  #   GENERATE
-  #     $0.date AS date,
-  #     $0.ad_id AS ad_id,
-  #     $0.api_key AS api_key,
-  #     $0.category,
-  #     $0.segment,
-  #     COUNT($1.ad_id)     AS unique_exposures,
-  #     SUM($1.impressions) AS unique_impressions,
-  #     SUM($1.engagements) AS unique_engagements,
-  #     SUM($1.click_thrus) AS unique_click_thrus;
-  # 
-  # action_category_<%= name %> =
-  #   FOREACH 
-  #     (GROUP actions BY (date, ad_id, api_key, action, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
-  #   GENERATE 
-  #     $0.date AS date,
-  #     $0.ad_id AS ad_id,
-  #     $0.api_key AS api_key,
-  #     $0.action AS action,
-  #     '<%= name %>' AS category,
-  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
-  #     SUM($1.exposure) AS engagements;
-  # 
-  # action_category_<%= name %>_by_user_id =
-  #   FOREACH
-  #     (GROUP actions BY (date, ad_id, api_key, action, user_id, <%= name == 'all' ? "'all'" : name %>) PARALLEL $PARALLELISM)
-  #   GENERATE
-  #     $0.date AS date,
-  #     $0.ad_id AS ad_id,
-  #     $0.api_key AS api_key,
-  #     $0.action AS action,
-  #     '<%= name %>' AS category,
-  #     <%= name == 'all' ? "'all'" : "$0.#{name}" %> AS segment,
-  #     1 AS exposures,
-  #     1 AS engagements;
-  # 
-  # unique_action_category_<%= name %> =
-  #   FOREACH
-  #     (GROUP action_category_<%= name %>_by_user_id BY (date, ad_id, api_key, action, category, segment) PARALLEL $PARALLELISM)
-  #   GENERATE
-  #     $0.date AS date,
-  #     $0.ad_id AS ad_id,
-  #     $0.api_key AS api_key,
-  #     $0.action AS action,
-  #     $0.category,
-  #     $0.segment,
-  #     SUM($1.engagements) AS unique_engagements;
-end
-
--- unions ----------------------------------------------------------------------
--- -----------------------------------------------------------------------------
-
-<% if @categories.size > 1 -%>
-report_metrics =
-  UNION
-    <%= @categories.map { |name| "session_category_#{name}" }.join(",\n    ") %>;
-<% else -%>
-report_metrics = FILTER session_category_<%= @categories.first %> BY 1 == 1;
-<% end -%>
-
-<% if @categories.size > 1 -%>
-unique_report_metrics =
-  UNION
-    <%= @categories.map { |name| "unique_session_category_#{name}" }.join(",\n    ") %>;
-<% else -%>
-unique_report_metrics = FILTER unique_session_category_<%= @categories.first %> BY 1 == 1;
-<% end -%>
-<% if @categories.size > 1 -%>  
-report_action_metrics =
-  UNION
-    <%= @categories.map { |name| "action_category_#{name}" }.join(",\n    ") %>;
-<% else -%>
-report_action_metrics = FILTER action_category_<%= @categories.first %> BY 1 == 1;
-<% end -%>
-<% if @categories.size > 1 -%>  
-unique_report_action_metrics =
-  UNION
-    <%= @categories.map { |name| "unique_action_category_#{name}" }.join(",\n    ") %>;
-<% else -%>
-unique_report_action_metrics = FILTER unique_action_category_<%= @categories.first %> BY 1 == 1;
-<% end %>
-
--- complete output -------------------------------------------------------------
--- -----------------------------------------------------------------------------
-
-
-<% %w(report_metrics unique_report_metrics report_action_metrics unique_report_action_metrics).each do |relation| -%>
-<%= relation %> = FILTER <%= relation %> BY date is not null AND date != '' AND api_key is not null AND api_key != '';
-<% end -%>
-
-STORE report_metrics INTO '$OUTPUT/report_metrics' USING PigStorage;
-STORE unique_report_metrics INTO '$OUTPUT/unique_report_metrics' USING PigStorage;
-STORE report_action_metrics INTO '$OUTPUT/report_action_metrics' USING PigStorage;
-STORE unique_report_action_metrics INTO '$OUTPUT/unique_report_action_metrics' USING PigStorage;

data/examples/scratch.rb

@@ -1,11 +0,0 @@
-module Piglet::Relation
-  def samples(*sizes)
-    sizes.map { |s| sample(s) }
-  end
-end
-
-input = load('input', :schema => %w(country browser site visit_duration))
-a, b, c = input.samples(0.1, 0.2, 0.3)
-store(a, 'output1')
-store(b, 'output2')
-store(c, 'output3')

data/examples/test1.rb

@@ -1,3 +0,0 @@
-raw_data = load 'test1-data.txt', :schema => %w(name city country)
-grouped_by_country = raw_data.group :country
-dump grouped_by_country

data/examples/test2.rb

@@ -1,5 +0,0 @@
-a = load('in', :schema => %w(x y z w))
-%w(x y z w).each do |f|
-  r = a.group(f)
-  store(r, 'out-' + f)
-end

data/examples/test3.rb

@@ -1,4 +0,0 @@
-a = load 'input', :schema => [:a, :b, :c]
-b = a.group :c
-c = b.foreach { |r| [r[0], r[1].a.max, r[1].b.max] }
-store c, 'output'